diff --git a/.clang-format b/.clang-format index 01a341ceec6c..c24b147cac01 100644 --- a/.clang-format +++ b/.clang-format @@ -109,6 +109,7 @@ ForEachMacros: - 'css_for_each_child' - 'css_for_each_descendant_post' - 'css_for_each_descendant_pre' + - 'cxl_for_each_cmd' - 'device_for_each_child_node' - 'dma_fence_chain_for_each' - 'do_for_each_ftrace_op' diff --git a/.gitignore b/.gitignore index d01cda8e1177..3af66272d6f1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ *.c.[012]*.* *.dt.yaml *.dtb +*.dtbo *.dtb.S *.dwo *.elf @@ -41,6 +42,7 @@ *.so.dbg *.su *.symtypes +*.symversions *.tab.[ch] *.tar *.xz diff --git a/.mailmap b/.mailmap index ade90fc9140a..85b93cdefc87 100644 --- a/.mailmap +++ b/.mailmap @@ -174,7 +174,6 @@ Juha Yrjola Juha Yrjola Juha Yrjola Julien Thierry -Kamil Konieczny Kay Sievers Kees Cook Kees Cook @@ -238,6 +237,7 @@ Maxime Ripard Mayuresh Janorkar Michael Buesch Michel Dänzer +Miguel Ojeda Mike Rapoport Mike Rapoport Mike Rapoport diff --git a/CREDITS b/CREDITS index 9add7e6a4fa0..cef83b958cbe 100644 --- a/CREDITS +++ b/CREDITS @@ -1244,10 +1244,10 @@ S: 80050-430 - Curitiba - Paraná S: Brazil N: Oded Gabbay -E: oded.gabbay@gmail.com -D: HabanaLabs and AMD KFD maintainer -S: 12 Shraga Raphaeli -S: Petah-Tikva, 4906418 +E: ogabbay@kernel.org +D: HabanaLabs maintainer +S: 29 Duchifat St. +S: Ra'anana 4372029 S: Israel N: Kumar Gala @@ -2841,14 +2841,11 @@ S: Subiaco, 6008 S: Perth, Western Australia S: Australia -N: Miguel Ojeda Sandonis -E: miguel.ojeda.sandonis@gmail.com -W: http://miguelojeda.es -W: http://jair.lab.fi.uva.es/~migojed/ +N: Miguel Ojeda +E: ojeda@kernel.org +W: https://ojeda.dev D: Author of the ks0108, cfag12864b and cfag12864bfb auxiliary display drivers. D: Maintainer of the auxiliary display drivers tree (drivers/auxdisplay/*) -S: C/ Mieses 20, 9-B -S: Valladolid 47009 S: Spain N: Peter Oruba diff --git a/Documentation/ABI/stable/sysfs-bus-fsl-mc b/Documentation/ABI/stable/sysfs-bus-fsl-mc new file mode 100644 index 000000000000..58f06c7eeed7 --- /dev/null +++ b/Documentation/ABI/stable/sysfs-bus-fsl-mc @@ -0,0 +1,19 @@ +What: /sys/bus/fsl-mc/rescan +Date: January 2021 +KernelVersion: 5.12 +Contact: Ioana Ciornei +Description: Writing a non-zero value to this attribute will + force a rescan of fsl-mc bus in the system and + synchronize the objects under fsl-mc bus and the + Management Complex firmware. +Users: Userspace drivers and management tools + +What: /sys/bus/fsl-mc/autorescan +Date: January 2021 +KernelVersion: 5.12 +Contact: Ioana Ciornei +Description: Writing a zero value to this attribute will + disable the DPRC IRQs on which automatic rescan + of the fsl-mc bus is performed. A non-zero value + will enable the DPRC IRQs. +Users: Userspace drivers and management tools diff --git a/Documentation/ABI/stable/sysfs-class-tpm b/Documentation/ABI/stable/sysfs-class-tpm index 91ca63ec7581..d897ecb9615f 100644 --- a/Documentation/ABI/stable/sysfs-class-tpm +++ b/Documentation/ABI/stable/sysfs-class-tpm @@ -194,3 +194,17 @@ Description: The "tpm_version_major" property shows the TCG spec major version Example output:: 2 + +What: /sys/class/tpm/tpmX/pcr-H/N +Date: March 2021 +KernelVersion: 5.12 +Contact: linux-integrity@vger.kernel.org +Description: produces output in compact hex representation for PCR + number N from hash bank H. N is the numeric value of + the PCR number and H is the crypto string + representation of the hash + + Example output:: + + cat /sys/class/tpm/tpm0/pcr-sha256/7 + 2ED93F199692DC6788EFA6A1FE74514AB9760B2A6CEEAEF6C808C13E4ABB0D42 diff --git a/Documentation/ABI/stable/sysfs-driver-speakup b/Documentation/ABI/stable/sysfs-driver-speakup index 792f58ba327d..dc2a6ba1674b 100644 --- a/Documentation/ABI/stable/sysfs-driver-speakup +++ b/Documentation/ABI/stable/sysfs-driver-speakup @@ -273,7 +273,7 @@ Description: In `/sys/accessibility/speakup` is a directory corresponding to Below is a description of values and parameters for soft synthesizer, which is currently the most commonly used. -What: /sys/accessibility/speakup/soft/caps_start +What: /sys/accessibility/speakup//caps_start KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: This is the string that is sent to the synthesizer to cause it @@ -281,7 +281,7 @@ Description: This is the string that is sent to the synthesizer to cause it and most others, this causes the pitch of the voice to rise above the currently set pitch. -What: /sys/accessibility/speakup/soft/caps_stop +What: /sys/accessibility/speakup//caps_stop KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: This is the string sent to the synthesizer to cause it to stop @@ -290,12 +290,12 @@ Description: This is the string sent to the synthesizer to cause it to stop down to the currently set pitch. -What: /sys/accessibility/speakup/soft/delay_time +What: /sys/accessibility/speakup//delay_time KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: TODO: -What: /sys/accessibility/speakup/soft/direct +What: /sys/accessibility/speakup//direct KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Controls if punctuation is spoken by speakup, or by the @@ -306,36 +306,43 @@ Description: Controls if punctuation is spoken by speakup, or by the than". Zero lets speakup speak the punctuation. One lets the synthesizer itself speak punctuation. -What: /sys/accessibility/speakup/soft/freq +What: /sys/accessibility/speakup//freq KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the frequency of the speech synthesizer. Range is 0-9. -What: /sys/accessibility/speakup/soft/full_time +What: /sys/accessibility/speakup//flush_time +KernelVersion: 5.12 +Contact: speakup@linux-speakup.org +Description: Gets or sets the timeout to wait for the synthesizer flush to + complete. This can be used when the cable gets faulty and flush + notifications are getting lost. + +What: /sys/accessibility/speakup//full_time KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: TODO: -What: /sys/accessibility/speakup/soft/jiffy_delta +What: /sys/accessibility/speakup//jiffy_delta KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: This controls how many jiffys the kernel gives to the synthesizer. Setting this too high can make a system unstable, or even crash it. -What: /sys/accessibility/speakup/soft/pitch +What: /sys/accessibility/speakup//pitch KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the pitch of the synthesizer. The range is 0-9. -What: /sys/accessibility/speakup/soft/inflection +What: /sys/accessibility/speakup//inflection KernelVersion: 5.8 Contact: speakup@linux-speakup.org Description: Gets or sets the inflection of the synthesizer, i.e. the pitch range. The range is 0-9. -What: /sys/accessibility/speakup/soft/punct +What: /sys/accessibility/speakup//punct KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the amount of punctuation spoken by the @@ -343,13 +350,13 @@ Description: Gets or sets the amount of punctuation spoken by the TODO: How is this related to speakup's punc_level, or reading_punc. -What: /sys/accessibility/speakup/soft/rate +What: /sys/accessibility/speakup//rate KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the rate of the synthesizer. Range is from zero slowest, to nine fastest. -What: /sys/accessibility/speakup/soft/tone +What: /sys/accessibility/speakup//tone KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the tone of the speech synthesizer. The range for @@ -357,12 +364,12 @@ Description: Gets or sets the tone of the speech synthesizer. The range for difference if using espeak and the espeakup connector. TODO: does espeakup support different tonalities? -What: /sys/accessibility/speakup/soft/trigger_time +What: /sys/accessibility/speakup//trigger_time KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: TODO: -What: /sys/accessibility/speakup/soft/voice +What: /sys/accessibility/speakup//voice KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the voice used by the synthesizer if the @@ -371,7 +378,7 @@ Description: Gets or sets the voice used by the synthesizer if the voices, this parameter will not set the voice when the espeakup connector is used between speakup and espeak. -What: /sys/accessibility/speakup/soft/vol +What: /sys/accessibility/speakup//vol KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the volume of the speech synthesizer. Range is 0-9, diff --git a/Documentation/ABI/testing/debugfs-driver-habanalabs b/Documentation/ABI/testing/debugfs-driver-habanalabs index c5d678d39144..d447a611c41b 100644 --- a/Documentation/ABI/testing/debugfs-driver-habanalabs +++ b/Documentation/ABI/testing/debugfs-driver-habanalabs @@ -1,7 +1,7 @@ What: /sys/kernel/debug/habanalabs/hl/addr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the device address to be used for read or write through PCI bar, or the device VA of a host mapped memory to be read or written directly from the host. The latter option is allowed @@ -11,7 +11,7 @@ Description: Sets the device address to be used for read or write through What: /sys/kernel/debug/habanalabs/hl/clk_gate Date: May 2020 KernelVersion: 5.8 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allow the root user to disable/enable in runtime the clock gating mechanism in Gaudi. Due to how Gaudi is built, the clock gating needs to be disabled in order to access the @@ -34,28 +34,28 @@ Description: Allow the root user to disable/enable in runtime the clock What: /sys/kernel/debug/habanalabs/hl/command_buffers Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about the currently allocated command buffers What: /sys/kernel/debug/habanalabs/hl/command_submission Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about the currently active command submissions What: /sys/kernel/debug/habanalabs/hl/command_submission_jobs Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with detailed information about each JOB (CB) of each active command submission What: /sys/kernel/debug/habanalabs/hl/data32 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the root user to read or write directly through the device's PCI bar. Writing to this file generates a write transaction while reading from the file generates a read @@ -70,7 +70,7 @@ Description: Allows the root user to read or write directly through the What: /sys/kernel/debug/habanalabs/hl/data64 Date: Jan 2020 KernelVersion: 5.6 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the root user to read or write 64 bit data directly through the device's PCI bar. Writing to this file generates a write transaction while reading from the file generates a read @@ -85,7 +85,7 @@ Description: Allows the root user to read or write 64 bit data directly What: /sys/kernel/debug/habanalabs/hl/device Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Enables the root user to set the device to specific state. Valid values are "disable", "enable", "suspend", "resume". User can read this property to see the valid values @@ -93,28 +93,28 @@ Description: Enables the root user to set the device to specific state. What: /sys/kernel/debug/habanalabs/hl/engines Date: Jul 2019 KernelVersion: 5.3 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the status registers values of the device engines and their derived idle status What: /sys/kernel/debug/habanalabs/hl/i2c_addr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets I2C device address for I2C transaction that is generated by the device's CPU What: /sys/kernel/debug/habanalabs/hl/i2c_bus Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets I2C bus address for I2C transaction that is generated by the device's CPU What: /sys/kernel/debug/habanalabs/hl/i2c_data Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Triggers an I2C transaction that is generated by the device's CPU. Writing to this file generates a write transaction while reading from the file generates a read transcation @@ -122,32 +122,32 @@ Description: Triggers an I2C transaction that is generated by the device's What: /sys/kernel/debug/habanalabs/hl/i2c_reg Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets I2C register id for I2C transaction that is generated by the device's CPU What: /sys/kernel/debug/habanalabs/hl/led0 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the state of the first S/W led on the device What: /sys/kernel/debug/habanalabs/hl/led1 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the state of the second S/W led on the device What: /sys/kernel/debug/habanalabs/hl/led2 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the state of the third S/W led on the device What: /sys/kernel/debug/habanalabs/hl/mmu Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the hop values and physical address for a given ASID and virtual address. The user should write the ASID and VA into the file and then read the file to get the result. @@ -157,14 +157,14 @@ Description: Displays the hop values and physical address for a given ASID What: /sys/kernel/debug/habanalabs/hl/set_power_state Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the PCI power state. Valid values are "1" for D0 and "2" for D3Hot What: /sys/kernel/debug/habanalabs/hl/userptr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about the currently user pointers (user virtual addresses) that are pinned and mapped to DMA addresses @@ -172,13 +172,21 @@ Description: Displays a list with information about the currently user What: /sys/kernel/debug/habanalabs/hl/vm Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about all the active virtual address mappings per ASID What: /sys/kernel/debug/habanalabs/hl/stop_on_err Date: Mar 2020 KernelVersion: 5.6 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the stop-on_error option for the device engines. Value of "0" is for disable, otherwise enable. + +What: /sys/kernel/debug/habanalabs/hl/dump_security_violations +Date: Jan 2021 +KernelVersion: 5.12 +Contact: ogabbay@kernel.org +Description: Dumps all security violations to dmesg. This will also ack + all security violations meanings those violations will not be + dumped next time user calls this API diff --git a/Documentation/ABI/testing/ima_policy b/Documentation/ABI/testing/ima_policy index e35263f97fc1..070779e8d836 100644 --- a/Documentation/ABI/testing/ima_policy +++ b/Documentation/ABI/testing/ima_policy @@ -29,10 +29,10 @@ Description: option: [[appraise_type=]] [template=] [permit_directio] [appraise_flag=] [keyrings=] base: - func:= [BPRM_CHECK][MMAP_CHECK][CREDS_CHECK][FILE_CHECK]MODULE_CHECK] + func:= [BPRM_CHECK][MMAP_CHECK][CREDS_CHECK][FILE_CHECK][MODULE_CHECK] [FIRMWARE_CHECK] [KEXEC_KERNEL_CHECK] [KEXEC_INITRAMFS_CHECK] - [KEXEC_CMDLINE] [KEY_CHECK] + [KEXEC_CMDLINE] [KEY_CHECK] [CRITICAL_DATA] mask:= [[^]MAY_READ] [[^]MAY_WRITE] [[^]MAY_APPEND] [[^]MAY_EXEC] fsmagic:= hex value @@ -52,6 +52,9 @@ Description: template:= name of a defined IMA template type (eg, ima-ng). Only valid when action is "measure". pcr:= decimal value + label:= [selinux]|[kernel_info]|[data_label] + data_label:= a unique string used for grouping and limiting critical data. + For example, "selinux" to measure critical data for SELinux. default policy: # PROC_SUPER_MAGIC diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x b/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x index 881f0cd99ce4..8e53a32f8150 100644 --- a/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x +++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x @@ -371,6 +371,14 @@ Contact: Mathieu Poirier Description: (Read) Print the content of the Device ID Register (0xFC8). The value is taken directly from the HW. +What: /sys/bus/coresight/devices/etm/mgmt/trcdevarch +Date: January 2021 +KernelVersion: 5.12 +Contact: Mathieu Poirier +Description: (Read) Print the content of the Device Architecture Register + (offset 0xFBC). The value is taken directly read + from the HW. + What: /sys/bus/coresight/devices/etm/mgmt/trcdevtype Date: April 2015 KernelVersion: 4.01 diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl new file mode 100644 index 000000000000..2fe7490ad6a8 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -0,0 +1,26 @@ +What: /sys/bus/cxl/devices/memX/firmware_version +Date: December, 2020 +KernelVersion: v5.12 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) "FW Revision" string as reported by the Identify + Memory Device Output Payload in the CXL-2.0 + specification. + +What: /sys/bus/cxl/devices/memX/ram/size +Date: December, 2020 +KernelVersion: v5.12 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) "Volatile Only Capacity" as bytes. Represents the + identically named field in the Identify Memory Device Output + Payload in the CXL-2.0 specification. + +What: /sys/bus/cxl/devices/memX/pmem/size +Date: December, 2020 +KernelVersion: v5.12 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) "Persistent Only Capacity" as bytes. Represents the + identically named field in the Identify Memory Device Output + Payload in the CXL-2.0 specification. diff --git a/Documentation/ABI/testing/sysfs-bus-dfl-devices-emif b/Documentation/ABI/testing/sysfs-bus-dfl-devices-emif new file mode 100644 index 000000000000..817d14126d4d --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-dfl-devices-emif @@ -0,0 +1,25 @@ +What: /sys/bus/dfl/devices/dfl_dev.X/infX_cal_fail +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. It indicates if the calibration failed on this + memory interface. "1" for calibration failure, "0" for OK. + Format: %u + +What: /sys/bus/dfl/devices/dfl_dev.X/infX_init_done +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. It indicates if the initialization completed on + this memory interface. "1" for initialization complete, "0" + for not yet. + Format: %u + +What: /sys/bus/dfl/devices/dfl_dev.X/infX_clear +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Write-only. Writing "1" to this file will zero out all memory + data in this memory interface. Writing of other values is + invalid. + Format: %u diff --git a/Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios b/Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios new file mode 100644 index 000000000000..5335d742bcaf --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios @@ -0,0 +1,47 @@ +What: /sys/bus/dfl/devices/dfl_dev.X/fec_mode +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the FEC mode of the 25G links of the + ethernet retimers configured by Nios firmware. "rs" for Reed + Solomon FEC, "kr" for Fire Code FEC, "no" for NO FEC. + "not supported" if the FEC mode setting is not supported, this + happens when the Nios firmware version major < 3, or no link is + configured to 25G. + Format: string + +What: /sys/bus/dfl/devices/dfl_dev.X/retimer_A_mode +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the enumeration value of the working mode of + the retimer A configured by the Nios firmware. The value is + read out from shared registers filled by the Nios firmware. Now + the values could be: + + - "0": Reset + - "1": 4x10G + - "2": 4x25G + - "3": 2x25G + - "4": 2x25G+2x10G + - "5": 1x25G + + If the Nios firmware is updated in future to support more + retimer modes, more enumeration value is expected. + Format: 0x%x + +What: /sys/bus/dfl/devices/dfl_dev.X/retimer_B_mode +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the enumeration value of the working mode of + the retimer B configured by the Nios firmware. The value format + is the same as retimer_A_mode. + +What: /sys/bus/dfl/devices/dfl_dev.X/nios_fw_version +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the version of the Nios firmware in the + FPGA. Its format is "major.minor.patch". + Format: %x.%x.%x diff --git a/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic b/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic new file mode 100644 index 000000000000..1936f7324155 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic @@ -0,0 +1,24 @@ +What: /sys/devices/pci0000:00/*/QEMU0001:00/capability +Date: Jan 2021 +Contact: zhenwei pi +Description: + Read-only attribute. Capabilities of pvpanic device which + are supported by QEMU. + + Format: %x. + + Detailed bit definition refers to section + from pvpanic device specification: + https://git.qemu.org/?p=qemu.git;a=blob_plain;f=docs/specs/pvpanic.txt + +What: /sys/devices/pci0000:00/*/QEMU0001:00/events +Date: Jan 2021 +Contact: zhenwei pi +Description: + RW attribute. Set/get which features in-use. This attribute + is used to enable/disable feature(s) of pvpanic device. + Notice that this value should be a subset of capability. + + Format: %x. + + Also refer to pvpanic device specification. diff --git a/Documentation/ABI/testing/sysfs-class-power-ltc4162l b/Documentation/ABI/testing/sysfs-class-power-ltc4162l new file mode 100644 index 000000000000..ba30db93052b --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-power-ltc4162l @@ -0,0 +1,82 @@ +What: /sys/class/power_supply/ltc4162-l/charge_status +Date: Januari 2021 +KernelVersion: 5.11 +Description: + Detailed charge status information as reported by the chip. + + Access: Read + + Valid values: + ilim_reg_active + thermal_reg_active + vin_uvcl_active + iin_limit_active + constant_current + constant_voltage + charger_off + +What: /sys/class/power_supply/ltc4162-l/ibat +Date: Januari 2021 +KernelVersion: 5.11 +Description: + Battery input current as measured by the charger. Negative value + means that the battery is discharging. + + Access: Read + + Valid values: Signed value in microamps + +What: /sys/class/power_supply/ltc4162-l/vbat +Date: Januari 2021 +KernelVersion: 5.11 +Description: + Battery voltage as measured by the charger. + + Access: Read + + Valid values: In microvolts + +What: /sys/class/power_supply/ltc4162-l/vbat_avg +Date: Januari 2021 +KernelVersion: 5.11 +Description: + Battery voltage, averaged over time, as measured by the charger. + + Access: Read + + Valid values: In microvolts + +What: /sys/class/power_supply/ltc4162-l/force_telemetry +Date: Januari 2021 +KernelVersion: 5.11 +Description: + To save battery current, the measurement system is disabled if + the battery is the only source of power. This affects all + voltage, current and temperature measurements. + Write a "1" to this to keep performing telemetry once every few + seconds, even when running on battery (as reported by the online + property, which is "1" when external power is available and "0" + when the system runs on battery). + + Access: Read, Write + + Valid values: 0 (disabled) or 1 (enabled) + +What: /sys/class/power_supply/ltc4162-l/arm_ship_mode +Date: Januari 2021 +KernelVersion: 5.11 +Description: + The charger will normally drain the battery while inactive, + typically drawing about 54 microamps. Write a "1" to this + property to arm a special "ship" mode that extends shelf life + by reducing the leakage to about 2.8 microamps. The chip will + remain in this mode (and no longer respond to I2C commands) + until some external power-supply is attached raising the input + voltage above 1V. It will then automatically revert to "0". + Writing a "0" to the property cancels the "ship" mode request. + The ship mode, when armed, activates once the input voltage + drops below 1V. + + Access: Read, Write + + Valid values: 0 (disable) or 1 (enable) diff --git a/Documentation/ABI/testing/sysfs-devices-memory b/Documentation/ABI/testing/sysfs-devices-memory index 246a45b96d22..d8b0f80b9e33 100644 --- a/Documentation/ABI/testing/sysfs-devices-memory +++ b/Documentation/ABI/testing/sysfs-devices-memory @@ -13,21 +13,22 @@ What: /sys/devices/system/memory/memoryX/removable Date: June 2008 Contact: Badari Pulavarty Description: - The file /sys/devices/system/memory/memoryX/removable - indicates whether this memory block is removable or not. - This is useful for a user-level agent to determine - identify removable sections of the memory before attempting - potentially expensive hot-remove memory operation + The file /sys/devices/system/memory/memoryX/removable is a + legacy interface used to indicated whether a memory block is + likely to be offlineable or not. Newer kernel versions return + "1" if and only if the kernel supports memory offlining. Users: hotplug memory remove tools http://www.ibm.com/developerworks/wikis/display/LinuxP/powerpc-utils + lsmem/chmem part of util-linux What: /sys/devices/system/memory/memoryX/phys_device Date: September 2008 Contact: Badari Pulavarty Description: The file /sys/devices/system/memory/memoryX/phys_device - is read-only and is designed to show the name of physical - memory device. Implementation is currently incomplete. + is read-only; it is a legacy interface only ever used on s390x + to expose the covered storage increment. +Users: Legacy s390-tools lsmem/chmem What: /sys/devices/system/memory/memoryX/phys_index Date: September 2008 @@ -43,23 +44,25 @@ Date: September 2008 Contact: Badari Pulavarty Description: The file /sys/devices/system/memory/memoryX/state - is read-write. When read, its contents show the - online/offline state of the memory section. When written, - root can toggle the the online/offline state of a removable - memory section (see removable file description above) - using the following commands:: + is read-write. When read, it returns the online/offline + state of the memory block. When written, root can toggle + the online/offline state of a memory block using the following + commands:: # echo online > /sys/devices/system/memory/memoryX/state # echo offline > /sys/devices/system/memory/memoryX/state - For example, if /sys/devices/system/memory/memory22/removable - contains a value of 1 and - /sys/devices/system/memory/memory22/state contains the - string "online" the following command can be executed by - by root to offline that section:: - - # echo offline > /sys/devices/system/memory/memory22/state + On newer kernel versions, advanced states can be specified + when onlining to select a target zone: "online_movable" + selects the movable zone. "online_kernel" selects the + applicable kernel zone (DMA, DMA32, or Normal). However, + after successfully setting one of the advanced states, + reading the file will return "online"; the zone information + can be obtained via "valid_zones" instead. + While onlining is unlikely to fail, there are no guarantees + that offlining will succeed. Offlining is more likely to + succeed if "valid_zones" indicates "Movable". Users: hotplug memory remove tools http://www.ibm.com/developerworks/wikis/display/LinuxP/powerpc-utils @@ -69,8 +72,19 @@ Date: July 2014 Contact: Zhang Zhen Description: The file /sys/devices/system/memory/memoryX/valid_zones is - read-only and is designed to show which zone this memory - block can be onlined to. + read-only. + + For online memory blocks, it returns in which zone memory + provided by a memory block is managed. If multiple zones + apply (not applicable for hotplugged memory), "None" is returned + and the memory block cannot be offlined. + + For offline memory blocks, it returns by which zone memory + provided by a memory block can be managed when onlining. + The first returned zone ("default") will be used when setting + the state of an offline memory block to "online". Only one of + the kernel zones (DMA, DMA32, Normal) is applicable for a single + memory block. What: /sys/devices/system/memoryX/nodeY Date: October 2009 diff --git a/Documentation/ABI/testing/sysfs-devices-xenbus b/Documentation/ABI/testing/sysfs-devices-xenbus new file mode 100644 index 000000000000..fd796cb4f315 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-xenbus @@ -0,0 +1,41 @@ +What: /sys/devices/*/xenbus/event_channels +Date: February 2021 +Contact: Xen Developers mailing list +Description: + Number of Xen event channels associated with a kernel based + paravirtualized device frontend or backend. + +What: /sys/devices/*/xenbus/events +Date: February 2021 +Contact: Xen Developers mailing list +Description: + Total number of Xen events received for a Xen pv device + frontend or backend. + +What: /sys/devices/*/xenbus/jiffies_eoi_delayed +Date: February 2021 +Contact: Xen Developers mailing list +Description: + Summed up time in jiffies the EOI of an interrupt for a Xen + pv device has been delayed in order to avoid stalls due to + event storms. This value rising is a first sign for a rogue + other end of the pv device. + +What: /sys/devices/*/xenbus/spurious_events +Date: February 2021 +Contact: Xen Developers mailing list +Description: + Number of events received for a Xen pv device which did not + require any action. Too many spurious events in a row will + trigger delayed EOI processing. + +What: /sys/devices/*/xenbus/spurious_threshold +Date: February 2021 +Contact: Xen Developers mailing list +Description: + Controls the tolerated number of subsequent spurious events + before delayed EOI processing is triggered for a Xen pv + device. Default is 1. This can be modified in case the other + end of the pv device is issuing spurious events on a regular + basis and is known not to be malicious on purpose. Raising + the value for such cases can improve pv device performance. diff --git a/Documentation/ABI/testing/sysfs-driver-habanalabs b/Documentation/ABI/testing/sysfs-driver-habanalabs index 169ae4b2a180..1f127f71d2b4 100644 --- a/Documentation/ABI/testing/sysfs-driver-habanalabs +++ b/Documentation/ABI/testing/sysfs-driver-habanalabs @@ -1,7 +1,7 @@ What: /sys/class/habanalabs/hl/armcp_kernel_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Linux kernel running on the device's CPU. Will be DEPRECATED in Linux kernel version 5.10, and be replaced with cpucp_kernel_ver @@ -9,7 +9,7 @@ Description: Version of the Linux kernel running on the device's CPU. What: /sys/class/habanalabs/hl/armcp_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the application running on the device's CPU Will be DEPRECATED in Linux kernel version 5.10, and be replaced with cpucp_ver @@ -17,7 +17,7 @@ Description: Version of the application running on the device's CPU What: /sys/class/habanalabs/hl/clk_max_freq_mhz Date: Jun 2019 KernelVersion: not yet upstreamed -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in MHz. The device clock might be set to lower value than the maximum. The user should read the clk_cur_freq_mhz to see the actual @@ -27,52 +27,52 @@ Description: Allows the user to set the maximum clock frequency, in MHz. What: /sys/class/habanalabs/hl/clk_cur_freq_mhz Date: Jun 2019 KernelVersion: not yet upstreamed -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current frequency, in MHz, of the device clock. This property is valid only for the Gaudi ASIC family What: /sys/class/habanalabs/hl/cpld_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Device's CPLD F/W What: /sys/class/habanalabs/hl/cpucp_kernel_ver Date: Oct 2020 KernelVersion: 5.10 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Linux kernel running on the device's CPU What: /sys/class/habanalabs/hl/cpucp_ver Date: Oct 2020 KernelVersion: 5.10 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the application running on the device's CPU What: /sys/class/habanalabs/hl/device_type Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the code name of the device according to its type. The supported values are: "GOYA" What: /sys/class/habanalabs/hl/eeprom Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: A binary file attribute that contains the contents of the on-board EEPROM What: /sys/class/habanalabs/hl/fuse_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the device's version from the eFuse What: /sys/class/habanalabs/hl/hard_reset Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Interface to trigger a hard-reset operation for the device. Hard-reset will reset ALL internal components of the device except for the PCI interface and the internal PLLs @@ -80,14 +80,14 @@ Description: Interface to trigger a hard-reset operation for the device. What: /sys/class/habanalabs/hl/hard_reset_cnt Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays how many times the device have undergone a hard-reset operation since the driver was loaded What: /sys/class/habanalabs/hl/high_pll Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency for MME, TPC and IC when the power management profile is set to "automatic". This property is valid only for the Goya ASIC family @@ -95,7 +95,7 @@ Description: Allows the user to set the maximum clock frequency for MME, TPC What: /sys/class/habanalabs/hl/ic_clk Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in Hz, of the Interconnect fabric. Writes to this parameter affect the device only when the power management profile is set to "manual" @@ -107,27 +107,27 @@ Description: Allows the user to set the maximum clock frequency, in Hz, of What: /sys/class/habanalabs/hl/ic_clk_curr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current clock frequency, in Hz, of the Interconnect fabric. This property is valid only for the Goya ASIC family What: /sys/class/habanalabs/hl/infineon_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Device's power supply F/W code What: /sys/class/habanalabs/hl/max_power Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum power consumption of the device in milliwatts. What: /sys/class/habanalabs/hl/mme_clk Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in Hz, of the MME compute engine. Writes to this parameter affect the device only when the power management profile is set to "manual" @@ -139,21 +139,21 @@ Description: Allows the user to set the maximum clock frequency, in Hz, of What: /sys/class/habanalabs/hl/mme_clk_curr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current clock frequency, in Hz, of the MME compute engine. This property is valid only for the Goya ASIC family What: /sys/class/habanalabs/hl/pci_addr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the PCI address of the device. This is needed so the user would be able to open a device based on its PCI address What: /sys/class/habanalabs/hl/pm_mng_profile Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Power management profile. Values are "auto", "manual". In "auto" mode, the driver will set the maximum clock frequency to a high value when a user-space process opens the device's file (unless @@ -167,13 +167,13 @@ Description: Power management profile. Values are "auto", "manual". In "auto" What: /sys/class/habanalabs/hl/preboot_btl_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the device's preboot F/W code What: /sys/class/habanalabs/hl/soft_reset Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Interface to trigger a soft-reset operation for the device. Soft-reset will reset only the compute and DMA engines of the device @@ -181,26 +181,26 @@ Description: Interface to trigger a soft-reset operation for the device. What: /sys/class/habanalabs/hl/soft_reset_cnt Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays how many times the device have undergone a soft-reset operation since the driver was loaded What: /sys/class/habanalabs/hl/status Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Status of the card: "Operational", "Malfunction", "In reset". What: /sys/class/habanalabs/hl/thermal_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Device's thermal daemon What: /sys/class/habanalabs/hl/tpc_clk Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in Hz, of the TPC compute engines. Writes to this parameter affect the device only when the power management profile is set to "manual" @@ -212,12 +212,12 @@ Description: Allows the user to set the maximum clock frequency, in Hz, of What: /sys/class/habanalabs/hl/tpc_clk_curr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current clock frequency, in Hz, of the TPC compute engines. This property is valid only for the Goya ASIC family What: /sys/class/habanalabs/hl/uboot_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the u-boot running on the device's CPU \ No newline at end of file diff --git a/Documentation/ABI/testing/sysfs-driver-input-cros-ec-keyb b/Documentation/ABI/testing/sysfs-driver-input-cros-ec-keyb new file mode 100644 index 000000000000..c7afc2328045 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-input-cros-ec-keyb @@ -0,0 +1,6 @@ +What: /sys/class/input/input(x)/device/function_row_physmap +Date: January 2021 +Contact: Philip Chen +Description: A space separated list of scancodes for the top row keys, + ordered by the physical positions of the keys, from left + to right. diff --git a/Documentation/ABI/testing/sysfs-driver-intel-m10-bmc b/Documentation/ABI/testing/sysfs-driver-intel-m10-bmc index 979a2d62513f..9773925138af 100644 --- a/Documentation/ABI/testing/sysfs-driver-intel-m10-bmc +++ b/Documentation/ABI/testing/sysfs-driver-intel-m10-bmc @@ -13,3 +13,24 @@ Contact: Xu Yilun Description: Read only. Returns the firmware version of Intel MAX10 BMC chip. Format: "0x%x". + +What: /sys/bus/spi/devices/.../mac_address +Date: January 2021 +KernelVersion: 5.12 +Contact: Russ Weight +Description: Read only. Returns the first MAC address in a block + of sequential MAC addresses assigned to the board + that is managed by the Intel MAX10 BMC. It is stored in + FLASH storage and is mirrored in the MAX10 BMC register + space. + Format: "%02x:%02x:%02x:%02x:%02x:%02x". + +What: /sys/bus/spi/devices/.../mac_count +Date: January 2021 +KernelVersion: 5.12 +Contact: Russ Weight +Description: Read only. Returns the number of sequential MAC + addresses assigned to the board managed by the Intel + MAX10 BMC. This value is stored in FLASH and is mirrored + in the MAX10 BMC register space. + Format: "%u". diff --git a/Documentation/ABI/testing/sysfs-driver-ufs b/Documentation/ABI/testing/sysfs-driver-ufs index 75ccc5c62b3c..d1bc23cb6a9d 100644 --- a/Documentation/ABI/testing/sysfs-driver-ufs +++ b/Documentation/ABI/testing/sysfs-driver-ufs @@ -1161,3 +1161,14 @@ Description: This entry shows the configured size of WriteBooster buffer. 0400h corresponds to 4GB. The file is read only. + +What: /sys/bus/platform/drivers/ufshcd/*/wb_on +Date: January 2021 +Contact: Bean Huo +Description: This node is used to set or display whether UFS WriteBooster is + enabled. Echo 0 to this file to disable UFS WriteBooster or 1 to + enable it. The WriteBooster is enabled after power-on/reset, + however, it will be disabled/enable while CLK scaling down/up + (if the platform supports UFSHCD_CAP_CLK_SCALING). For a + platform that doesn't support UFSHCD_CAP_CLK_SCALING, we can + disable/enable WriteBooster through this sysfs node. diff --git a/Documentation/ABI/testing/sysfs-firmware-acpi b/Documentation/ABI/testing/sysfs-firmware-acpi index b16d30a71709..819939d858c9 100644 --- a/Documentation/ABI/testing/sysfs-firmware-acpi +++ b/Documentation/ABI/testing/sysfs-firmware-acpi @@ -1,3 +1,46 @@ +What: /sys/firmware/acpi/fpdt/ +Date: Jan 2021 +Contact: Zhang Rui +Description: + ACPI Firmware Performance Data Table (FPDT) provides + information for firmware performance data for system boot, + S3 suspend and S3 resume. This sysfs entry contains the + performance data retrieved from the FPDT. + + boot: + firmware_start_ns: Timer value logged at the beginning + of firmware image execution. In nanoseconds. + bootloader_load_ns: Timer value logged just prior to + loading the OS boot loader into memory. + In nanoseconds. + bootloader_launch_ns: Timer value logged just prior to + launching the currently loaded OS boot loader + image. In nanoseconds. + exitbootservice_start_ns: Timer value logged at the + point when the OS loader calls the + ExitBootServices function for UEFI compatible + firmware. In nanoseconds. + exitbootservice_end_ns: Timer value logged at the point + just prior to the OS loader gaining control + back from the ExitBootServices function for + UEFI compatible firmware. In nanoseconds. + suspend: + suspend_start_ns: Timer value recorded at the previous + OS write to SLP_TYP upon entry to S3. In + nanoseconds. + suspend_end_ns: Timer value recorded at the previous + firmware write to SLP_TYP used to trigger + hardware entry to S3. In nanoseconds. + resume: + resume_count: A count of the number of S3 resume cycles + since the last full boot sequence. + resume_avg_ns: Average timer value of all resume cycles + logged since the last full boot sequence, + including the most recent resume. In nanoseconds. + resume_prev_ns: Timer recorded at the end of the previous + platform runtime firmware S3 resume, just prior to + handoff to the OS waking vector. In nanoseconds. + What: /sys/firmware/acpi/bgrt/ Date: January 2012 Contact: Matthew Garrett diff --git a/Documentation/ABI/testing/sysfs-firmware-sfi b/Documentation/ABI/testing/sysfs-firmware-sfi deleted file mode 100644 index 5210e0f06ddb..000000000000 --- a/Documentation/ABI/testing/sysfs-firmware-sfi +++ /dev/null @@ -1,15 +0,0 @@ -What: /sys/firmware/sfi/tables/ -Date: May 2010 -Contact: Len Brown -Description: - SFI defines a number of small static memory tables - so the kernel can get platform information from firmware. - - The tables are defined in the latest SFI specification: - http://simplefirmware.org/documentation - - While the tables are used by the kernel, user-space - can observe them this way:: - - # cd /sys/firmware/sfi/tables - # cat $TABLENAME > $TABLENAME.bin diff --git a/Documentation/ABI/testing/sysfs-platform-ideapad-laptop b/Documentation/ABI/testing/sysfs-platform-ideapad-laptop index fd2ac02bc5bd..4989ab266682 100644 --- a/Documentation/ABI/testing/sysfs-platform-ideapad-laptop +++ b/Documentation/ABI/testing/sysfs-platform-ideapad-laptop @@ -1,11 +1,11 @@ -What: /sys/devices/platform/ideapad/camera_power +What: /sys/bus/platform/devices/VPC2004:*/camera_power Date: Dec 2010 KernelVersion: 2.6.37 Contact: "Ike Panhc " Description: Control the power of camera module. 1 means on, 0 means off. -What: /sys/devices/platform/ideapad/fan_mode +What: /sys/bus/platform/devices/VPC2004:*/fan_mode Date: June 2012 KernelVersion: 3.6 Contact: "Maxim Mikityanskiy " @@ -18,7 +18,7 @@ Description: * 2 -> Dust Cleaning * 4 -> Efficient Thermal Dissipation Mode -What: /sys/devices/platform/ideapad/touchpad +What: /sys/bus/platform/devices/VPC2004:*/touchpad Date: May 2017 KernelVersion: 4.13 Contact: "Ritesh Raj Sarraf " @@ -27,7 +27,16 @@ Description: * 1 -> Switched On * 0 -> Switched Off -What: /sys/bus/pci/devices///VPC2004:00/fn_lock +What: /sys/bus/platform/devices/VPC2004:*/conservation_mode +Date: Aug 2017 +KernelVersion: 4.14 +Contact: platform-driver-x86@vger.kernel.org +Description: + Controls whether the conservation mode is enabled or not. + This feature limits the maximum battery charge percentage to + around 50-60% in order to prolong the lifetime of the battery. + +What: /sys/bus/platform/devices/VPC2004:*/fn_lock Date: May 2018 KernelVersion: 4.18 Contact: "Oleg Keri " @@ -41,3 +50,12 @@ Description: # echo "0" > \ /sys/bus/pci/devices/0000:00:1f.0/PNP0C09:00/VPC2004:00/fn_lock + +What: /sys/bus/platform/devices/VPC2004:*/usb_charging +Date: Feb 2021 +KernelVersion: 5.12 +Contact: platform-driver-x86@vger.kernel.org +Description: + Controls whether the "always on USB charging" feature is + enabled or not. This feature enables charging USB devices + even if the computer is not turned on. diff --git a/Documentation/ABI/testing/sysfs-platform-kim b/Documentation/ABI/testing/sysfs-platform-kim index a7f81de68046..6a52d6d2b601 100644 --- a/Documentation/ABI/testing/sysfs-platform-kim +++ b/Documentation/ABI/testing/sysfs-platform-kim @@ -7,7 +7,7 @@ Description: is connected. example: "/dev/ttyS0". The device name flows down to architecture specific board - initialization file from the SFI/ATAGS bootloader + initialization file from the ATAGS bootloader firmware. The name exposed is read from the user-space dameon and opens the device when install is requested. diff --git a/Documentation/ABI/testing/sysfs-platform_profile b/Documentation/ABI/testing/sysfs-platform_profile index 9d6b89b66cca..dae9c8941905 100644 --- a/Documentation/ABI/testing/sysfs-platform_profile +++ b/Documentation/ABI/testing/sysfs-platform_profile @@ -5,13 +5,17 @@ Description: This file contains a space-separated list of profiles supported for Drivers must use the following standard profile-names: - ============ ============================================ - low-power Low power consumption - cool Cooler operation - quiet Quieter operation - balanced Balance between low power consumption and performance - performance High performance operation - ============ ============================================ + ==================== ======================================== + low-power Low power consumption + cool Cooler operation + quiet Quieter operation + balanced Balance between low power consumption + and performance + balanced-performance Balance between performance and low + power consumption with a slight bias + towards performance + performance High performance operation + ==================== ======================================== Userspace may expect drivers to offer more than one of these standard profile names. diff --git a/Documentation/PCI/endpoint/function/binding/pci-ntb.rst b/Documentation/PCI/endpoint/function/binding/pci-ntb.rst new file mode 100644 index 000000000000..40253d3d5163 --- /dev/null +++ b/Documentation/PCI/endpoint/function/binding/pci-ntb.rst @@ -0,0 +1,38 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================== +PCI NTB Endpoint Function +========================== + +1) Create a subdirectory to pci_epf_ntb directory in configfs. + +Standard EPF Configurable Fields: + +================ =========================================================== +vendorid should be 0x104c +deviceid should be 0xb00d for TI's J721E SoC +revid don't care +progif_code don't care +subclass_code should be 0x00 +baseclass_code should be 0x5 +cache_line_size don't care +subsys_vendor_id don't care +subsys_id don't care +interrupt_pin don't care +msi_interrupts don't care +msix_interrupts don't care +================ =========================================================== + +2) Create a subdirectory to directory created in 1 + +NTB EPF specific configurable fields: + +================ =========================================================== +db_count Number of doorbells; default = 4 +mw1 size of memory window1 +mw2 size of memory window2 +mw3 size of memory window3 +mw4 size of memory window4 +num_mws Number of memory windows; max = 4 +spad_count Number of scratchpad registers; default = 64 +================ =========================================================== diff --git a/Documentation/PCI/endpoint/index.rst b/Documentation/PCI/endpoint/index.rst index 4ca7439fbfc9..38ea1f604b6d 100644 --- a/Documentation/PCI/endpoint/index.rst +++ b/Documentation/PCI/endpoint/index.rst @@ -11,5 +11,8 @@ PCI Endpoint Framework pci-endpoint-cfs pci-test-function pci-test-howto + pci-ntb-function + pci-ntb-howto function/binding/pci-test + function/binding/pci-ntb diff --git a/Documentation/PCI/endpoint/pci-endpoint-cfs.rst b/Documentation/PCI/endpoint/pci-endpoint-cfs.rst index 1bbd81ed06c8..696f8eeb4738 100644 --- a/Documentation/PCI/endpoint/pci-endpoint-cfs.rst +++ b/Documentation/PCI/endpoint/pci-endpoint-cfs.rst @@ -68,6 +68,16 @@ created) ... subsys_vendor_id ... subsys_id ... interrupt_pin + ... primary/ + ... / + ... secondary/ + ... / + +If an EPF device has to be associated with 2 EPCs (like in the case of +Non-transparent bridge), symlink of endpoint controller connected to primary +interface should be added in 'primary' directory and symlink of endpoint +controller connected to secondary interface should be added in 'secondary' +directory. EPC Device ========== diff --git a/Documentation/PCI/endpoint/pci-ntb-function.rst b/Documentation/PCI/endpoint/pci-ntb-function.rst new file mode 100644 index 000000000000..3b9d836a4924 --- /dev/null +++ b/Documentation/PCI/endpoint/pci-ntb-function.rst @@ -0,0 +1,348 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================= +PCI NTB Function +================= + +:Author: Kishon Vijay Abraham I + +PCI Non-Transparent Bridges (NTB) allow two host systems to communicate +with each other by exposing each host as a device to the other host. +NTBs typically support the ability to generate interrupts on the remote +machine, expose memory ranges as BARs, and perform DMA. They also support +scratchpads, which are areas of memory within the NTB that are accessible +from both machines. + +PCI NTB Function allows two different systems (or hosts) to communicate +with each other by configuring the endpoint instances in such a way that +transactions from one system are routed to the other system. + +In the below diagram, PCI NTB function configures the SoC with multiple +PCI Endpoint (EP) instances in such a way that transactions from one EP +controller are routed to the other EP controller. Once PCI NTB function +configures the SoC with multiple EP instances, HOST1 and HOST2 can +communicate with each other using SoC as a bridge. + +.. code-block:: text + + +-------------+ +-------------+ + | | | | + | HOST1 | | HOST2 | + | | | | + +------^------+ +------^------+ + | | + | | + +---------|-------------------------------------------------|---------+ + | +------v------+ +------v------+ | + | | | | | | + | | EP | | EP | | + | | CONTROLLER1 | | CONTROLLER2 | | + | | <-----------------------------------> | | + | | | | | | + | | | | | | + | | | SoC With Multiple EP Instances | | | + | | | (Configured using NTB Function) | | | + | +-------------+ +-------------+ | + +---------------------------------------------------------------------+ + +Constructs used for Implementing NTB +==================================== + + 1) Config Region + 2) Self Scratchpad Registers + 3) Peer Scratchpad Registers + 4) Doorbell (DB) Registers + 5) Memory Window (MW) + + +Config Region: +-------------- + +Config Region is a construct that is specific to NTB implemented using NTB +Endpoint Function Driver. The host and endpoint side NTB function driver will +exchange information with each other using this region. Config Region has +Control/Status Registers for configuring the Endpoint Controller. Host can +write into this region for configuring the outbound Address Translation Unit +(ATU) and to indicate the link status. Endpoint can indicate the status of +commands issued by host in this region. Endpoint can also indicate the +scratchpad offset and number of memory windows to the host using this region. + +The format of Config Region is given below. All the fields here are 32 bits. + +.. code-block:: text + + +------------------------+ + | COMMAND | + +------------------------+ + | ARGUMENT | + +------------------------+ + | STATUS | + +------------------------+ + | TOPOLOGY | + +------------------------+ + | ADDRESS (LOWER 32) | + +------------------------+ + | ADDRESS (UPPER 32) | + +------------------------+ + | SIZE | + +------------------------+ + | NO OF MEMORY WINDOW | + +------------------------+ + | MEMORY WINDOW1 OFFSET | + +------------------------+ + | SPAD OFFSET | + +------------------------+ + | SPAD COUNT | + +------------------------+ + | DB ENTRY SIZE | + +------------------------+ + | DB DATA | + +------------------------+ + | : | + +------------------------+ + | : | + +------------------------+ + | DB DATA | + +------------------------+ + + + COMMAND: + + NTB function supports three commands: + + CMD_CONFIGURE_DOORBELL (0x1): Command to configure doorbell. Before + invoking this command, the host should allocate and initialize + MSI/MSI-X vectors (i.e., initialize the MSI/MSI-X Capability in the + Endpoint). The endpoint on receiving this command will configure + the outbound ATU such that transactions to Doorbell BAR will be routed + to the MSI/MSI-X address programmed by the host. The ARGUMENT + register should be populated with number of DBs to configure (in the + lower 16 bits) and if MSI or MSI-X should be configured (BIT 16). + + CMD_CONFIGURE_MW (0x2): Command to configure memory window (MW). The + host invokes this command after allocating a buffer that can be + accessed by remote host. The allocated address should be programmed + in the ADDRESS register (64 bit), the size should be programmed in + the SIZE register and the memory window index should be programmed + in the ARGUMENT register. The endpoint on receiving this command + will configure the outbound ATU such that transactions to MW BAR + are routed to the address provided by the host. + + CMD_LINK_UP (0x3): Command to indicate an NTB application is + bound to the EP device on the host side. Once the endpoint + receives this command from both the hosts, the endpoint will + raise a LINK_UP event to both the hosts to indicate the host + NTB applications can start communicating with each other. + + ARGUMENT: + + The value of this register is based on the commands issued in + command register. See COMMAND section for more information. + + TOPOLOGY: + + Set to NTB_TOPO_B2B_USD for Primary interface + Set to NTB_TOPO_B2B_DSD for Secondary interface + + ADDRESS/SIZE: + + Address and Size to be used while configuring the memory window. + See "CMD_CONFIGURE_MW" for more info. + + MEMORY WINDOW1 OFFSET: + + Memory Window 1 and Doorbell registers are packed together in the + same BAR. The initial portion of the region will have doorbell + registers and the latter portion of the region is for memory window 1. + This register will specify the offset of the memory window 1. + + NO OF MEMORY WINDOW: + + Specifies the number of memory windows supported by the NTB device. + + SPAD OFFSET: + + Self scratchpad region and config region are packed together in the + same BAR. The initial portion of the region will have config region + and the latter portion of the region is for self scratchpad. This + register will specify the offset of the self scratchpad registers. + + SPAD COUNT: + + Specifies the number of scratchpad registers supported by the NTB + device. + + DB ENTRY SIZE: + + Used to determine the offset within the DB BAR that should be written + in order to raise doorbell. EPF NTB can use either MSI or MSI-X to + ring doorbell (MSI-X support will be added later). MSI uses same + address for all the interrupts and MSI-X can provide different + addresses for different interrupts. The MSI/MSI-X address is provided + by the host and the address it gives is based on the MSI/MSI-X + implementation supported by the host. For instance, ARM platform + using GIC ITS will have the same MSI-X address for all the interrupts. + In order to support all the combinations and use the same mechanism + for both MSI and MSI-X, EPF NTB allocates a separate region in the + Outbound Address Space for each of the interrupts. This region will + be mapped to the MSI/MSI-X address provided by the host. If a host + provides the same address for all the interrupts, all the regions + will be translated to the same address. If a host provides different + addresses, the regions will be translated to different addresses. This + will ensure there is no difference while raising the doorbell. + + DB DATA: + + EPF NTB supports 32 interrupts, so there are 32 DB DATA registers. + This holds the MSI/MSI-X data that has to be written to MSI address + for raising doorbell interrupt. This will be populated by EPF NTB + while invoking CMD_CONFIGURE_DOORBELL. + +Scratchpad Registers: +--------------------- + + Each host has its own register space allocated in the memory of NTB endpoint + controller. They are both readable and writable from both sides of the bridge. + They are used by applications built over NTB and can be used to pass control + and status information between both sides of a device. + + Scratchpad registers has 2 parts + 1) Self Scratchpad: Host's own register space + 2) Peer Scratchpad: Remote host's register space. + +Doorbell Registers: +------------------- + + Doorbell Registers are used by the hosts to interrupt each other. + +Memory Window: +-------------- + + Actual transfer of data between the two hosts will happen using the + memory window. + +Modeling Constructs: +==================== + +There are 5 or more distinct regions (config, self scratchpad, peer +scratchpad, doorbell, one or more memory windows) to be modeled to achieve +NTB functionality. At least one memory window is required while more than +one is permitted. All these regions should be mapped to BARs for hosts to +access these regions. + +If one 32-bit BAR is allocated for each of these regions, the scheme would +look like this: + +====== =============== +BAR NO CONSTRUCTS USED +====== =============== +BAR0 Config Region +BAR1 Self Scratchpad +BAR2 Peer Scratchpad +BAR3 Doorbell +BAR4 Memory Window 1 +BAR5 Memory Window 2 +====== =============== + +However if we allocate a separate BAR for each of the regions, there would not +be enough BARs for all the regions in a platform that supports only 64-bit +BARs. + +In order to be supported by most of the platforms, the regions should be +packed and mapped to BARs in a way that provides NTB functionality and +also makes sure the host doesn't access any region that it is not supposed +to. + +The following scheme is used in EPF NTB Function: + +====== =============================== +BAR NO CONSTRUCTS USED +====== =============================== +BAR0 Config Region + Self Scratchpad +BAR1 Peer Scratchpad +BAR2 Doorbell + Memory Window 1 +BAR3 Memory Window 2 +BAR4 Memory Window 3 +BAR5 Memory Window 4 +====== =============================== + +With this scheme, for the basic NTB functionality 3 BARs should be sufficient. + +Modeling Config/Scratchpad Region: +---------------------------------- + +.. code-block:: text + + +-----------------+------->+------------------+ +-----------------+ + | BAR0 | | CONFIG REGION | | BAR0 | + +-----------------+----+ +------------------+<-------+-----------------+ + | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + +-----------------+ +-->+------------------+<-------+-----------------+ + | BAR2 | Local Memory | BAR2 | + +-----------------+ +-----------------+ + | BAR3 | | BAR3 | + +-----------------+ +-----------------+ + | BAR4 | | BAR4 | + +-----------------+ +-----------------+ + | BAR5 | | BAR5 | + +-----------------+ +-----------------+ + EP CONTROLLER 1 EP CONTROLLER 2 + +Above diagram shows Config region + Scratchpad region for HOST1 (connected to +EP controller 1) allocated in local memory. The HOST1 can access the config +region and scratchpad region (self scratchpad) using BAR0 of EP controller 1. +The peer host (HOST2 connected to EP controller 2) can also access this +scratchpad region (peer scratchpad) using BAR1 of EP controller 2. This +diagram shows the case where Config region and Scratchpad regions are allocated +for HOST1, however the same is applicable for HOST2. + +Modeling Doorbell/Memory Window 1: +---------------------------------- + +.. code-block:: text + + +-----------------+ +----->+----------------+-----------+-----------------+ + | BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 | + +-----------------+ | +----------------+ +-----------------+ + | BAR1 | | | Doorbell 2 +---------+ | | + +-----------------+----+ +----------------+ | | | + | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + +-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 | + | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + +-----------------+ | |----------------+ | | | | + | BAR4 | | | | | | +-----------------+ + +-----------------+ | | MW1 +---+ | +-->+ MSI-X ADDRESS 3|| + | BAR5 | | | | | | +-----------------+ + +-----------------+ +----->-----------------+ | | | | + EP CONTROLLER 1 | | | | +-----------------+ + | | | +---->+ MSI-X ADDRESS 4 | + +----------------+ | +-----------------+ + EP CONTROLLER 2 | | | + (OB SPACE) | | | + +-------> MW1 | + | | + | | + +-----------------+ + | | + | | + | | + | | + | | + +-----------------+ + PCI Address Space + (Managed by HOST2) + +Above diagram shows how the doorbell and memory window 1 is mapped so that +HOST1 can raise doorbell interrupt on HOST2 and also how HOST1 can access +buffers exposed by HOST2 using memory window1 (MW1). Here doorbell and +memory window 1 regions are allocated in EP controller 2 outbound (OB) address +space. Allocating and configuring BARs for doorbell and memory window1 +is done during the initialization phase of NTB endpoint function driver. +Mapping from EP controller 2 OB space to PCI address space is done when HOST2 +sends CMD_CONFIGURE_MW/CMD_CONFIGURE_DOORBELL. + +Modeling Optional Memory Windows: +--------------------------------- + +This is modeled the same was as MW1 but each of the additional memory windows +is mapped to separate BARs. diff --git a/Documentation/PCI/endpoint/pci-ntb-howto.rst b/Documentation/PCI/endpoint/pci-ntb-howto.rst new file mode 100644 index 000000000000..1884bf29caba --- /dev/null +++ b/Documentation/PCI/endpoint/pci-ntb-howto.rst @@ -0,0 +1,161 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================================================================== +PCI Non-Transparent Bridge (NTB) Endpoint Function (EPF) User Guide +=================================================================== + +:Author: Kishon Vijay Abraham I + +This document is a guide to help users use pci-epf-ntb function driver +and ntb_hw_epf host driver for NTB functionality. The list of steps to +be followed in the host side and EP side is given below. For the hardware +configuration and internals of NTB using configurable endpoints see +Documentation/PCI/endpoint/pci-ntb-function.rst + +Endpoint Device +=============== + +Endpoint Controller Devices +--------------------------- + +For implementing NTB functionality at least two endpoint controller devices +are required. + +To find the list of endpoint controller devices in the system:: + + # ls /sys/class/pci_epc/ + 2900000.pcie-ep 2910000.pcie-ep + +If PCI_ENDPOINT_CONFIGFS is enabled:: + + # ls /sys/kernel/config/pci_ep/controllers + 2900000.pcie-ep 2910000.pcie-ep + + +Endpoint Function Drivers +------------------------- + +To find the list of endpoint function drivers in the system:: + + # ls /sys/bus/pci-epf/drivers + pci_epf_ntb pci_epf_ntb + +If PCI_ENDPOINT_CONFIGFS is enabled:: + + # ls /sys/kernel/config/pci_ep/functions + pci_epf_ntb pci_epf_ntb + + +Creating pci-epf-ntb Device +---------------------------- + +PCI endpoint function device can be created using the configfs. To create +pci-epf-ntb device, the following commands can be used:: + + # mount -t configfs none /sys/kernel/config + # cd /sys/kernel/config/pci_ep/ + # mkdir functions/pci_epf_ntb/func1 + +The "mkdir func1" above creates the pci-epf-ntb function device that will +be probed by pci_epf_ntb driver. + +The PCI endpoint framework populates the directory with the following +configurable fields:: + + # ls functions/pci_epf_ntb/func1 + baseclass_code deviceid msi_interrupts pci-epf-ntb.0 + progif_code secondary subsys_id vendorid + cache_line_size interrupt_pin msix_interrupts primary + revid subclass_code subsys_vendor_id + +The PCI endpoint function driver populates these entries with default values +when the device is bound to the driver. The pci-epf-ntb driver populates +vendorid with 0xffff and interrupt_pin with 0x0001:: + + # cat functions/pci_epf_ntb/func1/vendorid + 0xffff + # cat functions/pci_epf_ntb/func1/interrupt_pin + 0x0001 + + +Configuring pci-epf-ntb Device +------------------------------- + +The user can configure the pci-epf-ntb device using its configfs entry. In order +to change the vendorid and the deviceid, the following +commands can be used:: + + # echo 0x104c > functions/pci_epf_ntb/func1/vendorid + # echo 0xb00d > functions/pci_epf_ntb/func1/deviceid + +In order to configure NTB specific attributes, a new sub-directory to func1 +should be created:: + + # mkdir functions/pci_epf_ntb/func1/pci_epf_ntb.0/ + +The NTB function driver will populate this directory with various attributes +that can be configured by the user:: + + # ls functions/pci_epf_ntb/func1/pci_epf_ntb.0/ + db_count mw1 mw2 mw3 mw4 num_mws + spad_count + +A sample configuration for NTB function is given below:: + + # echo 4 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/db_count + # echo 128 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/spad_count + # echo 2 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/num_mws + # echo 0x100000 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/mw1 + # echo 0x100000 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/mw2 + +Binding pci-epf-ntb Device to EP Controller +-------------------------------------------- + +NTB function device should be attached to two PCI endpoint controllers +connected to the two hosts. Use the 'primary' and 'secondary' entries +inside NTB function device to attach one PCI endpoint controller to +primary interface and the other PCI endpoint controller to the secondary +interface:: + + # ln -s controllers/2900000.pcie-ep/ functions/pci-epf-ntb/func1/primary + # ln -s controllers/2910000.pcie-ep/ functions/pci-epf-ntb/func1/secondary + +Once the above step is completed, both the PCI endpoint controllers are ready to +establish a link with the host. + + +Start the Link +-------------- + +In order for the endpoint device to establish a link with the host, the _start_ +field should be populated with '1'. For NTB, both the PCI endpoint controllers +should establish link with the host:: + + # echo 1 > controllers/2900000.pcie-ep/start + # echo 1 > controllers/2910000.pcie-ep/start + + +RootComplex Device +================== + +lspci Output +------------ + +Note that the devices listed here correspond to the values populated in +"Creating pci-epf-ntb Device" section above:: + + # lspci + 0000:00:00.0 PCI bridge: Texas Instruments Device b00d + 0000:01:00.0 RAM memory: Texas Instruments Device b00d + + +Using ntb_hw_epf Device +----------------------- + +The host side software follows the standard NTB software architecture in Linux. +All the existing client side NTB utilities like NTB Transport Client and NTB +Netdev, NTB Ping Pong Test Client and NTB Tool Test Client can be used with NTB +function device. + +For more information on NTB see +:doc:`Non-Transparent Bridge <../../driver-api/ntb>` diff --git a/Documentation/RCU/RTFP.txt b/Documentation/RCU/RTFP.txt index 9bccf16736f7..3b0876c77355 100644 --- a/Documentation/RCU/RTFP.txt +++ b/Documentation/RCU/RTFP.txt @@ -683,7 +683,7 @@ Orran Krieger and Rusty Russell and Dipankar Sarma and Maneesh Soni" ,month="October" ,year="2001" ,note="Available: -\url{http://lkml.org/lkml/2001/10/13/105} +\url{https://lore.kernel.org/r/Pine.LNX.4.33.0110131015410.8707-100000@penguin.transmeta.com} [Viewed August 21, 2004]" ,annotation={ } @@ -826,7 +826,7 @@ Symposium on Distributed Computing} ,month="October" ,year="2002" ,note="Available: -\url{https://lkml.org/lkml/2002/10/24/262} +\url{https://lore.kernel.org/r/3DB86B05.447E7410@us.ibm.com} [Viewed February 15, 2014]" ,annotation={ Mingming Cao's patch to introduce RCU to SysV IPC. @@ -839,7 +839,7 @@ Symposium on Distributed Computing} ,month="March" ,year="2003" ,note="Available: -\url{http://lkml.org/lkml/2003/3/9/205} +\url{https://lore.kernel.org/r/Pine.LNX.4.44.0303091831560.2129-100000@home.transmeta.com} [Viewed March 13, 2006]" ,annotation={ Linus suggests replacing brlock with RCU and/or seqlocks: @@ -1036,15 +1036,15 @@ Add per-cpu batch counter" ,annotation={ RCU runs reasonably on a 512-CPU SGI using Manfred Spraul's patches, which may be found at: - https://lkml.org/lkml/2004/5/20/49 (split vars into cachelines) - https://lkml.org/lkml/2004/5/22/114 (cpu_quiet() patch) - https://lkml.org/lkml/2004/5/25/24 (0/5) - https://lkml.org/lkml/2004/5/25/23 (1/5) - https://lkml.org/lkml/2004/5/25/265 (works for Jack) - https://lkml.org/lkml/2004/5/25/20 (2/5) - https://lkml.org/lkml/2004/5/25/22 (3/5) - https://lkml.org/lkml/2004/5/25/19 (4/5) - https://lkml.org/lkml/2004/5/25/21 (5/5) + https://lore.kernel.org/r/40AC9823.6020709@colorfullife.com (split vars into cachelines) + https://lore.kernel.org/r/Pine.LNX.4.44.0405222141260.11106-100000@dbl.q-ag.de (cpu_quiet() patch) + https://lore.kernel.org/r/200405250535.i4P5ZJo8017583@dbl.q-ag.de (0/5) + https://lore.kernel.org/r/200405250535.i4P5ZKAQ017591@dbl.q-ag.de (1/5) + https://lore.kernel.org/r/20040525203215.GB5127@sgi.com (works for Jack) + https://lore.kernel.org/r/200405250535.i4P5ZLiR017599@dbl.q-ag.de (2/5) + https://lore.kernel.org/r/200405250535.i4P5ZMFt017607@dbl.q-ag.de (3/5) + https://lore.kernel.org/r/200405250535.i4P5ZN6g017615@dbl.q-ag.de (4/5) + https://lore.kernel.org/r/200405250535.i4P5ZO7I017623@dbl.q-ag.de (5/5) } } @@ -1106,7 +1106,7 @@ Oregon Health and Sciences University" ,month="August" ,year="2004" ,note="Available: -\url{http://lkml.org/lkml/2004/8/6/237} +\url{https://lore.kernel.org/r/20040807192424.GF3936@in.ibm.com} [Viewed June 8, 2010]" ,annotation={ Introduce rcu_dereference(). @@ -1119,7 +1119,7 @@ Oregon Health and Sciences University" ,month="August" ,year="2004" ,note="Available: -\url{http://lkml.org/lkml/2004/8/30/87} +\url{https://lore.kernel.org/r/1093873222.984.12.camel@new.localdomain} [Viewed February 17, 2005]" ,annotation={ Uses active code in rcu_read_lock() and rcu_read_unlock() to @@ -1186,7 +1186,7 @@ Oregon Health and Sciences University" ,month="October" ,year="2004" ,note="Available: -\url{http://lkml.org/lkml/2004/10/23/241} +\url{https://lore.kernel.org/r/20041023202723.GA1930@us.ibm.com} [Viewed June 8, 2010]" ,annotation={ Introduce rcu_assign_pointer(). @@ -1203,7 +1203,7 @@ Oregon Health and Sciences University" ,annotation={ James Morris posts Kaigai Kohei's patch to LKML. [Viewed December 10, 2004] - Kaigai's patch is at https://lkml.org/lkml/2004/9/27/52 + Kaigai's patch is at https://lore.kernel.org/r/200409271057.i8RAvcA1007873@mailsv.bs1.fc.nec.co.jp } } @@ -1241,7 +1241,7 @@ Oregon Health and Sciences University" ,year="2005" ,day="17" ,note="Available: -\url{http://lkml.org/lkml/2005/3/17/199} +\url{https://lore.kernel.org/r/20050318002026.GA2693@us.ibm.com} [Viewed September 5, 2005]" ,annotation={ First posting showing how RCU can be safely adapted for @@ -1256,7 +1256,7 @@ Oregon Health and Sciences University" ,year="2005" ,day="18" ,note="Available: -\url{http://lkml.org/lkml/2005/3/18/122} +\url{https://lore.kernel.org/r/Pine.OSF.4.05.10503181336310.2466-100000@da410.phys.au.dk} [Viewed March 30, 2006]" ,annotation={ Esben Neilsen suggests read-side suppression of grace-period @@ -1302,7 +1302,7 @@ Data Structures" ,month="May" ,year="2005" ,note="Available: -\url{http://lkml.org/lkml/2005/5/9/185} +\url{https://lore.kernel.org/r/20050510012444.GA3011@us.ibm.com} [Viewed May 13, 2005]" ,annotation={ First publication of working lock-based deferred free patches @@ -1385,7 +1385,7 @@ Data Structures" ,day="1" ,year="2005" ,note="Available: -\url{http://lkml.org/lkml/2005/8/1/155} +\url{https://lore.kernel.org/r/20050801171137.GA1754@us.ibm.com} [Viewed March 14, 2006]" ,annotation={ First operating counter-based realtime RCU patch posted to LKML. @@ -1399,7 +1399,7 @@ Data Structures" ,day="8" ,year="2005" ,note="Available: -\url{http://lkml.org/lkml/2005/8/8/108} +\url{https://lore.kernel.org/r/20050808144216.GA1307@us.ibm.com} [Viewed March 14, 2006]" ,annotation={ First operating counter-based realtime RCU patch posted to LKML, @@ -1415,7 +1415,7 @@ Data Structures" ,day="1" ,year="2005" ,note="Available: -\url{http://lkml.org/lkml/2005/10/1/70} +\url{https://lore.kernel.org/r/20051001182056.GA1613@us.ibm.com} [Viewed March 14, 2006]" ,annotation={ First rcutorture patch. @@ -1429,7 +1429,7 @@ Data Structures" ,day="6" ,year="2006" ,note="Available: -\url{https://lkml.org/lkml/2006/1/7/22} +\url{https://lore.kernel.org/r/20060106.231054.43576567.davem@davemloft.net} [Viewed February 29, 2012]" ,annotation={ David Miller's view on hashed arrays of locks: used to really @@ -1464,7 +1464,7 @@ Distributed Processing Symposium" ,day="20" ,year="2006" ,note="Available: -\url{http://lkml.org/lkml/2006/6/20/238} +\url{https://lore.kernel.org/r/20060408134707.22479.33814.sendpatchset@linux.site} [Viewed March 25, 2008]" ,annotation={ RCU-protected radix tree. @@ -1554,7 +1554,7 @@ Revised: ,day="28" ,year="2006" ,note="Available: -\url{http://lkml.org/lkml/2006/9/28/160} +\url{https://lore.kernel.org/r/20060928142616.GA20185@infradead.org} [Viewed March 27, 2008]" } @@ -1593,7 +1593,7 @@ Revised: ,year="2006" ,day=26 ,note="Available: -\url{http://lkml.org/lkml/2006/10/26/73} +\url{https://lore.kernel.org/r/20061026105731.GE11803@in.ibm.com} [Viewed January 26, 2009]" ,annotation={ RCU-based reader-writer lock that allows readers to proceed with @@ -1612,12 +1612,12 @@ Revised: ,year="2006" ,day=17 ,note="Available: -\url{http://lkml.org/lkml/2006/11/17/56} +\url{https://lore.kernel.org/r/20061117092925.GT7164@kernel.dk} [Viewed May 28, 2007]" ,annotation={ SRCU's grace periods are too slow for Jens, even after a factor-of-three speedup. - Sped-up version of SRCU at http://lkml.org/lkml/2006/11/17/359. + Sped-up version of SRCU at https://lore.kernel.org/r/20061118002845.GF2632@us.ibm.com. } } @@ -1629,7 +1629,7 @@ Revised: ,year="2006" ,day=19 ,note="Available: -\url{http://lkml.org/lkml/2006/11/19/69} +\url{https://lore.kernel.org/r/20061119190027.GA3676@oleg} [Viewed May 28, 2007]" ,annotation={ First cut of QRCU. Expanded/corrected versions followed. @@ -1644,7 +1644,7 @@ Revised: ,year="2006" ,day=30 ,note="Available: -\url{http://lkml.org/lkml/2006/11/29/330} +\url{https://lore.kernel.org/r/20061130015714.GC1350@oleg} [Viewed November 26, 2008]" ,annotation={ Expanded/corrected version of QRCU. @@ -1709,7 +1709,7 @@ Revised: ,year="2007" ,day=3 ,note="Available: -\url{http://lkml.org/lkml/2007/1/3/112} +\url{https://lore.kernel.org/r/20070103152738.GA16063@localdomain} [Viewed May 28, 2007]" ,annotation={ Patch for list_splice_rcu(). @@ -1737,7 +1737,7 @@ Revised: ,year="2007" ,day=28 ,note="Available: -\url{http://lkml.org/lkml/2007/1/28/34} +\url{https://lore.kernel.org/r/20070128120509.719287000@programming.kicks-ass.net} [Viewed March 27, 2008]" ,annotation={ RCU-like implementation for frequent updaters and rare readers(!). @@ -1767,7 +1767,7 @@ Revised: ,year="2007" ,day=24 ,note="Available: -\url{http://lkml.org/lkml/2007/2/25/18} +\url{https://lore.kernel.org/r/20070225062349.GA17468@linux.vnet.ibm.com} [Viewed March 27, 2008]" ,annotation={ Patch for QRCU supplying lock-free fast path. @@ -1846,7 +1846,7 @@ Revised: ,annotation={ LWN article describing Promela and spin, and also using Oleg Nesterov's QRCU as an example (with Paul McKenney's fastpath). - Merged patch at: http://lkml.org/lkml/2007/2/25/18 + Merged patch at: https://lore.kernel.org/r/20070225062349.GA17468@linux.vnet.ibm.com } } @@ -1885,7 +1885,7 @@ Revised: ,day="10" ,year="2007" ,note="Available: -\url{http://lkml.org/lkml/2007/9/10/213} +\url{https://lore.kernel.org/r/20070910183004.GA3299@linux.vnet.ibm.com} [Viewed October 25, 2007]" ,annotation={ Final patch for preemptable RCU to -rt. (Later patches were @@ -1933,7 +1933,7 @@ Revised: ,day="20" ,year="2007" ,note="Available: -\url{http://lkml.org/lkml/2007/12/20/244} +\url{https://lore.kernel.org/r/20071220142540.GB22523@Krystal} [Viewed March 27, 2008]" ,annotation={ Request for call_rcu_sched() and rcu_barrier_sched(). @@ -2013,7 +2013,7 @@ Revised: ,day="29" ,year="2008" ,note="Available: -\url{http://lkml.org/lkml/2008/1/29/208} +\url{https://lore.kernel.org/r/Pine.LNX.4.58.0801291113350.20371@gandalf.stny.rr.com} [Viewed March 27, 2008]" ,annotation={ Patch that prevents preemptible RCU from unnecessarily waking @@ -2028,7 +2028,7 @@ Revised: ,day="1" ,year="2008" ,note="Available: -\url{http://lkml.org/lkml/2008/2/2/255} +\url{https://lore.kernel.org/r/20080202214124.GA28612@linux.vnet.ibm.com} [Viewed October 18, 2008]" ,annotation={ Explanation of compilers violating dependency ordering. @@ -2088,7 +2088,7 @@ lot of {Linux} into your technology!!!" ,day="3" ,year="2008" ,note="Available: -\url{http://lkml.org/lkml/2008/6/2/539} +\url{https://lore.kernel.org/r/4844BE83.5010401@cn.fujitsu.com} [Viewed December 10, 2008]" ,annotation={ Updated RCU classic algorithm. Introduced multi-tailed list @@ -2122,7 +2122,7 @@ lot of {Linux} into your technology!!!" ,day="21" ,year="2008" ,note="Available: -\url{http://lkml.org/lkml/2008/8/21/336} +\url{https://lore.kernel.org/r/48AD8969.7060900@colorfullife.com} [Viewed December 8, 2008]" ,annotation={ State-based RCU. One key thing that this patch does is to @@ -2137,7 +2137,7 @@ lot of {Linux} into your technology!!!" ,day="6" ,year="2008" ,note="Available: -\url{http://lkml.org/lkml/2008/9/6/86} +\url{https://lore.kernel.org/r/48C2B1D2.5070801@colorfullife.com} [Viewed December 8, 2008]" ,annotation={ Manfred notes a fix required to my attempt to separate irq @@ -2183,7 +2183,7 @@ lot of {Linux} into your technology!!!" ,day="14" ,year="2009" ,note="Available: -\url{http://lkml.org/lkml/2009/1/14/449} +\url{https://lore.kernel.org/r/20090114202044.GJ6734@linux.vnet.ibm.com} [Viewed January 15, 2009]" ,annotation={ Small-footprint implementation of RCU for uniprocessor @@ -2218,7 +2218,7 @@ lot of {Linux} into your technology!!!" git://lttng.org/userspace-rcu.git http://lttng.org/cgi-bin/gitweb.cgi?p=userspace-rcu.git http://lttng.org/urcu - http://lkml.org/lkml/2009/2/5/572 + https://lore.kernel.org/r/20090206030543.GB8560@Krystal } } @@ -2258,7 +2258,7 @@ lot of {Linux} into your technology!!!" ,day="25" ,year="2009" ,note="Available: -\url{http://lkml.org/lkml/2009/6/25/306} +\url{https://lore.kernel.org/r/20090625160706.GA9467@linux.vnet.ibm.com} [Viewed August 16, 2009]" ,annotation={ First posting of expedited RCU to be accepted into -tip. @@ -2272,7 +2272,7 @@ lot of {Linux} into your technology!!!" ,day="23" ,year="2009" ,note="Available: -\url{http://lkml.org/lkml/2009/7/23/294} +\url{https://lore.kernel.org/r/20090724001429.GA17374@linux.vnet.ibm.com} [Viewed August 15, 2009]" ,annotation={ First posting of simple and fast preemptable RCU. @@ -2350,7 +2350,7 @@ lot of {Linux} into your technology!!!" ,month="December" ,year="2009" ,note="Available: -\url{http://lkml.org/lkml/2009/10/18/129} +\url{https://lore.kernel.org/r/20091018232918.GA7385@Krystal} [Viewed December 29, 2009]" ,annotation={ Mathieu proposed defer_rcu() with fixed-size per-thread pool @@ -2518,7 +2518,7 @@ lot of {Linux} into your technology!!!" ,month="January" ,year="2011" ,note="Available: -\url{https://lkml.org/lkml/2011/1/18/322} +\url{https://lore.kernel.org/r/AANLkTimajU0x1v6y3rH2+jr-bZ=tNLs1S_agXdGGAa3S@mail.gmail.com} [Viewed March 4, 2011]" ,annotation={ "The RCU-based name lookup is at the other end of the spectrum - the diff --git a/Documentation/accounting/cgroupstats.rst b/Documentation/accounting/cgroupstats.rst index b9afc48f4ea2..85186e7d4035 100644 --- a/Documentation/accounting/cgroupstats.rst +++ b/Documentation/accounting/cgroupstats.rst @@ -3,8 +3,8 @@ Control Groupstats ================== Control Groupstats is inspired by the discussion at -http://lkml.org/lkml/2007/4/11/187 and implements per cgroup statistics as -suggested by Andrew Morton in http://lkml.org/lkml/2007/4/11/263. +https://lore.kernel.org/r/461CF883.2030308@sw.ru and implements per cgroup statistics as +suggested by Andrew Morton in https://lore.kernel.org/r/20070411114927.1277d7c9.akpm@linux-foundation.org. Per cgroup statistics infrastructure re-uses code from the taskstats interface. A new set of cgroup operations are registered with commands diff --git a/Documentation/admin-guide/README.rst b/Documentation/admin-guide/README.rst index 261b7b4cca1f..35314b63008c 100644 --- a/Documentation/admin-guide/README.rst +++ b/Documentation/admin-guide/README.rst @@ -226,10 +226,11 @@ Configuring the kernel all module options to built in (=y) options. You can also preserve modules by LMC_KEEP. - "make kvmconfig" Enable additional options for kvm guest kernel support. + "make kvm_guest.config" Enable additional options for kvm guest kernel + support. - "make xenconfig" Enable additional options for xen dom0 guest kernel - support. + "make xen.config" Enable additional options for xen dom0 guest kernel + support. "make tinyconfig" Configure the tiniest possible kernel. diff --git a/Documentation/admin-guide/auxdisplay/cfag12864b.rst b/Documentation/admin-guide/auxdisplay/cfag12864b.rst index 18c2865bd322..da385d851acc 100644 --- a/Documentation/admin-guide/auxdisplay/cfag12864b.rst +++ b/Documentation/admin-guide/auxdisplay/cfag12864b.rst @@ -3,7 +3,7 @@ cfag12864b LCD Driver Documentation =================================== :License: GPLv2 -:Author & Maintainer: Miguel Ojeda Sandonis +:Author & Maintainer: Miguel Ojeda :Date: 2006-10-27 diff --git a/Documentation/admin-guide/auxdisplay/ks0108.rst b/Documentation/admin-guide/auxdisplay/ks0108.rst index c0b7faf73136..a7d3fe509373 100644 --- a/Documentation/admin-guide/auxdisplay/ks0108.rst +++ b/Documentation/admin-guide/auxdisplay/ks0108.rst @@ -3,7 +3,7 @@ ks0108 LCD Controller Driver Documentation ========================================== :License: GPLv2 -:Author & Maintainer: Miguel Ojeda Sandonis +:Author & Maintainer: Miguel Ojeda :Date: 2006-10-27 diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 52688ae34461..0936412e044e 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -963,21 +963,21 @@ References 2. Singh, Balbir. Memory Controller (RSS Control), http://lwn.net/Articles/222762/ 3. Emelianov, Pavel. Resource controllers based on process cgroups - http://lkml.org/lkml/2007/3/6/198 + https://lore.kernel.org/r/45ED7DEC.7010403@sw.ru 4. Emelianov, Pavel. RSS controller based on process cgroups (v2) - http://lkml.org/lkml/2007/4/9/78 + https://lore.kernel.org/r/461A3010.90403@sw.ru 5. Emelianov, Pavel. RSS controller based on process cgroups (v3) - http://lkml.org/lkml/2007/5/30/244 + https://lore.kernel.org/r/465D9739.8070209@openvz.org 6. Menage, Paul. Control Groups v10, http://lwn.net/Articles/236032/ 7. Vaidyanathan, Srinivasan, Control Groups: Pagecache accounting and control subsystem (v3), http://lwn.net/Articles/235534/ 8. Singh, Balbir. RSS controller v2 test results (lmbench), - http://lkml.org/lkml/2007/5/17/232 + https://lore.kernel.org/r/464C95D4.7070806@linux.vnet.ibm.com 9. Singh, Balbir. RSS controller v2 AIM9 results - http://lkml.org/lkml/2007/5/18/1 + https://lore.kernel.org/r/464D267A.50107@linux.vnet.ibm.com 10. Singh, Balbir. Memory controller v6 test results, - http://lkml.org/lkml/2007/8/19/36 + https://lore.kernel.org/r/20070819094658.654.84837.sendpatchset@balbir-laptop 11. Singh, Balbir. Memory controller introduction (v6), - http://lkml.org/lkml/2007/8/17/69 + https://lore.kernel.org/r/20070817084228.26003.12568.sendpatchset@balbir-laptop 12. Corbet, Jonathan, Controlling memory use in cgroups, http://lwn.net/Articles/243795/ diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 1de8695c264b..64c62b979f2f 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1,3 +1,5 @@ +.. _cgroup-v2: + ================ Control Group v2 ================ @@ -172,7 +174,6 @@ disabling controllers in v1 and make them always available in v2. cgroup v2 currently supports the following mount options. nsdelegate - Consider cgroup namespaces as delegation boundaries. This option is system wide and can only be set on mount or modified through remount from the init namespace. The mount option is @@ -180,7 +181,6 @@ cgroup v2 currently supports the following mount options. Delegation section for details. memory_localevents - Only populate memory.events with data for the current cgroup, and not any subtrees. This is legacy behaviour, the default behaviour without this option is to include subtree counts. @@ -189,7 +189,6 @@ cgroup v2 currently supports the following mount options. option is ignored on non-init namespace mounts. memory_recursiveprot - Recursively apply memory.min and memory.low protection to entire subtrees, without requiring explicit downward propagation into leaf cgroups. This allows protecting entire @@ -786,7 +785,6 @@ Core Interface Files All cgroup core files are prefixed with "cgroup." cgroup.type - A read-write single value file which exists on non-root cgroups. @@ -954,6 +952,8 @@ All cgroup core files are prefixed with "cgroup." Controllers =========== +.. _cgroup-v2-cpu: + CPU --- @@ -1259,9 +1259,9 @@ PAGE_SIZE multiple when read back. can show up in the middle. Don't rely on items remaining in a fixed position; use the keys to look up specific values! - If the entry has no per-node counter(or not show in the - mempry.numa_stat). We use 'npn'(non-per-node) as the tag - to indicate that it will not show in the mempry.numa_stat. + If the entry has no per-node counter (or not show in the + memory.numa_stat). We use 'npn' (non-per-node) as the tag + to indicate that it will not show in the memory.numa_stat. anon Amount of memory used in anonymous mappings such as @@ -1277,11 +1277,11 @@ PAGE_SIZE multiple when read back. pagetables Amount of memory allocated for page tables. - percpu(npn) + percpu (npn) Amount of memory used for storing per-cpu kernel data structures. - sock(npn) + sock (npn) Amount of memory used in network transmission buffers shmem @@ -1299,6 +1299,10 @@ PAGE_SIZE multiple when read back. Amount of cached filesystem data that was modified and is currently being written back to disk + swapcached + Amount of swap cached in memory. The swapcache is accounted + against both memory and swap usage. + anon_thp Amount of memory used in anonymous mappings backed by transparent hugepages @@ -1329,7 +1333,7 @@ PAGE_SIZE multiple when read back. Part of "slab" that cannot be reclaimed on memory pressure. - slab(npn) + slab (npn) Amount of memory used for storing in-kernel data structures. @@ -1357,39 +1361,39 @@ PAGE_SIZE multiple when read back. workingset_nodereclaim Number of times a shadow node has been reclaimed - pgfault(npn) + pgfault (npn) Total number of page faults incurred - pgmajfault(npn) + pgmajfault (npn) Number of major page faults incurred - pgrefill(npn) + pgrefill (npn) Amount of scanned pages (in an active LRU list) - pgscan(npn) + pgscan (npn) Amount of scanned pages (in an inactive LRU list) - pgsteal(npn) + pgsteal (npn) Amount of reclaimed pages - pgactivate(npn) + pgactivate (npn) Amount of pages moved to the active LRU list - pgdeactivate(npn) + pgdeactivate (npn) Amount of pages moved to the inactive LRU list - pglazyfree(npn) + pglazyfree (npn) Amount of pages postponed to be freed under memory pressure - pglazyfreed(npn) + pglazyfreed (npn) Amount of reclaimed lazyfree pages - thp_fault_alloc(npn) + thp_fault_alloc (npn) Number of transparent hugepages which were allocated to satisfy a page fault. This counter is not present when CONFIG_TRANSPARENT_HUGEPAGE is not set. - thp_collapse_alloc(npn) + thp_collapse_alloc (npn) Number of transparent hugepages which were allocated to allow collapsing an existing range of pages. This counter is not present when CONFIG_TRANSPARENT_HUGEPAGE is not set. @@ -1558,7 +1562,7 @@ IO Interface Files 8:0 rbytes=90430464 wbytes=299008000 rios=8950 wios=1252 dbytes=50331648 dios=3021 io.cost.qos - A read-write nested-keyed file with exists only on the root + A read-write nested-keyed file which exists only on the root cgroup. This file configures the Quality of Service of the IO cost @@ -1613,7 +1617,7 @@ IO Interface Files automatic mode can be restored by setting "ctrl" to "auto". io.cost.model - A read-write nested-keyed file with exists only on the root + A read-write nested-keyed file which exists only on the root cgroup. This file configures the cost model of the IO cost model based @@ -2000,10 +2004,12 @@ Cpuset Interface Files cpuset-enabled cgroups. This flag is owned by the parent cgroup and is not delegatable. - It accepts only the following input values when written to. + It accepts only the following input values when written to. - "root" - a partition root - "member" - a non-root member of a partition + ======== ================================ + "root" a partition root + "member" a non-root member of a partition + ======== ================================ When set to be a partition root, the current cgroup is the root of a new partition or scheduling domain that comprises @@ -2044,9 +2050,11 @@ Cpuset Interface Files root to change. On read, the "cpuset.sched.partition" file can show the following values. - "member" Non-root member of a partition - "root" Partition root - "root invalid" Invalid partition root + ============== ============================== + "member" Non-root member of a partition + "root" Partition root + "root invalid" Invalid partition root + ============== ============================== It is a partition root if the first 2 partition root conditions above are true and at least one CPU from "cpuset.cpus" is @@ -2090,7 +2098,7 @@ If the program returns 0, the attempt fails with -EPERM, otherwise it succeeds. An example of BPF_CGROUP_DEVICE program may be found in the kernel -source tree in the tools/testing/selftests/bpf/dev_cgroup.c file. +source tree in the tools/testing/selftests/bpf/progs/dev_cgroup.c file. RDMA @@ -2219,7 +2227,7 @@ Without cgroup namespace, the "/proc/$PID/cgroup" file shows the complete path of the cgroup of a process. In a container setup where a set of cgroups and namespaces are intended to isolate processes the "/proc/$PID/cgroup" file may leak potential system level information -to the isolated processes. For Example:: +to the isolated processes. For example:: # cat /proc/self/cgroup 0::/batchjobs/container_id1 diff --git a/Documentation/admin-guide/cifs/authors.rst b/Documentation/admin-guide/cifs/authors.rst index b02d6dd6c070..5c1d2f0fa7d1 100644 --- a/Documentation/admin-guide/cifs/authors.rst +++ b/Documentation/admin-guide/cifs/authors.rst @@ -5,10 +5,10 @@ Authors Original Author --------------- -Steve French (sfrench@samba.org) +Steve French (smfrench@gmail.com, sfrench@samba.org) The author wishes to express his appreciation and thanks to: -Andrew Tridgell (Samba team) for his early suggestions about smb/cifs VFS +Andrew Tridgell (Samba team) for his early suggestions about SMB/CIFS VFS improvements. Thanks to IBM for allowing me time and test resources to pursue this project, to Jim McDonough from IBM (and the Samba Team) for his help, to the IBM Linux JFS team for explaining many esoteric Linux filesystem features. @@ -51,7 +51,7 @@ Patch Contributors - Ronnie Sahlberg (for SMB3 xattr work, bug fixes, and lots of great work on compounding) - Shirish Pargaonkar (for many ACL patches over the years) - Sachin Prabhu (many bug fixes, including for reconnect, copy offload and security) -- Paulo Alcantara +- Paulo Alcantara (for some excellent work in DFS, and in booting from SMB3) - Long Li (some great work on RDMA, SMB Direct) diff --git a/Documentation/admin-guide/cifs/changes.rst b/Documentation/admin-guide/cifs/changes.rst index 71f2ecb62299..3147bbae9c43 100644 --- a/Documentation/admin-guide/cifs/changes.rst +++ b/Documentation/admin-guide/cifs/changes.rst @@ -3,6 +3,7 @@ Changes ======= See https://wiki.samba.org/index.php/LinuxCIFSKernel for summary -information (that may be easier to read than parsing the output of -"git log fs/cifs") about fixes/improvements to CIFS/SMB2/SMB3 support (changes +information about fixes/improvements to CIFS/SMB2/SMB3 support (changes to cifs.ko module) by kernel version (and cifs internal module version). +This may be easier to read than parsing the output of "git log fs/cifs" +by release. diff --git a/Documentation/admin-guide/cifs/introduction.rst b/Documentation/admin-guide/cifs/introduction.rst index cc2851d93d17..53ea62906aa5 100644 --- a/Documentation/admin-guide/cifs/introduction.rst +++ b/Documentation/admin-guide/cifs/introduction.rst @@ -7,19 +7,19 @@ Introduction protocol which was the successor to the Server Message Block (SMB) protocol, the native file sharing mechanism for most early PC operating systems. New and improved versions of CIFS are now - called SMB2 and SMB3. Use of SMB3 (and later, including SMB3.1.1) - is strongly preferred over using older dialects like CIFS due to - security reasons. All modern dialects, including the most recent, - SMB3.1.1 are supported by the CIFS VFS module. The SMB3 protocol - is implemented and supported by all major file servers - such as all modern versions of Windows (including Windows 2016 - Server), as well as by Samba (which provides excellent - CIFS/SMB2/SMB3 server support and tools for Linux and many other - operating systems). Apple systems also support SMB3 well, as - do most Network Attached Storage vendors, so this network - filesystem client can mount to a wide variety of systems. - It also supports mounting to the cloud (for example - Microsoft Azure), including the necessary security features. + called SMB2 and SMB3. Use of SMB3 (and later, including SMB3.1.1 + the most current dialect) is strongly preferred over using older + dialects like CIFS due to security reasons. All modern dialects, + including the most recent, SMB3.1.1, are supported by the CIFS VFS + module. The SMB3 protocol is implemented and supported by all major + file servers such as Windows (including Windows 2019 Server), as + well as by Samba (which provides excellent CIFS/SMB2/SMB3 server + support and tools for Linux and many other operating systems). + Apple systems also support SMB3 well, as do most Network Attached + Storage vendors, so this network filesystem client can mount to a + wide variety of systems. It also supports mounting to the cloud + (for example Microsoft Azure), including the necessary security + features. The intent of this module is to provide the most advanced network file system function for SMB3 compliant servers, including advanced @@ -27,8 +27,8 @@ Introduction POSIX compliance, secure per-user session establishment, encryption, high performance safe distributed caching (leases/oplocks), optional packet signing, large files, Unicode support and other internationalization - improvements. Since both Samba server and this filesystem client support - the CIFS Unix extensions (and in the future SMB3 POSIX extensions), + improvements. Since both Samba server and this filesystem client support the + CIFS Unix extensions, and the Linux client also suppors SMB3 POSIX extensions, the combination can provide a reasonable alternative to other network and cluster file systems for fileserving in some Linux to Linux environments, not just in Linux to Windows (or Linux to Mac) environments. diff --git a/Documentation/admin-guide/cifs/todo.rst b/Documentation/admin-guide/cifs/todo.rst index 25f11576e7b9..2646ed2e2d3e 100644 --- a/Documentation/admin-guide/cifs/todo.rst +++ b/Documentation/admin-guide/cifs/todo.rst @@ -13,24 +13,26 @@ is a partial list of the known problems and missing features: a) SMB3 (and SMB3.1.1) missing optional features: - - multichannel (started), integration with RDMA - - directory leases (improved metadata caching), started (root dir only) + - multichannel (partially integrated), integration of multichannel with RDMA + - directory leases (improved metadata caching). Currently only implemented for root dir - T10 copy offload ie "ODX" (copy chunk, and "Duplicate Extents" ioctl currently the only two server side copy mechanisms supported) b) improved sparse file support (fiemap and SEEK_HOLE are implemented - but additional features would be supportable by the protocol). + but additional features would be supportable by the protocol such + as FALLOC_FL_COLLAPSE_RANGE and FALLOC_FL_INSERT_RANGE) c) Directory entry caching relies on a 1 second timer, rather than using Directory Leases, currently only the root file handle is cached longer + by leveraging Directory Leases -d) quota support (needs minor kernel change since quota calls - to make it to network filesystems or deviceless filesystems) +d) quota support (needs minor kernel change since quota calls otherwise + won't make it to network filesystems or deviceless filesystems). e) Additional use cases can be optimized to use "compounding" (e.g. open/query/close and open/setinfo/close) to reduce the number of roundtrips to the server and improve performance. Various cases - (stat, statfs, create, unlink, mkdir) already have been improved by + (stat, statfs, create, unlink, mkdir, xattrs) already have been improved by using compounding but more can be done. In addition we could significantly reduce redundant opens by using deferred close (with handle caching leases) and better using reference counters on file @@ -60,7 +62,9 @@ k) Add tools to take advantage of more smb3 specific ioctls and features metadata attributes easier from tools (e.g. extending what was done in smb-info tool). -l) encrypted file support +l) encrypted file support (currently the attribute showing the file is + encrypted on the server is reported, but changing the attribute is not + supported). m) improved stats gathering tools (perhaps integration with nfsometer?) to extend and make easier to use what is currently in /proc/fs/cifs/Stats @@ -69,14 +73,13 @@ n) Add support for claims based ACLs ("DAC") o) mount helper GUI (to simplify the various configuration options on mount) -p) Add support for witness protocol (perhaps ioctl to cifs.ko from user space - tool listening on witness protocol RPC) to allow for notification of share - move, server failover, and server adapter changes. And also improve other - failover scenarios, e.g. when client knows multiple DFS entries point to - different servers, and the server we are connected to has gone down. +p) Expand support for witness protocol to allow for notification of share + move, and server network adapter changes. Currently only notifications by + the witness protocol for server move is supported by the Linux client. q) Allow mount.cifs to be more verbose in reporting errors with dialect - or unsupported feature errors. + or unsupported feature errors. This would now be easier due to the + implementation of the new mount API. r) updating cifs documentation, and user guide. @@ -87,11 +90,10 @@ t) split cifs and smb3 support into separate modules so legacy (and less secure) CIFS dialect can be disabled in environments that don't need it and simplify the code. -v) POSIX Extensions for SMB3.1.1 (started, create and mkdir support added - so far). +v) Additional testing of POSIX Extensions for SMB3.1.1 w) Add support for additional strong encryption types, and additional spnego - authentication mechanisms (see MS-SMB2) + authentication mechanisms (see MS-SMB2). GCM-256 is now partially implemented. x) Finish support for SMB3.1.1 compression diff --git a/Documentation/admin-guide/cifs/usage.rst b/Documentation/admin-guide/cifs/usage.rst index b6d9f02bc12b..13783dc68ab7 100644 --- a/Documentation/admin-guide/cifs/usage.rst +++ b/Documentation/admin-guide/cifs/usage.rst @@ -83,7 +83,7 @@ and encrypted shares and stronger signing and authentication algorithms. There are additional mount options that may be helpful for SMB3 to get improved POSIX behavior (NB: can use vers=3.0 to force only SMB3, never 2.1): - ``mfsymlinks`` and ``cifsacl`` and ``idsfromsid`` + ``mfsymlinks`` and either ``cifsacl`` or ``modefromsid`` (usually with ``idsfromsid``) Allowing User Mounts ==================== diff --git a/Documentation/admin-guide/cpu-load.rst b/Documentation/admin-guide/cpu-load.rst index f3ada90e9ca8..21a984337080 100644 --- a/Documentation/admin-guide/cpu-load.rst +++ b/Documentation/admin-guide/cpu-load.rst @@ -107,7 +107,7 @@ will lead to quite erratic information inside ``/proc/stat``:: References ---------- -- http://lkml.org/lkml/2007/2/12/6 +- https://lore.kernel.org/r/loom.20070212T063225-663@post.gmane.org - Documentation/filesystems/proc.rst (1.8) diff --git a/Documentation/admin-guide/device-mapper/dm-crypt.rst b/Documentation/admin-guide/device-mapper/dm-crypt.rst index 1a6753b76dbb..aa2d04d95df6 100644 --- a/Documentation/admin-guide/device-mapper/dm-crypt.rst +++ b/Documentation/admin-guide/device-mapper/dm-crypt.rst @@ -67,7 +67,7 @@ Parameters:: the value passed in . - Either 'logon', 'user' or 'encrypted' kernel key type. + Either 'logon', 'user', 'encrypted' or 'trusted' kernel key type. The kernel keyring key description crypt target should look for diff --git a/Documentation/admin-guide/device-mapper/dm-integrity.rst b/Documentation/admin-guide/device-mapper/dm-integrity.rst index 2cc5488acbd9..8db172efa272 100644 --- a/Documentation/admin-guide/device-mapper/dm-integrity.rst +++ b/Documentation/admin-guide/device-mapper/dm-integrity.rst @@ -143,8 +143,8 @@ recalculate journal_crypt:algorithm(:key) (the key is optional) Encrypt the journal using given algorithm to make sure that the attacker can't read the journal. You can use a block cipher here - (such as "cbc(aes)") or a stream cipher (for example "chacha20", - "salsa20" or "ctr(aes)"). + (such as "cbc(aes)") or a stream cipher (for example "chacha20" + or "ctr(aes)"). The journal contains history of last writes to the block device, an attacker reading the journal could see the last sector numbers @@ -186,6 +186,17 @@ fix_padding space-efficient. If this option is not present, large padding is used - that is for compatibility with older kernels. +fix_hmac + Improve security of internal_hash and journal_mac: + + - the section number is mixed to the mac, so that an attacker can't + copy sectors from one journal section to another journal section + - the superblock is protected by journal_mac + - a 16-byte salt stored in the superblock is mixed to the mac, so + that the attacker can't detect that two disks have the same hmac + key and also to disallow the attacker to move sectors from one + disk to another + legacy_recalculate Allow recalculating of volumes with HMAC keys. This is disabled by default for security reasons - an attacker could modify the volume, diff --git a/Documentation/admin-guide/kernel-parameters.rst b/Documentation/admin-guide/kernel-parameters.rst index 682ab28b5c94..1132796a8d96 100644 --- a/Documentation/admin-guide/kernel-parameters.rst +++ b/Documentation/admin-guide/kernel-parameters.rst @@ -60,7 +60,7 @@ Note that for the special case of a range one can split the range into equal sized groups and for each group use some amount from the beginning of that group: - -cpu number>:/ + -:/ For example one can add to the command line following parameter: diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 07f9372b494a..04545725f187 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -606,7 +606,7 @@ kernel/dma/contiguous.c cma_pernuma=nn[MG] - [ARM64,KNL] + [ARM64,KNL,CMA] Sets the size of kernel per-numa memory area for contiguous memory allocations. A value of 0 disables per-numa CMA altogether. And If this option is not @@ -1434,6 +1434,11 @@ to enforce probe and suspend/resume ordering. rpm -- Like "on", but also use to order runtime PM. + fw_devlink.strict= + [KNL] Treat all inferred dependencies as mandatory + dependencies. This only applies for fw_devlink=on|rpm. + Format: + gamecon.map[2|3]= [HW,JOY] Multisystem joystick and NES/SNES/PSX pad support via parallel port (up to 5 devices per port) @@ -1525,12 +1530,12 @@ hpet_mmap= [X86, HPET_MMAP] Allow userspace to mmap HPET registers. Default set by CONFIG_HPET_MMAP_DEFAULT. - hugetlb_cma= [HW] The size of a cma area used for allocation + hugetlb_cma= [HW,CMA] The size of a CMA area used for allocation of gigantic hugepages. Format: nn[KMGTPE] - Reserve a cma area of given size and allocate gigantic - hugepages using the cma allocator. If enabled, the + Reserve a CMA area of given size and allocate gigantic + hugepages using the CMA allocator. If enabled, the boot-time allocation of gigantic hugepages is skipped. hugepages= [HW] Number of HugeTLB pages to allocate at boot. @@ -1674,6 +1679,12 @@ In such case C2/C3 won't be used again. idle=nomwait: Disable mwait for CPU C-states + idxd.sva= [HW] + Format: + Allow force disabling of Shared Virtual Memory (SVA) + support for the idxd driver. By default it is set to + true (1). + ieee754= [MIPS] Select IEEE Std 754 conformance mode Format: { strict | legacy | 2008 | relaxed } Default: strict @@ -1747,7 +1758,7 @@ ima_policy= [IMA] The builtin policies to load during IMA setup. Format: "tcb | appraise_tcb | secure_boot | - fail_securely" + fail_securely | critical_data" The "tcb" policy measures all programs exec'd, files mmap'd for exec, and all files opened with the read @@ -1766,6 +1777,9 @@ filesystems with the SB_I_UNVERIFIABLE_SIGNATURE flag. + The "critical_data" policy measures kernel integrity + critical data. + ima_tcb [IMA] Deprecated. Use ima_policy= instead. Load a policy which meets the needs of the Trusted Computing Base. This means IMA will measure all @@ -3270,9 +3284,14 @@ parameter, xsave area per process might occupy more memory on xsaves enabled systems. - nohlt [BUGS=ARM,SH] Tells the kernel that the sleep(SH) or - wfi(ARM) instruction doesn't work correctly and not to - use it. This is also useful when using JTAG debugger. + nohlt [ARM,ARM64,MICROBLAZE,SH] Forces the kernel to busy wait + in do_idle() and not use the arch_cpu_idle() + implementation; requires CONFIG_GENERIC_IDLE_POLL_SETUP + to be effective. This is useful on platforms where the + sleep(SH) or wfi(ARM,ARM64) instructions do not work + correctly or when doing power measurements to evalute + the impact of the sleep instructions. This is also + useful when using JTAG debugger. no_file_caps Tells the kernel not to honor file capabilities. The only way then for a file to be executed with privilege @@ -3285,6 +3304,21 @@ in certain environments such as networked servers or real-time systems. + no_hash_pointers + Force pointers printed to the console or buffers to be + unhashed. By default, when a pointer is printed via %p + format string, that pointer is "hashed", i.e. obscured + by hashing the pointer value. This is a security feature + that hides actual kernel addresses from unprivileged + users, but it also makes debugging the kernel more + difficult since unequal pointers can no longer be + compared. However, if this command-line option is + specified, then all normal pointers will have their true + value printed. Pointers printed via %pK may still be + hashed. This option should only be specified when + debugging the kernel. Please do not use on production + kernels. + nohibernate [HIBERNATION] Disable hibernation and resume. nohz= [KNL] Boottime enable/disable dynamic ticks @@ -4870,14 +4904,6 @@ last alloc / free. For more information see Documentation/vm/slub.rst. - slub_memcg_sysfs= [MM, SLUB] - Determines whether to enable sysfs directories for - memory cgroup sub-caches. 1 to enable, 0 to disable. - The default is determined by CONFIG_SLUB_MEMCG_SYSFS_ON. - Enabling this can lead to a very high number of debug - directories and files being created under - /sys/kernel/slub. - slub_max_order= [MM, SLUB] Determines the maximum allowed order for slabs. A high setting may cause OOMs due to memory @@ -5156,6 +5182,12 @@ growing up) the main stack are reserved for no other mapping. Default value is 256 pages. + stack_depot_disable= [KNL] + Setting this to true through kernel command line will + disable the stack depot thereby saving the static memory + consumed by the stack hash table. By default this is set + to false. + stacktrace [FTRACE] Enabled the stack tracer on boot up. @@ -5956,12 +5988,6 @@ default x2apic cluster mode on platforms supporting x2apic. - x86_intel_mid_timer= [X86-32,APBT] - Choose timer option for x86 Intel MID platform. - Two valid options are apbt timer only and lapic timer - plus one apbt timer for broadcast timer. - x86_intel_mid_timer=apbt_only | lapic_and_apbt - xen_512gb_limit [KNL,X86-64,XEN] Restricts the kernel running paravirtualized under Xen to use only up to 512 GB of RAM. The reason to do so is diff --git a/Documentation/admin-guide/kernel-per-CPU-kthreads.rst b/Documentation/admin-guide/kernel-per-CPU-kthreads.rst index dc36aeb65d0a..531f689311f2 100644 --- a/Documentation/admin-guide/kernel-per-CPU-kthreads.rst +++ b/Documentation/admin-guide/kernel-per-CPU-kthreads.rst @@ -273,7 +273,7 @@ To reduce its OS jitter, do any of the following: However, there is an RFC patch from Christoph Lameter (based on an earlier one from Gilad Ben-Yossef) that reduces or even eliminates vmstat overhead for some - workloads at https://lkml.org/lkml/2013/9/4/379. + workloads at https://lore.kernel.org/r/00000140e9dfd6bd-40db3d4f-c1be-434f-8132-7820f81bb586-000000@email.amazonses.com. e. If running on high-end powerpc servers, build with CONFIG_PPC_RTAS_DAEMON=n. This prevents the RTAS daemon from running on each CPU every second or so. diff --git a/Documentation/admin-guide/laptops/thinkpad-acpi.rst b/Documentation/admin-guide/laptops/thinkpad-acpi.rst index 5fe1ade88c17..91fd6846ce17 100644 --- a/Documentation/admin-guide/laptops/thinkpad-acpi.rst +++ b/Documentation/admin-guide/laptops/thinkpad-acpi.rst @@ -51,6 +51,7 @@ detailed description): - UWB enable and disable - LCD Shadow (PrivacyGuard) enable and disable - Lap mode sensor + - Setting keyboard language A compatibility table by model and feature is maintained on the web site, http://ibm-acpi.sf.net/. I appreciate any success or failure @@ -1466,6 +1467,30 @@ Sysfs notes rfkill controller switch "tpacpi_uwb_sw": refer to Documentation/driver-api/rfkill.rst for details. + +Setting keyboard language +------------------------- + +sysfs: keyboard_lang + +This feature is used to set keyboard language to ECFW using ASL interface. +Fewer thinkpads models like T580 , T590 , T15 Gen 1 etc.. has "=", "(', +")" numeric keys, which are not displaying correctly, when keyboard language +is other than "english". This is because the default keyboard language in ECFW +is set as "english". Hence using this sysfs, user can set the correct keyboard +language to ECFW and then these key's will work correctly. + +Example of command to set keyboard language is mentioned below:: + + echo jp > /sys/devices/platform/thinkpad_acpi/keyboard_lang + +Text corresponding to keyboard layout to be set in sysfs are: be(Belgian), +cz(Czech), da(Danish), de(German), en(English), es(Spain), et(Estonian), +fr(French), fr-ch(French(Switzerland)), hu(Hungarian), it(Italy), jp (Japan), +nl(Dutch), nn(Norway), pl(Polish), pt(portugese), sl(Slovenian), sv(Sweden), +tr(Turkey) + + Adaptive keyboard ----------------- diff --git a/Documentation/admin-guide/mm/memory-hotplug.rst b/Documentation/admin-guide/mm/memory-hotplug.rst index 5c4432c96c4b..5307f90738aa 100644 --- a/Documentation/admin-guide/mm/memory-hotplug.rst +++ b/Documentation/admin-guide/mm/memory-hotplug.rst @@ -160,16 +160,16 @@ Under each memory block, you can see 5 files: "online_movable", "online", "offline" command which will be performed on all sections in the block. -``phys_device`` read-only: designed to show the name of physical memory - device. This is not well implemented now. -``removable`` read-only: contains an integer value indicating - whether the memory block is removable or not - removable. A value of 1 indicates that the memory - block is removable and a value of 0 indicates that - it is not removable. A memory block is removable only if - every section in the block is removable. -``valid_zones`` read-only: designed to show which zones this memory block - can be onlined to. +``phys_device`` read-only: legacy interface only ever used on s390x to + expose the covered storage increment. +``removable`` read-only: legacy interface that indicated whether a memory + block was likely to be offlineable or not. Newer kernel + versions return "1" if and only if the kernel supports + memory offlining. +``valid_zones`` read-only: designed to show by which zone memory provided by + a memory block is managed, and to show by which zone memory + provided by an offline memory block could be managed when + onlining. The first column shows it`s default zone. diff --git a/Documentation/admin-guide/perf-security.rst b/Documentation/admin-guide/perf-security.rst index 904e4eb37f99..34aa334320ca 100644 --- a/Documentation/admin-guide/perf-security.rst +++ b/Documentation/admin-guide/perf-security.rst @@ -72,7 +72,7 @@ monitoring and observability operations, thus, bypass *scope* permissions checks in the kernel. CAP_PERFMON implements the principle of least privilege [13]_ (POSIX 1003.1e: 2.2.2.39) for performance monitoring and observability operations in the kernel and provides a secure approach to -perfomance monitoring and observability in the system. +performance monitoring and observability in the system. For backward compatibility reasons the access to perf_events monitoring and observability operations is also open for CAP_SYS_ADMIN privileged diff --git a/Documentation/admin-guide/spkguide.txt b/Documentation/admin-guide/spkguide.txt index 5ff6a0fe87d1..977ab3f5a0a8 100644 --- a/Documentation/admin-guide/spkguide.txt +++ b/Documentation/admin-guide/spkguide.txt @@ -1033,7 +1033,9 @@ speakup + keypad 3, you would hear: The speakup key is depressed, so the name of the key state is speakup. This part of the message comes from the states collection. -14.2. Loading Your Own Messages +14.2. Changing language + +14.2.1. Loading Your Own Messages The files under the i18n subdirectory all follow the same format. They consist of lines, with one message per line. @@ -1066,8 +1068,50 @@ echo '1 azul' > /speakup/i18n/colors The next time that Speakup says message 1 from the colors group, it will say "azul", rather than "blue." +14.2.2. Choose a language + In the future, translations into various languages will be made available, -and most users will just load the files necessary for their language. +and most users will just load the files necessary for their language. So far, +only French language is available beyond native Canadian English language. + +French is only available after you are logged in. + +Canadian English is the default language. To toggle another language, +download the source of Speakup and untar it in your home directory. The +following command should let you do this: + +tar xvjf speakup-.tar.bz2 + +where is the version number of the application. + +Next, change to the newly created directory, then into the tools/ directory, and +run the script speakup_setlocale. You are asked the language that you want to +use. Type the number associated to your language (e.g. fr for French) then press +Enter. Needed files are copied in the i18n directory. + +Note: the speakupconf must be installed on your system so that settings are saved. +Otherwise, you will have an error: your language will be loaded but you will +have to run the script again every time Speakup restarts. +See section 16.1. for information about speakupconf. + +You will have to repeat these steps for any change of locale, i.e. if you wish +change the speakup's language or charset (iso-8859-15 ou UTF-8). + +If you wish store the settings, note that at your next login, you will need to +do: + +speakup load + +Alternatively, you can add the above line to your file +~/.bashrc or ~/.bash_profile. + +If your system administrator ran himself the script, all the users will be able +to change from English to the language choosed by root and do directly +speakupconf load (or add this to the ~/.bashrc or +~/.bash_profile file). If there are several languages to handle, the +administrator (or every user) will have to run the first steps until speakupconf +save, choosing the appropriate language, in every user's home directory. Every +user will then be able to do speakupconf load, Speakup will load his own settings. 14.3. No Support for Non-Western-European Languages diff --git a/Documentation/admin-guide/sysctl/fs.rst b/Documentation/admin-guide/sysctl/fs.rst index f48277a0a850..2a501c9ddc55 100644 --- a/Documentation/admin-guide/sysctl/fs.rst +++ b/Documentation/admin-guide/sysctl/fs.rst @@ -380,5 +380,5 @@ This configuration option sets the maximum number of "watches" that are allowed for each user. Each "watch" costs roughly 90 bytes on a 32bit kernel, and roughly 160 bytes on a 64bit one. -The current default value for max_user_watches is the 1/32 of the available -low memory, divided for the "watch" cost in bytes. +The current default value for max_user_watches is the 1/25 (4%) of the +available low memory, divided for the "watch" cost in bytes. diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index e35a3f2fb006..586cd4b86428 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -983,11 +983,11 @@ that benefit from having their data cached, zone_reclaim_mode should be left disabled as the caching effect is likely to be more important than data locality. -zone_reclaim may be enabled if it's known that the workload is partitioned -such that each partition fits within a NUMA node and that accessing remote -memory would cause a measurable performance reduction. The page allocator -will then reclaim easily reusable pages (those page cache pages that are -currently not used) before allocating off node pages. +Consider enabling one or more zone_reclaim mode bits if it's known that the +workload is partitioned such that each partition fits within a NUMA node +and that accessing remote memory would cause a measurable performance +reduction. The page allocator will take additional actions before +allocating off node pages. Allowing zone reclaim to write out pages stops processes that are writing large amounts of data from dirtying pages on other nodes. Zone diff --git a/Documentation/admin-guide/xfs.rst b/Documentation/admin-guide/xfs.rst index 6178153d3320..5422407a96d7 100644 --- a/Documentation/admin-guide/xfs.rst +++ b/Documentation/admin-guide/xfs.rst @@ -284,6 +284,9 @@ The following sysctls are available for the XFS filesystem: removes unused preallocation from clean inodes and releases the unused space back to the free pool. + fs.xfs.speculative_cow_prealloc_lifetime + This is an alias for speculative_prealloc_lifetime. + fs.xfs.error_level (Min: 0 Default: 3 Max: 11) A volume knob for error reporting when internal errors occur. This will generate detailed messages & backtraces for filesystem @@ -356,12 +359,13 @@ The following sysctls are available for the XFS filesystem: Deprecated Sysctls ================== -=========================== ================ - Name Removal Schedule -=========================== ================ -fs.xfs.irix_sgid_inherit September 2025 -fs.xfs.irix_symlink_mode September 2025 -=========================== ================ +=========================================== ================ + Name Removal Schedule +=========================================== ================ +fs.xfs.irix_sgid_inherit September 2025 +fs.xfs.irix_symlink_mode September 2025 +fs.xfs.speculative_cow_prealloc_lifetime September 2025 +=========================================== ================ Removed Sysctls diff --git a/Documentation/arm/booting.rst b/Documentation/arm/booting.rst index a2263451dc2c..5974e37b3d20 100644 --- a/Documentation/arm/booting.rst +++ b/Documentation/arm/booting.rst @@ -128,7 +128,7 @@ it. The recommended placement is in the first 16KiB of RAM. The boot loader must load a device tree image (dtb) into system ram at a 64bit aligned address and initialize it with the boot data. The -dtb format is documented in Documentation/devicetree/booting-without-of.rst. +dtb format is documented at https://www.devicetree.org/specifications/. The kernel will look for the dtb magic value of 0xd00dfeed at the dtb physical address to determine if a dtb has been passed instead of a tagged list. diff --git a/Documentation/arm/index.rst b/Documentation/arm/index.rst index a2e9e1bba7b9..b4bea32472b6 100644 --- a/Documentation/arm/index.rst +++ b/Documentation/arm/index.rst @@ -33,7 +33,7 @@ SoC-specific documents ixp4xx - marvel + marvell microchip netwinder diff --git a/Documentation/arm/marvel.rst b/Documentation/arm/marvell.rst similarity index 97% rename from Documentation/arm/marvel.rst rename to Documentation/arm/marvell.rst index 16ab2eb085b8..94cd73383594 100644 --- a/Documentation/arm/marvel.rst +++ b/Documentation/arm/marvell.rst @@ -127,7 +127,7 @@ EBU Armada family - 88F6828 Armada 388 - Product infos: http://www.marvell.com/embedded-processors/armada-38x/ - - Functional Spec: https://marvellcorp.wufoo.com/forms/marvell-armada-38x-functional-specifications/ + - Functional Spec: http://www.marvell.com/content/dam/marvell/en/public-collateral/embedded-processors/marvell-embedded-processors-armada-38x-functional-specifications-2015-11.pdf Core: ARM Cortex-A9 @@ -183,7 +183,10 @@ EBU Armada family ARMv8 http://www.marvell.com/embedded-processors/armada-3700/ Product Brief: - http://www.marvell.com/embedded-processors/assets/PB-88F3700-FNL.pdf + http://www.marvell.com/content/dam/marvell/en/public-collateral/embedded-processors/marvell-embedded-processors-armada-37xx-product-brief-2016-01.pdf + + Hardware Spec: + http://www.marvell.com/content/dam/marvell/en/public-collateral/embedded-processors/marvell-embedded-processors-armada-37xx-hardware-specifications-2019-09.pdf Device tree files: arch/arm64/boot/dts/marvell/armada-37* diff --git a/Documentation/block/inline-encryption.rst b/Documentation/block/inline-encryption.rst index e75151e467d3..7f9b40d6b416 100644 --- a/Documentation/block/inline-encryption.rst +++ b/Documentation/block/inline-encryption.rst @@ -182,8 +182,9 @@ API presented to device drivers A :c:type:``struct blk_keyslot_manager`` should be set up by device drivers in the ``request_queue`` of the device. The device driver needs to call -``blk_ksm_init`` on the ``blk_keyslot_manager``, which specifying the number of -keyslots supported by the hardware. +``blk_ksm_init`` (or its resource-managed variant ``devm_blk_ksm_init``) on the +``blk_keyslot_manager``, while specifying the number of keyslots supported by +the hardware. The device driver also needs to tell the KSM how to actually manipulate the IE hardware in the device to do things like programming the crypto key into @@ -202,10 +203,9 @@ needs each and every of its keyslots to be reprogrammed with the key it "should have" at the point in time when the function is called. This is useful e.g. if a device loses all its keys on runtime power down/up. -``blk_ksm_destroy`` should be called to free up all resources used by a keyslot -manager upon ``blk_ksm_init``, once the ``blk_keyslot_manager`` is no longer -needed. - +If the driver used ``blk_ksm_init`` instead of ``devm_blk_ksm_init``, then +``blk_ksm_destroy`` should be called to free up all resources used by a +``blk_keyslot_manager`` once it is no longer needed. Layered Devices =============== diff --git a/Documentation/conf.py b/Documentation/conf.py index 6a767294887e..fd65168c10f8 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -31,7 +31,7 @@ from load_config import loadConfig # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.3' +needs_sphinx = '1.7' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -49,8 +49,7 @@ extensions = ['kerneldoc', 'rstFlatTable', 'kernel_include', if major >= 3: sys.stderr.write('''WARNING: The kernel documentation build process support for Sphinx v3.0 and above is brand new. Be prepared for - possible issues in the generated output. - ''') + possible issues in the generated output.\n''') if (major > 3) or (minor > 0 or patch >= 2): # Sphinx c function parser is more pedantic with regards to type # checking. Due to that, having macros at c:function cause problems. @@ -112,19 +111,12 @@ if major >= 3: else: extensions.append('cdomain') - if major == 1 and minor < 7: - sys.stderr.write('WARNING: Sphinx 1.7 or greater will be required as of ' - 'the 5.12 release\n') # Ensure that autosectionlabel will produce unique names autosectionlabel_prefix_document = True autosectionlabel_maxdepth = 2 -# The name of the math extension changed on Sphinx 1.4 -if (major == 1 and minor > 3) or (major > 1): - extensions.append("sphinx.ext.imgmath") -else: - extensions.append("sphinx.ext.pngmath") +extensions.append("sphinx.ext.imgmath") # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -375,71 +367,9 @@ if cjk_cmd.find("Noto Sans CJK SC") >= 0: ''' # Fix reference escape troubles with Sphinx 1.4.x -if major == 1 and minor > 3: +if major == 1: latex_elements['preamble'] += '\\renewcommand*{\\DUrole}[2]{ #2 }\n' -if major == 1 and minor <= 4: - latex_elements['preamble'] += '\\usepackage[margin=0.5in, top=1in, bottom=1in]{geometry}' -elif major == 1 and (minor > 5 or (minor == 5 and patch >= 3)): - latex_elements['sphinxsetup'] = 'hmargin=0.5in, vmargin=1in' - latex_elements['preamble'] += '\\fvset{fontsize=auto}\n' - -# Customize notice background colors on Sphinx < 1.6: -if major == 1 and minor < 6: - latex_elements['preamble'] += ''' - \\usepackage{ifthen} - - % Put notes in color and let them be inside a table - \\definecolor{NoteColor}{RGB}{204,255,255} - \\definecolor{WarningColor}{RGB}{255,204,204} - \\definecolor{AttentionColor}{RGB}{255,255,204} - \\definecolor{ImportantColor}{RGB}{192,255,204} - \\definecolor{OtherColor}{RGB}{204,204,204} - \\newlength{\\mynoticelength} - \\makeatletter\\newenvironment{coloredbox}[1]{% - \\setlength{\\fboxrule}{1pt} - \\setlength{\\fboxsep}{7pt} - \\setlength{\\mynoticelength}{\\linewidth} - \\addtolength{\\mynoticelength}{-2\\fboxsep} - \\addtolength{\\mynoticelength}{-2\\fboxrule} - \\begin{lrbox}{\\@tempboxa}\\begin{minipage}{\\mynoticelength}}{\\end{minipage}\\end{lrbox}% - \\ifthenelse% - {\\equal{\\py@noticetype}{note}}% - {\\colorbox{NoteColor}{\\usebox{\\@tempboxa}}}% - {% - \\ifthenelse% - {\\equal{\\py@noticetype}{warning}}% - {\\colorbox{WarningColor}{\\usebox{\\@tempboxa}}}% - {% - \\ifthenelse% - {\\equal{\\py@noticetype}{attention}}% - {\\colorbox{AttentionColor}{\\usebox{\\@tempboxa}}}% - {% - \\ifthenelse% - {\\equal{\\py@noticetype}{important}}% - {\\colorbox{ImportantColor}{\\usebox{\\@tempboxa}}}% - {\\colorbox{OtherColor}{\\usebox{\\@tempboxa}}}% - }% - }% - }% - }\\makeatother - - \\makeatletter - \\renewenvironment{notice}[2]{% - \\def\\py@noticetype{#1} - \\begin{coloredbox}{#1} - \\bf\\it - \\par\\strong{#2} - \\csname py@noticestart@#1\\endcsname - } - { - \\csname py@noticeend@\\py@noticetype\\endcsname - \\end{coloredbox} - } - \\makeatother - - ''' - # With Sphinx 1.6, it is possible to change the Bg color directly # by using: # \definecolor{sphinxnoteBgColor}{RGB}{204,255,255} diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst index 75cb757bbff0..e6d23f117308 100644 --- a/Documentation/core-api/dma-api.rst +++ b/Documentation/core-api/dma-api.rst @@ -526,46 +526,6 @@ for the kernel vs the device. If you don't understand how cache line coherency works between a processor and an I/O device, you should not be using this part of the API. -:: - - void * - dma_alloc_noncoherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, enum dma_data_direction dir, - gfp_t gfp) - -This routine allocates a region of bytes of consistent memory. It -returns a pointer to the allocated region (in the processor's virtual address -space) or NULL if the allocation failed. The returned memory may or may not -be in the kernel direct mapping. Drivers must not call virt_to_page on -the returned memory region. - -It also returns a which may be cast to an unsigned integer the -same width as the bus and given to the device as the DMA address base of -the region. - -The dir parameter specified if data is read and/or written by the device, -see dma_map_single() for details. - -The gfp parameter allows the caller to specify the ``GFP_`` flags (see -kmalloc()) for the allocation, but rejects flags used to specify a memory -zone such as GFP_DMA or GFP_HIGHMEM. - -Before giving the memory to the device, dma_sync_single_for_device() needs -to be called, and before reading memory written by the device, -dma_sync_single_for_cpu(), just like for streaming DMA mappings that are -reused. - -:: - - void - dma_free_noncoherent(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, enum dma_data_direction dir) - -Free a region of memory previously allocated using dma_alloc_noncoherent(). -dev, size and dma_handle and dir must all be the same as those passed into -dma_alloc_noncoherent(). cpu_addr must be the virtual address returned by -dma_alloc_noncoherent(). - :: struct page * @@ -600,9 +560,29 @@ reused. dma_addr_t dma_handle, enum dma_data_direction dir) Free a region of memory previously allocated using dma_alloc_pages(). -dev, size and dma_handle and dir must all be the same as those passed into -dma_alloc_noncoherent(). page must be the pointer returned by -dma_alloc_pages(). +dev, size, dma_handle and dir must all be the same as those passed into +dma_alloc_pages(). page must be the pointer returned by dma_alloc_pages(). + +:: + + void * + dma_alloc_noncoherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, enum dma_data_direction dir, + gfp_t gfp) + +This routine is a convenient wrapper around dma_alloc_pages that returns the +kernel virtual address for the allocated memory instead of the page structure. + +:: + + void + dma_free_noncoherent(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_handle, enum dma_data_direction dir) + +Free a region of memory previously allocated using dma_alloc_noncoherent(). +dev, size, dma_handle and dir must all be the same as those passed into +dma_alloc_noncoherent(). cpu_addr must be the virtual address returned by +dma_alloc_noncoherent(). :: diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index 2adffb3f7914..201b5423303b 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -19,11 +19,8 @@ User Space Memory Access Memory Allocation Controls ========================== -Functions which need to allocate memory often use GFP flags to express -how that memory should be allocated. The GFP acronym stands for "get -free pages", the underlying memory allocation function. Not every GFP -flag is allowed to every function which may allocate memory. Most -users will want to use a plain ``GFP_KERNEL``. +.. kernel-doc:: include/linux/gfp.h + :internal: .. kernel-doc:: include/linux/gfp.h :doc: Page mobility and placement hints diff --git a/Documentation/crypto/api-skcipher.rst b/Documentation/crypto/api-skcipher.rst index 1aaf8985894b..04d6cc5357c8 100644 --- a/Documentation/crypto/api-skcipher.rst +++ b/Documentation/crypto/api-skcipher.rst @@ -28,8 +28,8 @@ Symmetric Key Cipher Request Handle Single Block Cipher API ----------------------- -.. kernel-doc:: include/linux/crypto.h +.. kernel-doc:: include/crypto/internal/cipher.h :doc: Single Block Cipher API -.. kernel-doc:: include/linux/crypto.h +.. kernel-doc:: include/crypto/internal/cipher.h :functions: crypto_alloc_cipher crypto_free_cipher crypto_has_cipher crypto_cipher_blocksize crypto_cipher_setkey crypto_cipher_encrypt_one crypto_cipher_decrypt_one diff --git a/Documentation/dev-tools/index.rst b/Documentation/dev-tools/index.rst index f7809c7b1ba9..1b1cf4f5c9d9 100644 --- a/Documentation/dev-tools/index.rst +++ b/Documentation/dev-tools/index.rst @@ -22,6 +22,7 @@ whole; patches welcome! ubsan kmemleak kcsan + kfence gdb-kernel-debugging kgdb kselftest diff --git a/Documentation/dev-tools/kasan.rst b/Documentation/dev-tools/kasan.rst index a248ac3941be..ddf4239a5890 100644 --- a/Documentation/dev-tools/kasan.rst +++ b/Documentation/dev-tools/kasan.rst @@ -147,16 +147,15 @@ negative values to distinguish between different kinds of inaccessible memory like redzones or freed memory (see mm/kasan/kasan.h). In the report above the arrows point to the shadow byte 03, which means that -the accessed address is partially accessible. - -For tag-based KASAN this last report section shows the memory tags around the -accessed address (see `Implementation details`_ section). +the accessed address is partially accessible. For tag-based KASAN modes this +last report section shows the memory tags around the accessed address +(see the `Implementation details`_ section). Boot parameters ~~~~~~~~~~~~~~~ -Hardware tag-based KASAN mode (see the section about different mode below) is -intended for use in production as a security mitigation. Therefore it supports +Hardware tag-based KASAN mode (see the section about various modes below) is +intended for use in production as a security mitigation. Therefore, it supports boot parameters that allow to disable KASAN competely or otherwise control particular KASAN features. @@ -166,7 +165,8 @@ particular KASAN features. traces collection (default: ``on``). - ``kasan.fault=report`` or ``=panic`` controls whether to only print a KASAN - report or also panic the kernel (default: ``report``). + report or also panic the kernel (default: ``report``). Note, that tag + checking gets disabled after the first reported bug. For developers ~~~~~~~~~~~~~~ @@ -289,6 +289,16 @@ reserved to tag freed memory regions. Hardware tag-based KASAN currently only supports tagging of kmem_cache_alloc/kmalloc and page_alloc memory. +If the hardware doesn't support MTE (pre ARMv8.5), hardware tag-based KASAN +won't be enabled. In this case all boot parameters are ignored. + +Note, that enabling CONFIG_KASAN_HW_TAGS always results in in-kernel TBI being +enabled. Even when kasan.mode=off is provided, or when the hardware doesn't +support MTE (but supports TBI). + +Hardware tag-based KASAN only reports the first found bug. After that MTE tag +checking gets disabled. + What memory accesses are sanitised by KASAN? -------------------------------------------- @@ -352,17 +362,17 @@ unmapped. This will require changes in arch-specific code. This allows ``VMAP_STACK`` support on x86, and can simplify support of architectures that do not have a fixed module region. -CONFIG_KASAN_KUNIT_TEST & CONFIG_TEST_KASAN_MODULE --------------------------------------------------- +CONFIG_KASAN_KUNIT_TEST and CONFIG_KASAN_MODULE_TEST +---------------------------------------------------- -KASAN tests consist on two parts: +KASAN tests consist of two parts: 1. Tests that are integrated with the KUnit Test Framework. Enabled with ``CONFIG_KASAN_KUNIT_TEST``. These tests can be run and partially verified automatically in a few different ways, see the instructions below. 2. Tests that are currently incompatible with KUnit. Enabled with -``CONFIG_TEST_KASAN_MODULE`` and can only be run as a module. These tests can +``CONFIG_KASAN_MODULE_TEST`` and can only be run as a module. These tests can only be verified manually, by loading the kernel module and inspecting the kernel log for KASAN reports. diff --git a/Documentation/dev-tools/kfence.rst b/Documentation/dev-tools/kfence.rst new file mode 100644 index 000000000000..fdf04e741ea5 --- /dev/null +++ b/Documentation/dev-tools/kfence.rst @@ -0,0 +1,298 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. Copyright (C) 2020, Google LLC. + +Kernel Electric-Fence (KFENCE) +============================== + +Kernel Electric-Fence (KFENCE) is a low-overhead sampling-based memory safety +error detector. KFENCE detects heap out-of-bounds access, use-after-free, and +invalid-free errors. + +KFENCE is designed to be enabled in production kernels, and has near zero +performance overhead. Compared to KASAN, KFENCE trades performance for +precision. The main motivation behind KFENCE's design, is that with enough +total uptime KFENCE will detect bugs in code paths not typically exercised by +non-production test workloads. One way to quickly achieve a large enough total +uptime is when the tool is deployed across a large fleet of machines. + +Usage +----- + +To enable KFENCE, configure the kernel with:: + + CONFIG_KFENCE=y + +To build a kernel with KFENCE support, but disabled by default (to enable, set +``kfence.sample_interval`` to non-zero value), configure the kernel with:: + + CONFIG_KFENCE=y + CONFIG_KFENCE_SAMPLE_INTERVAL=0 + +KFENCE provides several other configuration options to customize behaviour (see +the respective help text in ``lib/Kconfig.kfence`` for more info). + +Tuning performance +~~~~~~~~~~~~~~~~~~ + +The most important parameter is KFENCE's sample interval, which can be set via +the kernel boot parameter ``kfence.sample_interval`` in milliseconds. The +sample interval determines the frequency with which heap allocations will be +guarded by KFENCE. The default is configurable via the Kconfig option +``CONFIG_KFENCE_SAMPLE_INTERVAL``. Setting ``kfence.sample_interval=0`` +disables KFENCE. + +The KFENCE memory pool is of fixed size, and if the pool is exhausted, no +further KFENCE allocations occur. With ``CONFIG_KFENCE_NUM_OBJECTS`` (default +255), the number of available guarded objects can be controlled. Each object +requires 2 pages, one for the object itself and the other one used as a guard +page; object pages are interleaved with guard pages, and every object page is +therefore surrounded by two guard pages. + +The total memory dedicated to the KFENCE memory pool can be computed as:: + + ( #objects + 1 ) * 2 * PAGE_SIZE + +Using the default config, and assuming a page size of 4 KiB, results in +dedicating 2 MiB to the KFENCE memory pool. + +Note: On architectures that support huge pages, KFENCE will ensure that the +pool is using pages of size ``PAGE_SIZE``. This will result in additional page +tables being allocated. + +Error reports +~~~~~~~~~~~~~ + +A typical out-of-bounds access looks like this:: + + ================================================================== + BUG: KFENCE: out-of-bounds read in test_out_of_bounds_read+0xa3/0x22b + + Out-of-bounds read at 0xffffffffb672efff (1B left of kfence-#17): + test_out_of_bounds_read+0xa3/0x22b + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + kfence-#17 [0xffffffffb672f000-0xffffffffb672f01f, size=32, cache=kmalloc-32] allocated by task 507: + test_alloc+0xf3/0x25b + test_out_of_bounds_read+0x98/0x22b + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + CPU: 4 PID: 107 Comm: kunit_try_catch Not tainted 5.8.0-rc6+ #7 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014 + ================================================================== + +The header of the report provides a short summary of the function involved in +the access. It is followed by more detailed information about the access and +its origin. Note that, real kernel addresses are only shown when using the +kernel command line option ``no_hash_pointers``. + +Use-after-free accesses are reported as:: + + ================================================================== + BUG: KFENCE: use-after-free read in test_use_after_free_read+0xb3/0x143 + + Use-after-free read at 0xffffffffb673dfe0 (in kfence-#24): + test_use_after_free_read+0xb3/0x143 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + kfence-#24 [0xffffffffb673dfe0-0xffffffffb673dfff, size=32, cache=kmalloc-32] allocated by task 507: + test_alloc+0xf3/0x25b + test_use_after_free_read+0x76/0x143 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + freed by task 507: + test_use_after_free_read+0xa8/0x143 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + CPU: 4 PID: 109 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014 + ================================================================== + +KFENCE also reports on invalid frees, such as double-frees:: + + ================================================================== + BUG: KFENCE: invalid free in test_double_free+0xdc/0x171 + + Invalid free of 0xffffffffb6741000: + test_double_free+0xdc/0x171 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + kfence-#26 [0xffffffffb6741000-0xffffffffb674101f, size=32, cache=kmalloc-32] allocated by task 507: + test_alloc+0xf3/0x25b + test_double_free+0x76/0x171 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + freed by task 507: + test_double_free+0xa8/0x171 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + CPU: 4 PID: 111 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014 + ================================================================== + +KFENCE also uses pattern-based redzones on the other side of an object's guard +page, to detect out-of-bounds writes on the unprotected side of the object. +These are reported on frees:: + + ================================================================== + BUG: KFENCE: memory corruption in test_kmalloc_aligned_oob_write+0xef/0x184 + + Corrupted memory at 0xffffffffb6797ff9 [ 0xac . . . . . . ] (in kfence-#69): + test_kmalloc_aligned_oob_write+0xef/0x184 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + kfence-#69 [0xffffffffb6797fb0-0xffffffffb6797ff8, size=73, cache=kmalloc-96] allocated by task 507: + test_alloc+0xf3/0x25b + test_kmalloc_aligned_oob_write+0x57/0x184 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + CPU: 4 PID: 120 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014 + ================================================================== + +For such errors, the address where the corruption occurred as well as the +invalidly written bytes (offset from the address) are shown; in this +representation, '.' denote untouched bytes. In the example above ``0xac`` is +the value written to the invalid address at offset 0, and the remaining '.' +denote that no following bytes have been touched. Note that, real values are +only shown if the kernel was booted with ``no_hash_pointers``; to avoid +information disclosure otherwise, '!' is used instead to denote invalidly +written bytes. + +And finally, KFENCE may also report on invalid accesses to any protected page +where it was not possible to determine an associated object, e.g. if adjacent +object pages had not yet been allocated:: + + ================================================================== + BUG: KFENCE: invalid read in test_invalid_access+0x26/0xe0 + + Invalid read at 0xffffffffb670b00a: + test_invalid_access+0x26/0xe0 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + CPU: 4 PID: 124 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014 + ================================================================== + +DebugFS interface +~~~~~~~~~~~~~~~~~ + +Some debugging information is exposed via debugfs: + +* The file ``/sys/kernel/debug/kfence/stats`` provides runtime statistics. + +* The file ``/sys/kernel/debug/kfence/objects`` provides a list of objects + allocated via KFENCE, including those already freed but protected. + +Implementation Details +---------------------- + +Guarded allocations are set up based on the sample interval. After expiration +of the sample interval, the next allocation through the main allocator (SLAB or +SLUB) returns a guarded allocation from the KFENCE object pool (allocation +sizes up to PAGE_SIZE are supported). At this point, the timer is reset, and +the next allocation is set up after the expiration of the interval. To "gate" a +KFENCE allocation through the main allocator's fast-path without overhead, +KFENCE relies on static branches via the static keys infrastructure. The static +branch is toggled to redirect the allocation to KFENCE. + +KFENCE objects each reside on a dedicated page, at either the left or right +page boundaries selected at random. The pages to the left and right of the +object page are "guard pages", whose attributes are changed to a protected +state, and cause page faults on any attempted access. Such page faults are then +intercepted by KFENCE, which handles the fault gracefully by reporting an +out-of-bounds access, and marking the page as accessible so that the faulting +code can (wrongly) continue executing (set ``panic_on_warn`` to panic instead). + +To detect out-of-bounds writes to memory within the object's page itself, +KFENCE also uses pattern-based redzones. For each object page, a redzone is set +up for all non-object memory. For typical alignments, the redzone is only +required on the unguarded side of an object. Because KFENCE must honor the +cache's requested alignment, special alignments may result in unprotected gaps +on either side of an object, all of which are redzoned. + +The following figure illustrates the page layout:: + + ---+-----------+-----------+-----------+-----------+-----------+--- + | xxxxxxxxx | O : | xxxxxxxxx | : O | xxxxxxxxx | + | xxxxxxxxx | B : | xxxxxxxxx | : B | xxxxxxxxx | + | x GUARD x | J : RED- | x GUARD x | RED- : J | x GUARD x | + | xxxxxxxxx | E : ZONE | xxxxxxxxx | ZONE : E | xxxxxxxxx | + | xxxxxxxxx | C : | xxxxxxxxx | : C | xxxxxxxxx | + | xxxxxxxxx | T : | xxxxxxxxx | : T | xxxxxxxxx | + ---+-----------+-----------+-----------+-----------+-----------+--- + +Upon deallocation of a KFENCE object, the object's page is again protected and +the object is marked as freed. Any further access to the object causes a fault +and KFENCE reports a use-after-free access. Freed objects are inserted at the +tail of KFENCE's freelist, so that the least recently freed objects are reused +first, and the chances of detecting use-after-frees of recently freed objects +is increased. + +Interface +--------- + +The following describes the functions which are used by allocators as well as +page handling code to set up and deal with KFENCE allocations. + +.. kernel-doc:: include/linux/kfence.h + :functions: is_kfence_address + kfence_shutdown_cache + kfence_alloc kfence_free __kfence_free + kfence_ksize kfence_object_start + kfence_handle_page_fault + +Related Tools +------------- + +In userspace, a similar approach is taken by `GWP-ASan +`_. GWP-ASan also relies on guard pages and +a sampling strategy to detect memory unsafety bugs at scale. KFENCE's design is +directly influenced by GWP-ASan, and can be seen as its kernel sibling. Another +similar but non-sampling approach, that also inspired the name "KFENCE", can be +found in the userspace `Electric Fence Malloc Debugger +`_. + +In the kernel, several tools exist to debug memory access errors, and in +particular KASAN can detect all bug classes that KFENCE can detect. While KASAN +is more precise, relying on compiler instrumentation, this comes at a +performance cost. + +It is worth highlighting that KASAN and KFENCE are complementary, with +different target environments. For instance, KASAN is the better debugging-aid, +where test cases or reproducers exists: due to the lower chance to detect the +error, it would require more effort using KFENCE to debug. Deployments at scale +that cannot afford to enable KASAN, however, would benefit from using KFENCE to +discover bugs due to code paths not exercised by test cases or fuzzers. diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst index c234a3ab3c34..848478838347 100644 --- a/Documentation/dev-tools/kunit/index.rst +++ b/Documentation/dev-tools/kunit/index.rst @@ -13,6 +13,7 @@ KUnit - Unit Testing for the Linux Kernel api/index style faq + tips What is KUnit? ============== @@ -88,6 +89,7 @@ How do I use it? ================ * :doc:`start` - for new users of KUnit +* :doc:`tips` - for short examples of best practices * :doc:`usage` - for a more detailed explanation of KUnit features * :doc:`api/index` - for the list of KUnit APIs used for testing * :doc:`kunit-tool` - for more information on the kunit_tool helper script diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst index 454f307813ea..0e65cabe08eb 100644 --- a/Documentation/dev-tools/kunit/start.rst +++ b/Documentation/dev-tools/kunit/start.rst @@ -196,8 +196,9 @@ Now add the following to ``drivers/misc/Kconfig``: .. code-block:: kconfig config MISC_EXAMPLE_TEST - bool "Test for my example" + tristate "Test for my example" if !KUNIT_ALL_TESTS depends on MISC_EXAMPLE && KUNIT=y + default KUNIT_ALL_TESTS and the following to ``drivers/misc/Makefile``: @@ -233,5 +234,7 @@ Congrats! You just wrote your first KUnit test! Next Steps ========== -* Check out the :doc:`usage` page for a more +* Check out the :doc:`tips` page for tips on + writing idiomatic KUnit tests. +* Optional: see the :doc:`usage` page for a more in-depth explanation of KUnit. diff --git a/Documentation/dev-tools/kunit/tips.rst b/Documentation/dev-tools/kunit/tips.rst new file mode 100644 index 000000000000..a6ca0af14098 --- /dev/null +++ b/Documentation/dev-tools/kunit/tips.rst @@ -0,0 +1,115 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================ +Tips For Writing KUnit Tests +============================ + +Exiting early on failed expectations +------------------------------------ + +``KUNIT_EXPECT_EQ`` and friends will mark the test as failed and continue +execution. In some cases, it's unsafe to continue and you can use the +``KUNIT_ASSERT`` variant to exit on failure. + +.. code-block:: c + + void example_test_user_alloc_function(struct kunit *test) + { + void *object = alloc_some_object_for_me(); + + /* Make sure we got a valid pointer back. */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, object); + do_something_with_object(object); + } + +Allocating memory +----------------- + +Where you would use ``kzalloc``, you should prefer ``kunit_kzalloc`` instead. +KUnit will ensure the memory is freed once the test completes. + +This is particularly useful since it lets you use the ``KUNIT_ASSERT_EQ`` +macros to exit early from a test without having to worry about remembering to +call ``kfree``. + +Example: + +.. code-block:: c + + void example_test_allocation(struct kunit *test) + { + char *buffer = kunit_kzalloc(test, 16, GFP_KERNEL); + /* Ensure allocation succeeded. */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buffer); + + KUNIT_ASSERT_STREQ(test, buffer, ""); + } + + +Testing static functions +------------------------ + +If you don't want to expose functions or variables just for testing, one option +is to conditionally ``#include`` the test file at the end of your .c file, e.g. + +.. code-block:: c + + /* In my_file.c */ + + static int do_interesting_thing(); + + #ifdef CONFIG_MY_KUNIT_TEST + #include "my_kunit_test.c" + #endif + +Injecting test-only code +------------------------ + +Similarly to the above, it can be useful to add test-specific logic. + +.. code-block:: c + + /* In my_file.h */ + + #ifdef CONFIG_MY_KUNIT_TEST + /* Defined in my_kunit_test.c */ + void test_only_hook(void); + #else + void test_only_hook(void) { } + #endif + +TODO(dlatypov@google.com): add an example of using ``current->kunit_test`` in +such a hook when it's not only updated for ``CONFIG_KASAN=y``. + +Customizing error messages +-------------------------- + +Each of the ``KUNIT_EXPECT`` and ``KUNIT_ASSERT`` macros have a ``_MSG`` variant. +These take a format string and arguments to provide additional context to the automatically generated error messages. + +.. code-block:: c + + char some_str[41]; + generate_sha1_hex_string(some_str); + + /* Before. Not easy to tell why the test failed. */ + KUNIT_EXPECT_EQ(test, strlen(some_str), 40); + + /* After. Now we see the offending string. */ + KUNIT_EXPECT_EQ_MSG(test, strlen(some_str), 40, "some_str='%s'", some_str); + +Alternatively, one can take full control over the error message by using ``KUNIT_FAIL()``, e.g. + +.. code-block:: c + + /* Before */ + KUNIT_EXPECT_EQ(test, some_setup_function(), 0); + + /* After: full control over the failure message. */ + if (some_setup_function()) + KUNIT_FAIL(test, "Failed to setup thing for testing"); + +Next Steps +========== +* Optional: see the :doc:`usage` page for a more + in-depth explanation of KUnit. diff --git a/Documentation/devicetree/bindings/Makefile b/Documentation/devicetree/bindings/Makefile index 8f2b054bec5a..780e5618ec0a 100644 --- a/Documentation/devicetree/bindings/Makefile +++ b/Documentation/devicetree/bindings/Makefile @@ -10,7 +10,7 @@ DT_SCHEMA_MIN_VERSION = 2020.8.1 PHONY += check_dtschema_version check_dtschema_version: @{ echo $(DT_SCHEMA_MIN_VERSION); \ - $(DT_DOC_CHECKER) --version 2>/dev/null || echo 0; } | sort -VC || \ + $(DT_DOC_CHECKER) --version 2>/dev/null || echo 0; } | sort -Vc >/dev/null || \ { echo "ERROR: dtschema minimum version is v$(DT_SCHEMA_MIN_VERSION)" >&2; false; } quiet_cmd_extract_ex = DTEX $@ @@ -78,10 +78,10 @@ $(obj)/processed-schema.json: $(DT_SCHEMA_FILES) check_dtschema_version FORCE endif -extra-$(CHECK_DT_BINDING) += processed-schema-examples.json -extra-$(CHECK_DTBS) += processed-schema.json -extra-$(CHECK_DT_BINDING) += $(patsubst $(src)/%.yaml,%.example.dts, $(DT_SCHEMA_FILES)) -extra-$(CHECK_DT_BINDING) += $(patsubst $(src)/%.yaml,%.example.dt.yaml, $(DT_SCHEMA_FILES)) +always-$(CHECK_DT_BINDING) += processed-schema-examples.json +always-$(CHECK_DTBS) += processed-schema.json +always-$(CHECK_DT_BINDING) += $(patsubst $(src)/%.yaml,%.example.dts, $(DT_SCHEMA_FILES)) +always-$(CHECK_DT_BINDING) += $(patsubst $(src)/%.yaml,%.example.dt.yaml, $(DT_SCHEMA_FILES)) # Hack: avoid 'Argument list too long' error for 'make clean'. Remove most of # build artifacts here before they are processed by scripts/Makefile.clean diff --git a/Documentation/devicetree/bindings/arm/atmel-sysregs.txt b/Documentation/devicetree/bindings/arm/atmel-sysregs.txt index 67719f15eb4c..807264a78edc 100644 --- a/Documentation/devicetree/bindings/arm/atmel-sysregs.txt +++ b/Documentation/devicetree/bindings/arm/atmel-sysregs.txt @@ -91,7 +91,8 @@ SHDWC SAMA5D2-Compatible Shutdown Controller 1) shdwc node required properties: -- compatible: should be "atmel,sama5d2-shdwc" or "microchip,sam9x60-shdwc". +- compatible: should be "atmel,sama5d2-shdwc", "microchip,sam9x60-shdwc" or + "microchip,sama7g5-shdwc" - reg: should contain registers location and length - clocks: phandle to input clock. - #address-cells: should be one. The cell is the wake-up input index. @@ -103,7 +104,7 @@ optional properties: microseconds. It's usually a board-related property. - atmel,wakeup-rtc-timer: boolean to enable Real-Time Clock wake-up. -optional microchip,sam9x60-shdwc properties: +optional microchip,sam9x60-shdwc or microchip,sama7g5-shdwc properties: - atmel,wakeup-rtt-timer: boolean to enable Real-time Timer Wake-up. The node contains child nodes for each wake-up input that the platform uses. diff --git a/Documentation/devicetree/bindings/arm/coresight.txt b/Documentation/devicetree/bindings/arm/coresight.txt index d711676b4a51..7f9c1ca87487 100644 --- a/Documentation/devicetree/bindings/arm/coresight.txt +++ b/Documentation/devicetree/bindings/arm/coresight.txt @@ -34,9 +34,12 @@ its hardware characteristcs. Program Flow Trace Macrocell: "arm,coresight-etm3x", "arm,primecell"; - - Embedded Trace Macrocell (version 4.x): + - Embedded Trace Macrocell (version 4.x), with memory mapped access. "arm,coresight-etm4x", "arm,primecell"; + - Embedded Trace Macrocell (version 4.x), with system register access only. + "arm,coresight-etm4x-sysreg"; + - Coresight programmable Replicator : "arm,coresight-dynamic-replicator", "arm,primecell"; diff --git a/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt b/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt index e31511255d8e..052a967c1f28 100644 --- a/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt +++ b/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt @@ -80,6 +80,11 @@ Required properties: - offset: offset address inside the syscon block +Optional properties: + +- marvell,pwm-offset: offset address of PWM duration control registers inside + the syscon block + Example: ap_syscon: system-controller@6f4000 { compatible = "syscon", "simple-mfd"; @@ -101,6 +106,9 @@ ap_syscon: system-controller@6f4000 { gpio-controller; #gpio-cells = <2>; gpio-ranges = <&ap_pinctrl 0 0 19>; + marvell,pwm-offset = <0x10c0>; + #pwm-cells = <2>; + clocks = <&ap_clk 3>; }; }; diff --git a/Documentation/devicetree/bindings/arm/rockchip.yaml b/Documentation/devicetree/bindings/arm/rockchip.yaml index 8a2dd9f1cff2..c3036f95c7bc 100644 --- a/Documentation/devicetree/bindings/arm/rockchip.yaml +++ b/Documentation/devicetree/bindings/arm/rockchip.yaml @@ -132,6 +132,7 @@ properties: - enum: - friendlyarm,nanopc-t4 - friendlyarm,nanopi-m4 + - friendlyarm,nanopi-m4b - friendlyarm,nanopi-neo4 - const: rockchip,rk3399 diff --git a/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml b/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml index 2e765bb3e6f6..7ca5375f278f 100644 --- a/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml +++ b/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml @@ -30,8 +30,8 @@ properties: Interrupts can be used to notify the completion of cache operations. The number of interrupts should match to the number of CPU cores. The specified interrupts correspond to CPU0, CPU1, ... in this order. - minItems: 1 - maxItems: 4 + minItems: 1 + maxItems: 4 cache-unified: true diff --git a/Documentation/devicetree/bindings/ata/sata_highbank.yaml b/Documentation/devicetree/bindings/ata/sata_highbank.yaml index 5e2a2394e600..ce75d77e9289 100644 --- a/Documentation/devicetree/bindings/ata/sata_highbank.yaml +++ b/Documentation/devicetree/bindings/ata/sata_highbank.yaml @@ -61,6 +61,7 @@ properties: maxItems: 8 calxeda,sgpio-gpio: + maxItems: 3 description: | phandle-gpio bank, bit offset, and default on or off, which indicates that the driver supports SGPIO indicator lights using the indicated diff --git a/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml b/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml new file mode 100644 index 000000000000..64ffff460026 --- /dev/null +++ b/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/auxdisplay/holtek,ht16k33.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Holtek HT16K33 RAM mapping 16*8 LED controller with keyscan + +maintainers: + - Robin van der Gracht + +allOf: + - $ref: "/schemas/input/matrix-keymap.yaml#" + +properties: + compatible: + const: holtek,ht16k33 + + reg: + maxItems: 1 + + refresh-rate-hz: + maxItems: 1 + description: Display update interval in Hertz + + interrupts: + maxItems: 1 + + debounce-delay-ms: + maxItems: 1 + description: Debouncing interval time in milliseconds + + linux,keymap: true + + linux,no-autorepeat: + description: Disable keyrepeat + + default-brightness-level: + minimum: 1 + maximum: 16 + default: 16 + description: Initial brightness level + +required: + - compatible + - reg + - refresh-rate-hz + +additionalProperties: false + +examples: + - | + #include + #include + i2c1 { + #address-cells = <1>; + #size-cells = <0>; + + ht16k33: ht16k33@70 { + compatible = "holtek,ht16k33"; + reg = <0x70>; + refresh-rate-hz = <20>; + interrupt-parent = <&gpio4>; + interrupts = <5 (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_EDGE_RISING)>; + debounce-delay-ms = <50>; + linux,keymap = , + , + , + , + , + , + , + , + , + ; + }; + }; diff --git a/Documentation/devicetree/bindings/clock/adi,axi-clkgen.yaml b/Documentation/devicetree/bindings/clock/adi,axi-clkgen.yaml index 0d06387184d6..983033fe5b17 100644 --- a/Documentation/devicetree/bindings/clock/adi,axi-clkgen.yaml +++ b/Documentation/devicetree/bindings/clock/adi,axi-clkgen.yaml @@ -20,6 +20,7 @@ properties: compatible: enum: - adi,axi-clkgen-2.00.a + - adi,zynqmp-axi-clkgen-2.00.a clocks: description: diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml index 3b45344ed758..a27025cd3909 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml @@ -41,6 +41,8 @@ properties: - allwinner,sun50i-h5-ccu - allwinner,sun50i-h6-ccu - allwinner,sun50i-h6-r-ccu + - allwinner,sun50i-h616-ccu + - allwinner,sun50i-h616-r-ccu - allwinner,suniv-f1c100s-ccu - nextthing,gr8-ccu @@ -82,6 +84,7 @@ if: - allwinner,sun50i-a64-r-ccu - allwinner,sun50i-a100-r-ccu - allwinner,sun50i-h6-r-ccu + - allwinner,sun50i-h616-r-ccu then: properties: @@ -100,6 +103,7 @@ else: enum: - allwinner,sun50i-a100-ccu - allwinner,sun50i-h6-ccu + - allwinner,sun50i-h616-ccu then: properties: diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clocks.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clks.yaml similarity index 95% rename from Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clocks.yaml rename to Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clks.yaml index fa0ee03a527f..6532fb6821bc 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clocks.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clks.yaml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+ %YAML 1.2 --- -$id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-usb-clocks.yaml# +$id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-usb-clks.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# title: Allwinner A80 USB Clock Controller Device Tree Bindings @@ -18,7 +18,7 @@ properties: const: 1 compatible: - const: allwinner,sun9i-a80-usb-clocks + const: allwinner,sun9i-a80-usb-clks reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/clock/arm,syscon-icst.yaml b/Documentation/devicetree/bindings/clock/arm,syscon-icst.yaml index eb241587efd1..118c5543e037 100644 --- a/Documentation/devicetree/bindings/clock/arm,syscon-icst.yaml +++ b/Documentation/devicetree/bindings/clock/arm,syscon-icst.yaml @@ -66,8 +66,8 @@ properties: - arm,syscon-icst525-integratorcp-cm-mem - arm,integrator-cm-auxosc - arm,versatile-cm-auxosc - - arm,impd-vco1 - - arm,impd-vco2 + - arm,impd1-vco1 + - arm,impd1-vco2 clocks: description: Parent clock for the ICST VCO diff --git a/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml b/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml index 565ca468cb44..7f5cf4001f76 100644 --- a/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml +++ b/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml @@ -22,6 +22,7 @@ properties: const: canaan,k210-clk clocks: + maxItems: 1 description: Phandle of the SoC 26MHz fixed-rate oscillator clock. diff --git a/Documentation/devicetree/bindings/clock/csr,atlas7-car.txt b/Documentation/devicetree/bindings/clock/csr,atlas7-car.txt deleted file mode 100644 index 54d6d1358339..000000000000 --- a/Documentation/devicetree/bindings/clock/csr,atlas7-car.txt +++ /dev/null @@ -1,55 +0,0 @@ -* Clock and reset bindings for CSR atlas7 - -Required properties: -- compatible: Should be "sirf,atlas7-car" -- reg: Address and length of the register set -- #clock-cells: Should be <1> -- #reset-cells: Should be <1> - -The clock consumer should specify the desired clock by having the clock -ID in its "clocks" phandle cell. -The ID list atlas7_clks defined in drivers/clk/sirf/clk-atlas7.c - -The reset consumer should specify the desired reset by having the reset -ID in its "reset" phandle cell. -The ID list atlas7_reset_unit defined in drivers/clk/sirf/clk-atlas7.c - -Examples: Clock and reset controller node: - -car: clock-controller@18620000 { - compatible = "sirf,atlas7-car"; - reg = <0x18620000 0x1000>; - #clock-cells = <1>; - #reset-cells = <1>; -}; - -Examples: Consumers using clock or reset: - -timer@10dc0000 { - compatible = "sirf,macro-tick"; - reg = <0x10dc0000 0x1000>; - clocks = <&car 54>; - interrupts = <0 0 0>, - <0 1 0>, - <0 2 0>, - <0 49 0>, - <0 50 0>, - <0 51 0>; -}; - -uart1: uart@18020000 { - cell-index = <1>; - compatible = "sirf,macro-uart"; - reg = <0x18020000 0x1000>; - clocks = <&clks 95>; - interrupts = <0 18 0>; - fifosize = <32>; -}; - -vpp@13110000 { - compatible = "sirf,prima2-vpp"; - reg = <0x13110000 0x10000>; - interrupts = <0 31 0>; - clocks = <&car 85>; - resets = <&car 29>; -}; diff --git a/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml b/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml index 2ac1131fd922..c268debe5b8d 100644 --- a/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml +++ b/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml @@ -59,6 +59,12 @@ properties: minItems: 1 maxItems: 2 + idt,xtal-load-femtofarads: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 9000 + maximum: 22760 + description: Optional load capacitor for XTAL1 and XTAL2 + patternProperties: "^OUT[1-4]$": type: object diff --git a/Documentation/devicetree/bindings/clock/imx27-clock.yaml b/Documentation/devicetree/bindings/clock/imx27-clock.yaml index a75365453dbc..160268f24487 100644 --- a/Documentation/devicetree/bindings/clock/imx27-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx27-clock.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Clock bindings for Freescale i.MX27 maintainers: - - Fabio Estevam + - Fabio Estevam description: | The clock consumer should specify the desired clock by having the clock diff --git a/Documentation/devicetree/bindings/clock/imx31-clock.yaml b/Documentation/devicetree/bindings/clock/imx31-clock.yaml index a25a374b3b2a..d2336261c922 100644 --- a/Documentation/devicetree/bindings/clock/imx31-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx31-clock.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Clock bindings for Freescale i.MX31 maintainers: - - Fabio Estevam + - Fabio Estevam description: | The clock consumer should specify the desired clock by having the clock diff --git a/Documentation/devicetree/bindings/clock/imx5-clock.yaml b/Documentation/devicetree/bindings/clock/imx5-clock.yaml index 90775c2669b8..b1740d7abe68 100644 --- a/Documentation/devicetree/bindings/clock/imx5-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx5-clock.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Clock bindings for Freescale i.MX5 maintainers: - - Fabio Estevam + - Fabio Estevam description: | The clock consumer should specify the desired clock by having the clock diff --git a/Documentation/devicetree/bindings/clock/intel,easic-n5x.yaml b/Documentation/devicetree/bindings/clock/intel,easic-n5x.yaml new file mode 100644 index 000000000000..8f45976e946e --- /dev/null +++ b/Documentation/devicetree/bindings/clock/intel,easic-n5x.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/intel,easic-n5x.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel SoCFPGA eASIC N5X platform clock controller binding + +maintainers: + - Dinh Nguyen + +description: + The Intel eASIC N5X Clock controller is an integrated clock controller, which + generates and supplies to all modules. + +properties: + compatible: + const: intel,easic-n5x-clkmgr + + '#clock-cells': + const: 1 + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - '#clock-cells' + +additionalProperties: false + +examples: + # Clock controller node: + - | + clkmgr: clock-controller@ffd10000 { + compatible = "intel,easic-n5x-clkmgr"; + reg = <0xffd10000 0x1000>; + clocks = <&osc1>; + #clock-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/mstar,msc313-mpll.yaml b/Documentation/devicetree/bindings/clock/mstar,msc313-mpll.yaml new file mode 100644 index 000000000000..0df5d75d4ebc --- /dev/null +++ b/Documentation/devicetree/bindings/clock/mstar,msc313-mpll.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/mstar,msc313-mpll.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MStar/Sigmastar MSC313 MPLL + +maintainers: + - Daniel Palmer + +description: | + The MStar/SigmaStar MSC313 and later ARMv7 chips have an MPLL block that + takes the external xtal input and multiplies it to create a high + frequency clock and divides that down into a number of clocks that + peripherals use. + +properties: + compatible: + const: mstar,msc313-mpll + + "#clock-cells": + const: 1 + + clocks: + maxItems: 1 + + reg: + maxItems: 1 + +required: + - compatible + - "#clock-cells" + - clocks + - reg + +additionalProperties: false + +examples: + - | + mpll@206000 { + compatible = "mstar,msc313-mpll"; + reg = <0x206000 0x200>; + #clock-cells = <1>; + clocks = <&xtal>; + }; diff --git a/Documentation/devicetree/bindings/clock/prima2-clock.txt b/Documentation/devicetree/bindings/clock/prima2-clock.txt deleted file mode 100644 index 5016979c0f78..000000000000 --- a/Documentation/devicetree/bindings/clock/prima2-clock.txt +++ /dev/null @@ -1,73 +0,0 @@ -* Clock bindings for CSR SiRFprimaII - -Required properties: -- compatible: Should be "sirf,prima2-clkc" -- reg: Address and length of the register set -- interrupts: Should contain clock controller interrupt -- #clock-cells: Should be <1> - -The clock consumer should specify the desired clock by having the clock -ID in its "clocks" phandle cell. The following is a full list of prima2 -clocks and IDs. - - Clock ID - --------------------------- - rtc 0 - osc 1 - pll1 2 - pll2 3 - pll3 4 - mem 5 - sys 6 - security 7 - dsp 8 - gps 9 - mf 10 - io 11 - cpu 12 - uart0 13 - uart1 14 - uart2 15 - tsc 16 - i2c0 17 - i2c1 18 - spi0 19 - spi1 20 - pwmc 21 - efuse 22 - pulse 23 - dmac0 24 - dmac1 25 - nand 26 - audio 27 - usp0 28 - usp1 29 - usp2 30 - vip 31 - gfx 32 - mm 33 - lcd 34 - vpp 35 - mmc01 36 - mmc23 37 - mmc45 38 - usbpll 39 - usb0 40 - usb1 41 - -Examples: - -clks: clock-controller@88000000 { - compatible = "sirf,prima2-clkc"; - reg = <0x88000000 0x1000>; - interrupts = <3>; - #clock-cells = <1>; -}; - -i2c0: i2c@b00e0000 { - cell-index = <0>; - compatible = "sirf,prima2-i2c"; - reg = <0xb00e0000 0x10000>; - interrupts = <24>; - clocks = <&clks 17>; -}; diff --git a/Documentation/devicetree/bindings/clock/qcom,a7pll.yaml b/Documentation/devicetree/bindings/clock/qcom,a7pll.yaml new file mode 100644 index 000000000000..8666e995725f --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,a7pll.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,a7pll.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm A7 PLL Binding + +maintainers: + - Manivannan Sadhasivam + +description: + The A7 PLL on the Qualcomm platforms like SDX55 is used to provide high + frequency clock to the CPU. + +properties: + compatible: + enum: + - qcom,sdx55-a7pll + + reg: + maxItems: 1 + + '#clock-cells': + const: 0 + + clocks: + items: + - description: board XO clock + + clock-names: + items: + - const: bi_tcxo + +required: + - compatible + - reg + - '#clock-cells' + +additionalProperties: false + +examples: + - | + #include + a7pll: clock@17808000 { + compatible = "qcom,sdx55-a7pll"; + reg = <0x17808000 0x1000>; + clocks = <&rpmhcc RPMH_CXO_CLK>; + clock-names = "bi_tcxo"; + #clock-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sc7280.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sc7280.yaml new file mode 100644 index 000000000000..5693b8997570 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sc7280.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gcc-sc7280.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for SC7280 + +maintainers: + - Taniya Das + +description: | + Qualcomm global clock control module which supports the clocks, resets and + power domains on SC7280. + + See also: + - dt-bindings/clock/qcom,gcc-sc7280.h + +properties: + compatible: + const: qcom,gcc-sc7280 + + clocks: + items: + - description: Board XO source + - description: Board active XO source + - description: Sleep clock source + - description: PCIE-0 pipe clock source + - description: PCIE-1 pipe clock source + - description: USF phy rx symbol 0 clock source + - description: USF phy rx symbol 1 clock source + - description: USF phy tx symbol 0 clock source + - description: USB30 phy wrapper pipe clock source + + clock-names: + items: + - const: bi_tcxo + - const: bi_tcxo_ao + - const: sleep_clk + - const: pcie_0_pipe_clk + - const: pcie_1_pipe_clk + - const: ufs_phy_rx_symbol_0_clk + - const: ufs_phy_rx_symbol_1_clk + - const: ufs_phy_tx_symbol_0_clk + - const: usb3_phy_wrapper_gcc_usb30_pipe_clk + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + +required: + - compatible + - clocks + - clock-names + - reg + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + clock-controller@100000 { + compatible = "qcom,gcc-sc7280"; + reg = <0x00100000 0x1f0000>; + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&rpmhcc RPMH_CXO_CLK_A>, + <&sleep_clk>, + <&pcie_0_pipe_clk>, <&pcie_1_pipe_clk>, + <&ufs_phy_rx_symbol_0_clk>, <&ufs_phy_rx_symbol_1_clk>, + <&ufs_phy_tx_symbol_0_clk>, + <&usb3_phy_wrapper_gcc_usb30_pipe_clk>; + + clock-names = "bi_tcxo", "bi_tcxo_ao", "sleep_clk", "pcie_0_pipe_clk", + "pcie_1_pipe_clk", "ufs_phy_rx_symbol_0_clk", + "ufs_phy_rx_symbol_1_clk", "ufs_phy_tx_symbol_0_clk", + "usb3_phy_wrapper_gcc_usb30_pipe_clk"; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sc8180x.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sc8180x.yaml new file mode 100644 index 000000000000..f03ef96e57fa --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sc8180x.yaml @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gcc-sc8180x.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for SC8180x + +maintainers: + - Bjorn Andersson + +description: | + Qualcomm global clock control module which supports the clocks, resets and + power domains on SC8180x. + + See also: + - dt-bindings/clock/qcom,gcc-sc8180x.h + +properties: + compatible: + const: qcom,gcc-sc8180x + + clocks: + items: + - description: Board XO source + - description: Board active XO source + - description: Sleep clock source + + clock-names: + items: + - const: bi_tcxo + - const: bi_tcxo_ao + - const: sleep_clk + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + + protected-clocks: + description: + Protected clock specifier list as per common clock binding. + +required: + - compatible + - clocks + - clock-names + - reg + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + clock-controller@100000 { + compatible = "qcom,gcc-sc8180x"; + reg = <0x00100000 0x1f0000>; + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&rpmhcc RPMH_CXO_CLK_A>, + <&sleep_clk>; + clock-names = "bi_tcxo", "bi_tcxo_ao", "sleep_clk"; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml new file mode 100644 index 000000000000..78f35832aa41 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gcc-sm8350.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for SM8350 + +maintainers: + - Vinod Koul + +description: | + Qualcomm global clock control module which supports the clocks, resets and + power domains on SM8350. + + See also: + - dt-bindings/clock/qcom,gcc-sm8350.h + +properties: + compatible: + const: qcom,gcc-sm8350 + + clocks: + items: + - description: Board XO source + - description: Sleep clock source + - description: PLL test clock source (Optional clock) + - description: PCIE 0 Pipe clock source (Optional clock) + - description: PCIE 1 Pipe clock source (Optional clock) + - description: UFS card Rx symbol 0 clock source (Optional clock) + - description: UFS card Rx symbol 1 clock source (Optional clock) + - description: UFS card Tx symbol 0 clock source (Optional clock) + - description: UFS phy Rx symbol 0 clock source (Optional clock) + - description: UFS phy Rx symbol 1 clock source (Optional clock) + - description: UFS phy Tx symbol 0 clock source (Optional clock) + - description: USB3 phy wrapper pipe clock source (Optional clock) + - description: USB3 phy sec pipe clock source (Optional clock) + minItems: 2 + maxItems: 13 + + clock-names: + items: + - const: bi_tcxo + - const: sleep_clk + - const: core_bi_pll_test_se # Optional clock + - const: pcie_0_pipe_clk # Optional clock + - const: pcie_1_pipe_clk # Optional clock + - const: ufs_card_rx_symbol_0_clk # Optional clock + - const: ufs_card_rx_symbol_1_clk # Optional clock + - const: ufs_card_tx_symbol_0_clk # Optional clock + - const: ufs_phy_rx_symbol_0_clk # Optional clock + - const: ufs_phy_rx_symbol_1_clk # Optional clock + - const: ufs_phy_tx_symbol_0_clk # Optional clock + - const: usb3_phy_wrapper_gcc_usb30_pipe_clk # Optional clock + - const: usb3_uni_phy_sec_gcc_usb30_pipe_clk # Optional clock + minItems: 2 + maxItems: 13 + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + +required: + - compatible + - clocks + - clock-names + - reg + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + clock-controller@100000 { + compatible = "qcom,gcc-sm8350"; + reg = <0x00100000 0x1f0000>; + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&sleep_clk>; + clock-names = "bi_tcxo", "sleep_clk"; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; + +... diff --git a/Documentation/devicetree/bindings/clock/qcom,gpucc-sdm660.yaml b/Documentation/devicetree/bindings/clock/qcom,gpucc-sdm660.yaml new file mode 100644 index 000000000000..3f70eb59aae3 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gpucc-sdm660.yaml @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gpucc-sdm660.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Graphics Clock & Reset Controller Binding for SDM630 and SDM660 + +maintainers: + - AngeloGioacchino Del Regno + +description: | + Qualcomm graphics clock control module which supports the clocks, resets and + power domains on SDM630 and SDM660. + + See also dt-bindings/clock/qcom,gpucc-sdm660.h. + +properties: + compatible: + enum: + - qcom,gpucc-sdm630 + - qcom,gpucc-sdm660 + + clocks: + items: + - description: Board XO source + - description: GPLL0 main gpu branch + - description: GPLL0 divider gpu branch + + clock-names: + items: + - const: xo + - const: gcc_gpu_gpll0_clk + - const: gcc_gpu_gpll0_div_clk + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - clock-names + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + #include + + clock-controller@5065000 { + compatible = "qcom,gpucc-sdm660"; + reg = <0x05065000 0x9038>; + clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>, + <&gcc GCC_GPU_GPLL0_CLK>, + <&gcc GCC_GPU_GPLL0_DIV_CLK>; + clock-names = "xo", "gcc_gpu_gpll0_clk", + "gcc_gpu_gpll0_div_clk"; + #clock-cells = <1>; + #power-domain-cells = <1>; + #reset-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml b/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml index af32dee14fc6..8b0b1c56f354 100644 --- a/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml @@ -24,6 +24,8 @@ properties: - qcom,mmcc-msm8974 - qcom,mmcc-msm8996 - qcom,mmcc-msm8998 + - qcom,mmcc-sdm630 + - qcom,mmcc-sdm660 clocks: items: diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml index 12c9cbc0ebf9..9ea0b3f5a4f2 100644 --- a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml @@ -18,6 +18,8 @@ properties: compatible: enum: - qcom,sc7180-rpmh-clk + - qcom,sc7280-rpmh-clk + - qcom,sc8180x-rpmh-clk - qcom,sdm845-rpmh-clk - qcom,sdx55-rpmh-clk - qcom,sm8150-rpmh-clk diff --git a/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml b/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml index 5be1229b3d6e..6eaabb4d82ec 100644 --- a/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml +++ b/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml @@ -35,6 +35,9 @@ properties: compatible: items: - enum: + - renesas,r8a774a1-rcar-usb2-clock-sel # RZ/G2M + - renesas,r8a774b1-rcar-usb2-clock-sel # RZ/G2N + - renesas,r8a774e1-rcar-usb2-clock-sel # RZ/G2H - renesas,r8a7795-rcar-usb2-clock-sel # R-Car H3 - renesas,r8a7796-rcar-usb2-clock-sel # R-Car M3-W - renesas,r8a77961-rcar-usb2-clock-sel # R-Car M3-W+ diff --git a/Documentation/devicetree/bindings/clock/silabs,si570.txt b/Documentation/devicetree/bindings/clock/silabs,si570.txt index 901935e929d2..5dda17df1ac5 100644 --- a/Documentation/devicetree/bindings/clock/silabs,si570.txt +++ b/Documentation/devicetree/bindings/clock/silabs,si570.txt @@ -28,6 +28,8 @@ Optional properties: - clock-frequency: Output frequency to generate. This defines the output frequency set during boot. It can be reprogrammed during runtime through the common clock framework. + - silabs,skip-recall: Do not perform NVM->RAM recall operation. It will rely + on hardware loading of RAM from NVM at power on. Example: si570: clock-generator@5d { diff --git a/Documentation/devicetree/bindings/clock/ste-u300-syscon-clock.txt b/Documentation/devicetree/bindings/clock/ste-u300-syscon-clock.txt deleted file mode 100644 index 7cafcb98ead7..000000000000 --- a/Documentation/devicetree/bindings/clock/ste-u300-syscon-clock.txt +++ /dev/null @@ -1,80 +0,0 @@ -Clock bindings for ST-Ericsson U300 System Controller Clocks - -Bindings for the gated system controller clocks: - -Required properties: -- compatible: must be "stericsson,u300-syscon-clk" -- #clock-cells: must be <0> -- clock-type: specifies the type of clock: - 0 = slow clock - 1 = fast clock - 2 = rest/remaining clock -- clock-id: specifies the clock in the type range - -Optional properties: -- clocks: parent clock(s) - -The available clocks per type are as follows: - -Type: ID: Clock: -------------------- -0 0 Slow peripheral bridge clock -0 1 UART0 clock -0 4 GPIO clock -0 6 RTC clock -0 7 Application timer clock -0 8 Access timer clock - -1 0 Fast peripheral bridge clock -1 1 I2C bus 0 clock -1 2 I2C bus 1 clock -1 5 MMC interface peripheral (silicon) clock -1 6 SPI clock - -2 3 CPU clock -2 4 DMA controller clock -2 5 External Memory Interface (EMIF) clock -2 6 NAND flask interface clock -2 8 XGAM graphics engine clock -2 9 Shared External Memory Interface (SEMI) clock -2 10 AHB Subsystem Bridge clock -2 12 Interrupt controller clock - -Example: - -gpio_clk: gpio_clk@13M { - #clock-cells = <0>; - compatible = "stericsson,u300-syscon-clk"; - clock-type = <0>; /* Slow */ - clock-id = <4>; - clocks = <&slow_clk>; -}; - -gpio: gpio@c0016000 { - compatible = "stericsson,gpio-coh901"; - (...) - clocks = <&gpio_clk>; -}; - - -Bindings for the MMC/SD card clock: - -Required properties: -- compatible: must be "stericsson,u300-syscon-mclk" -- #clock-cells: must be <0> - -Optional properties: -- clocks: parent clock(s) - -mmc_mclk: mmc_mclk { - #clock-cells = <0>; - compatible = "stericsson,u300-syscon-mclk"; - clocks = <&mmc_pclk>; -}; - -mmcsd: mmcsd@c0001000 { - compatible = "arm,pl18x", "arm,primecell"; - clocks = <&mmc_pclk>, <&mmc_mclk>; - clock-names = "apb_pclk", "mclk"; - (...) -}; diff --git a/Documentation/devicetree/bindings/clock/tango4-clock.txt b/Documentation/devicetree/bindings/clock/tango4-clock.txt deleted file mode 100644 index 19c580a7bda2..000000000000 --- a/Documentation/devicetree/bindings/clock/tango4-clock.txt +++ /dev/null @@ -1,23 +0,0 @@ -* Sigma Designs Tango4 Clock Generator - -The Tango4 clock generator outputs cpu_clk and sys_clk (the latter is used -for RAM and various peripheral devices). The clock binding described here -is applicable to all Tango4 SoCs. - -Required Properties: - -- compatible: should be "sigma,tango4-clkgen". -- reg: physical base address of the device and length of memory mapped region. -- clocks: phandle of the input clock (crystal oscillator). -- clock-output-names: should be "cpuclk" and "sysclk". -- #clock-cells: should be set to 1. - -Example: - - clkgen: clkgen@10000 { - compatible = "sigma,tango4-clkgen"; - reg = <0x10000 0x40>; - clocks = <&xtal>; - clock-output-names = "cpuclk", "sysclk"; - #clock-cells = <1>; - }; diff --git a/Documentation/devicetree/bindings/clock/zx296702-clk.txt b/Documentation/devicetree/bindings/clock/zx296702-clk.txt deleted file mode 100644 index 5c91c9e4f1be..000000000000 --- a/Documentation/devicetree/bindings/clock/zx296702-clk.txt +++ /dev/null @@ -1,34 +0,0 @@ -Device Tree Clock bindings for ZTE zx296702 - -This binding uses the common clock binding[1]. - -[1] Documentation/devicetree/bindings/clock/clock-bindings.txt - -Required properties: -- compatible : shall be one of the following: - "zte,zx296702-topcrm-clk": - zx296702 top clock selection, divider and gating - - "zte,zx296702-lsp0crpm-clk" and - "zte,zx296702-lsp1crpm-clk": - zx296702 device level clock selection and gating - -- reg: Address and length of the register set - -The clock consumer should specify the desired clock by having the clock -ID in its "clocks" phandle cell. See include/dt-bindings/clock/zx296702-clock.h -for the full list of zx296702 clock IDs. - - -topclk: topcrm@09800000 { - compatible = "zte,zx296702-topcrm-clk"; - reg = <0x09800000 0x1000>; - #clock-cells = <1>; -}; - -uart0: serial@09405000 { - compatible = "zte,zx296702-uart"; - reg = <0x09405000 0x1000>; - interrupts = ; - clocks = <&lsp1clk ZX296702_UART0_PCLK>; -}; diff --git a/Documentation/devicetree/bindings/clock/zx296718-clk.txt b/Documentation/devicetree/bindings/clock/zx296718-clk.txt deleted file mode 100644 index 3a46bf0b2540..000000000000 --- a/Documentation/devicetree/bindings/clock/zx296718-clk.txt +++ /dev/null @@ -1,37 +0,0 @@ -Device Tree Clock bindings for ZTE zx296718 - -This binding uses the common clock binding[1]. - -[1] Documentation/devicetree/bindings/clock/clock-bindings.txt - -Required properties: -- compatible : shall be one of the following: - "zte,zx296718-topcrm": - zx296718 top clock selection, divider and gating - - "zte,zx296718-lsp0crm" and - "zte,zx296718-lsp1crm": - zx296718 device level clock selection and gating - - "zte,zx296718-audiocrm": - zx296718 audio clock selection, divider and gating - -- reg: Address and length of the register set - -The clock consumer should specify the desired clock by having the clock -ID in its "clocks" phandle cell. See include/dt-bindings/clock/zx296718-clock.h -for the full list of zx296718 clock IDs. - - -topclk: topcrm@1461000 { - compatible = "zte,zx296718-topcrm-clk"; - reg = <0x01461000 0x1000>; - #clock-cells = <1>; -}; - -usbphy0:usb-phy0 { - compatible = "zte,zx296718-usb-phy"; - #phy-cells = <0>; - clocks = <&topclk USB20_PHY_CLK>; - clock-names = "phyclk"; -}; diff --git a/Documentation/devicetree/bindings/connector/usb-connector.yaml b/Documentation/devicetree/bindings/connector/usb-connector.yaml index ce11b2027199..b6daedd62516 100644 --- a/Documentation/devicetree/bindings/connector/usb-connector.yaml +++ b/Documentation/devicetree/bindings/connector/usb-connector.yaml @@ -155,22 +155,23 @@ properties: power dual role. ports: - description: OF graph bindings (specified in bindings/graph.txt) that model - any data bus to the connector unless the bus is between parent node and - the connector. Since a single connector can have multiple data buses every - bus has an assigned OF graph port number as described below. - type: object + $ref: /schemas/graph.yaml#/properties/ports + description: OF graph bindings modeling any data bus to the connector + unless the bus is between parent node and the connector. Since a single + connector can have multiple data buses every bus has an assigned OF graph + port number as described below. + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: High Speed (HS), present in all connectors. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Super Speed (SS), present in SS capable connectors. port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Sideband Use (SBU), present in USB-C. This describes the alternate mode connection of which SBU is a part. diff --git a/Documentation/devicetree/bindings/crypto/allwinner,sun8i-ce.yaml b/Documentation/devicetree/bindings/crypto/allwinner,sun8i-ce.yaml index 7a60d84289cc..6ab07eba7778 100644 --- a/Documentation/devicetree/bindings/crypto/allwinner,sun8i-ce.yaml +++ b/Documentation/devicetree/bindings/crypto/allwinner,sun8i-ce.yaml @@ -46,8 +46,7 @@ properties: if: properties: compatible: - items: - const: allwinner,sun50i-h6-crypto + const: allwinner,sun50i-h6-crypto then: properties: clocks: diff --git a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml new file mode 100644 index 000000000000..acb92706d280 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/intel,keembay-ocs-hcu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel Keem Bay OCS HCU Device Tree Bindings + +maintainers: + - Declan Murphy + - Daniele Alessandrelli + +description: + The Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash Control Unit (HCU) + provides hardware-accelerated hashing and HMAC. + +properties: + compatible: + const: intel,keembay-ocs-hcu + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + +additionalProperties: false + +examples: + - | + #include + crypto@3000b000 { + compatible = "intel,keembay-ocs-hcu"; + reg = <0x3000b000 0x1000>; + interrupts = ; + clocks = <&scmi_clk 94>; + }; diff --git a/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml b/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml index 7743eae049ab..676950bb7b37 100644 --- a/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml +++ b/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml @@ -8,7 +8,6 @@ title: Samsung Exynos SoC SlimSSS (Slim Security SubSystem) module maintainers: - Krzysztof Kozlowski - - Kamil Konieczny description: |+ The SlimSSS module in Exynos5433 SoC supports the following: diff --git a/Documentation/devicetree/bindings/crypto/samsung-sss.yaml b/Documentation/devicetree/bindings/crypto/samsung-sss.yaml index cf1c47a81d7f..6d62b0e42fc9 100644 --- a/Documentation/devicetree/bindings/crypto/samsung-sss.yaml +++ b/Documentation/devicetree/bindings/crypto/samsung-sss.yaml @@ -8,7 +8,6 @@ title: Samsung Exynos SoC SSS (Security SubSystem) module maintainers: - Krzysztof Kozlowski - - Kamil Konieczny description: |+ The SSS module in S5PV210 SoC supports the following: diff --git a/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml b/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml index 1465c9ebaf93..1d48ac712b23 100644 --- a/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml +++ b/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml @@ -66,7 +66,7 @@ examples: #include main_crypto: crypto@4e00000 { - compatible = "ti,j721-sa2ul"; + compatible = "ti,j721e-sa2ul"; reg = <0x4e00000 0x1200>; power-domains = <&k3_pds 264 TI_SCI_PD_EXCLUSIVE>; dmas = <&main_udmap 0xc000>, <&main_udmap 0x4000>, diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml index 86057d541065..12a7df0e38b2 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml @@ -84,36 +84,23 @@ properties: const: dma-mem ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Input endpoints of the controller. port@1: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Output endpoints of the controller. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml index 3eb1c2bbf4e7..055157fbf3bf 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml @@ -57,35 +57,22 @@ properties: maxItems: 1 ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. required: - - "#address-cells" - - "#size-cells" - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml index 75e6479397a5..7f11452539f4 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml @@ -76,37 +76,24 @@ properties: - const: audio-tx ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. Usually an HDMI connector. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml index 4c15a2644a7c..c13faf3e6581 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml @@ -115,31 +115,24 @@ properties: - const: lvds ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: | Output endpoints of the controller. patternProperties: "^endpoint(@[0-9])$": - type: object + $ref: /schemas/graph.yaml#/$defs/endpoint-base + unevaluatedProperties: false properties: allwinner,tcon-channel: @@ -156,16 +149,10 @@ properties: property is not present, the endpoint number will be used as the channel number. - unevaluatedProperties: true - required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml index 6009324be967..afc0ed799e0e 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml @@ -24,11 +24,9 @@ properties: maxItems: 1 port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: - A port node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. The - first port should be the input endpoint, usually coming from the + The first port should be the input endpoint, usually coming from the associated TCON. required: diff --git a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml index 0c1ce55940e1..71cce5687580 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml @@ -46,36 +46,23 @@ properties: maxItems: 1 ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml index 7aa330dabc44..a738d7c12a97 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml @@ -47,11 +47,9 @@ properties: const: dphy port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: - A port node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. That - port should be the input endpoint, usually coming from the + The port should be the input endpoint, usually coming from the associated TCON. required: diff --git a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml index c040eef56518..4f91eec26de9 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml @@ -43,35 +43,22 @@ properties: maxItems: 1 ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. required: - - "#address-cells" - - "#size-cells" - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml index fa4769a0b26e..b3e9992525c2 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml @@ -93,38 +93,25 @@ properties: The VCC power supply of the controller ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. Usually the associated TCON. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. Usually an HDMI connector. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml b/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml index b98ca609824b..ec21e8bf2767 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml @@ -80,141 +80,45 @@ properties: maxItems: 1 ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. - All ports should have only one endpoint connected to - remote endpoint. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoint for Mixer 0 mux. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoint for Mixer 0 mux - properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - - reg: true - - patternProperties: - "^endpoint@[0-9]$": - type: object - - properties: - reg: - description: | - ID of the target TCON - - required: - - reg - - required: - - "#address-cells" - - "#size-cells" - - additionalProperties: false - port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoint for Mixer 1 mux. port@3: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoint for Mixer 1 mux - properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - - reg: true - - patternProperties: - "^endpoint@[0-9]$": - type: object - - properties: - reg: - description: | - ID of the target TCON - - required: - - reg - - required: - - "#address-cells" - - "#size-cells" - - additionalProperties: false - port@4: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoint for HDMI mux. - properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - - reg: true - - patternProperties: - "^endpoint@[0-9]$": - type: object - - properties: - reg: - description: | - ID of the target TCON - - required: - - reg - - required: - - "#address-cells" - - "#size-cells" - - additionalProperties: false - port@5: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoint for HDMI mux required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - port@4 - port@5 - additionalProperties: false - required: - "#clock-cells" - compatible diff --git a/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml b/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml index 96de41d32b3e..637372ec4614 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml @@ -40,36 +40,23 @@ properties: maxItems: 1 ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml index 0da42ab8fd3a..cf5a208f2f10 100644 --- a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml @@ -81,12 +81,12 @@ properties: description: phandle to an external 5V regulator to power the HDMI logic port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to the VENC Input port node. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to the TMDS Output port node. diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml index a8d202c9d004..851cb0781217 100644 --- a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml +++ b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml @@ -83,12 +83,12 @@ properties: description: phandle to the associated power domain port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to the CVBS VDAC port node. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to the HDMI-TX port node. diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml index 7ce06f9f9f8e..57324a5f0271 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml @@ -53,6 +53,24 @@ properties: - const: audio - const: cec + interrupts: + items: + - description: CEC TX interrupt + - description: CEC RX interrupt + - description: CEC stuck at low interrupt + - description: Wake-up interrupt + - description: Hotplug connected interrupt + - description: Hotplug removed interrupt + + interrupt-names: + items: + - const: cec-tx + - const: cec-rx + - const: cec-low + - const: wakeup + - const: hpd-connected + - const: hpd-removed + ddc: allOf: - $ref: /schemas/types.yaml#/definitions/phandle @@ -60,6 +78,7 @@ properties: Phandle of the I2C controller used for DDC EDID probing hpd-gpios: + maxItems: 1 description: > The GPIO pin for the HDMI hotplug detect (if it doesn't appear as an interrupt/status bit in the HDMI controller itself) @@ -90,7 +109,7 @@ required: - resets - ddc -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-dpi.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-dpi.yaml index 5c1024bbc1b3..c9ad0ecc9b6d 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2835-dpi.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-dpi.yaml @@ -27,10 +27,9 @@ properties: - const: pixel port: - type: object - description: > - Port node with a single endpoint connecting to the panel, as - defined in Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/port + description: + Port node with a single endpoint connecting to the panel. required: - compatible diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-dsi0.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-dsi0.yaml index eb44e072b6e5..55c60919991f 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2835-dsi0.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-dsi0.yaml @@ -18,6 +18,7 @@ properties: compatible: enum: + - brcm,bcm2711-dsi1 - brcm,bcm2835-dsi0 - brcm,bcm2835-dsi1 diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-hdmi.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-hdmi.yaml index f54b4e4808f0..031e35e76db2 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2835-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-hdmi.yaml @@ -37,6 +37,7 @@ properties: Phandle of the I2C controller used for DDC EDID probing hpd-gpios: + maxItems: 1 description: > The GPIO pin for the HDMI hotplug detect (if it doesn't appear as an interrupt/status bit in the HDMI controller itself) diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml index e826ab0adb75..2e8566f47e63 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml @@ -36,7 +36,7 @@ if: properties: compatible: contains: - const: brcm,bcm2711-hvs" + const: brcm,bcm2711-hvs then: required: diff --git a/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml b/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml index 9392b5502a32..c789784efe30 100644 --- a/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml +++ b/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml @@ -35,16 +35,16 @@ properties: maxItems: 1 ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Video port for MIPI DSI input. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Video port for panel or connector. diff --git a/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml b/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml index 3ba477aefdd7..8e13f27b28ed 100644 --- a/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml +++ b/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml @@ -42,31 +42,18 @@ properties: description: Regulator for 1.0V digital core power. ports: - type: object - description: - A node containing input and output port nodes with endpoint - definitions as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - Documentation/devicetree/bindings/graph.txt + $ref: /schemas/graph.yaml#/properties/ports properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Video port for HDMI input. - properties: - reg: - const: 0 - port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Video port for SlimPort, DisplayPort, eDP or MyDP output. - properties: - reg: - const: 1 - required: - port@0 - port@1 diff --git a/Documentation/devicetree/bindings/display/bridge/anx6345.yaml b/Documentation/devicetree/bindings/display/bridge/anx6345.yaml index fccd63521a8c..1c0406c38fe5 100644 --- a/Documentation/devicetree/bindings/display/bridge/anx6345.yaml +++ b/Documentation/devicetree/bindings/display/bridge/anx6345.yaml @@ -32,31 +32,23 @@ properties: description: Regulator for 2.5V digital core power. ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for LVTTL input port@1: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for eDP output (panel or connector). May be omitted if EDID works reliably. required: - port@0 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/bridge/cdns,mhdp8546.yaml b/Documentation/devicetree/bindings/display/bridge/cdns,mhdp8546.yaml index 74d675fc6e7b..63427878715e 100644 --- a/Documentation/devicetree/bindings/display/bridge/cdns,mhdp8546.yaml +++ b/Documentation/devicetree/bindings/display/bridge/cdns,mhdp8546.yaml @@ -57,47 +57,37 @@ properties: maxItems: 1 ports: - type: object - description: - Ports as described in Documentation/devicetree/bindings/graph.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: First input port representing the DP bridge input. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Second input port representing the DP bridge input. port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Third input port representing the DP bridge input. port@3: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Fourth input port representing the DP bridge input. port@4: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Output port representing the DP bridge output. required: - port@0 - port@4 - - '#address-cells' - - '#size-cells' allOf: - if: diff --git a/Documentation/devicetree/bindings/display/bridge/chrontel,ch7033.yaml b/Documentation/devicetree/bindings/display/bridge/chrontel,ch7033.yaml index 9f38f55fc990..bb6289c7d375 100644 --- a/Documentation/devicetree/bindings/display/bridge/chrontel,ch7033.yaml +++ b/Documentation/devicetree/bindings/display/bridge/chrontel,ch7033.yaml @@ -19,16 +19,16 @@ properties: description: I2C address of the device ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Video port for RGB input. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | DVI port, should be connected to a node compatible with the dvi-connector binding. diff --git a/Documentation/devicetree/bindings/display/bridge/intel,keembay-dsi.yaml b/Documentation/devicetree/bindings/display/bridge/intel,keembay-dsi.yaml index 35c9dfd86650..dcb1336ee2a5 100644 --- a/Documentation/devicetree/bindings/display/bridge/intel,keembay-dsi.yaml +++ b/Documentation/devicetree/bindings/display/bridge/intel,keembay-dsi.yaml @@ -35,29 +35,21 @@ properties: - const: clk_mipi_cfg ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: MIPI DSI input port. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DSI output port. required: - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/bridge/ite,it6505.yaml b/Documentation/devicetree/bindings/display/bridge/ite,it6505.yaml index 02cfc0a3b550..833d11b2303a 100644 --- a/Documentation/devicetree/bindings/display/bridge/ite,it6505.yaml +++ b/Documentation/devicetree/bindings/display/bridge/ite,it6505.yaml @@ -53,7 +53,7 @@ properties: description: extcon specifier for the Power Delivery port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to DPI host port node required: diff --git a/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml b/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml index 7a1c89b995e2..5b9d36f7af30 100644 --- a/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml +++ b/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml @@ -38,82 +38,26 @@ properties: description: Regulator for 3.3V IO power. ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Primary MIPI port-1 for MIPI input - properties: - reg: - const: 0 - - patternProperties: - "^endpoint(@[0-9])$": - type: object - additionalProperties: false - - properties: - remote-endpoint: - $ref: /schemas/types.yaml#/definitions/phandle - - required: - - reg - port@1: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Additional MIPI port-2 for MIPI input, used in combination with primary MIPI port-1 to drive higher resolution displays - properties: - reg: - const: 1 - - patternProperties: - "^endpoint(@[0-9])$": - type: object - additionalProperties: false - - properties: - remote-endpoint: - $ref: /schemas/types.yaml#/definitions/phandle - - required: - - reg - port@2: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: HDMI port for HDMI output - properties: - reg: - const: 2 - - patternProperties: - "^endpoint(@[0-9])$": - type: object - additionalProperties: false - - properties: - remote-endpoint: - $ref: /schemas/types.yaml#/definitions/phandle - - required: - - reg - required: - - "#address-cells" - - "#size-cells" - port@0 - port@2 diff --git a/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml b/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml index 66a14d60ce1d..304a1367faaa 100644 --- a/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml +++ b/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml @@ -45,25 +45,17 @@ properties: - thine,thc63lvdm83d # For the THC63LVDM83D LVDS serializer ports: - type: object - description: | - This device has two video ports. Their connections are modeled using the - OF graph bindings specified in Documentation/devicetree/bindings/graph.txt + $ref: /schemas/graph.yaml#/properties/ports + properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | For LVDS encoders, port 0 is the parallel input For LVDS decoders, port 0 is the LVDS input port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | For LVDS encoders, port 1 is the LVDS output For LVDS decoders, port 1 is the parallel output @@ -72,8 +64,6 @@ properties: - port@0 - port@1 - additionalProperties: false - powerdown-gpios: description: The GPIO used to control the power down line of this device. diff --git a/Documentation/devicetree/bindings/display/bridge/nwl-dsi.yaml b/Documentation/devicetree/bindings/display/bridge/nwl-dsi.yaml index a125b2dd3a2f..350fb8f400f0 100644 --- a/Documentation/devicetree/bindings/display/bridge/nwl-dsi.yaml +++ b/Documentation/devicetree/bindings/display/bridge/nwl-dsi.yaml @@ -84,40 +84,23 @@ properties: - const: pclk ports: - type: object - description: - A node containing DSI input & output port nodes with endpoint - definitions as documented in - Documentation/devicetree/bindings/graph.txt. + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/$defs/port-base description: Input port node to receive pixel data from the display controller. Exactly one endpoint must be specified. properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - endpoint@0: + $ref: /schemas/graph.yaml#/properties/endpoint description: sub-node describing the input from LCDIF - type: object endpoint@1: + $ref: /schemas/graph.yaml#/properties/endpoint description: sub-node describing the input from DCSS - type: object - - reg: - const: 0 - - required: - - '#address-cells' - - '#size-cells' - - reg oneOf: - required: @@ -125,28 +108,18 @@ properties: - required: - endpoint@1 - additionalProperties: false + unevaluatedProperties: false port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DSI output port node to the panel or the next bridge in the chain - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - required: - - '#address-cells' - - '#size-cells' - port@0 - port@1 - additionalProperties: false - required: - '#address-cells' - '#size-cells' diff --git a/Documentation/devicetree/bindings/display/bridge/ps8640.yaml b/Documentation/devicetree/bindings/display/bridge/ps8640.yaml index 763c7909473e..fce82b605c8b 100644 --- a/Documentation/devicetree/bindings/display/bridge/ps8640.yaml +++ b/Documentation/devicetree/bindings/display/bridge/ps8640.yaml @@ -41,34 +41,22 @@ properties: description: Regulator for 3.3V digital core power. ports: - type: object - description: - A node containing DSI input & output port nodes with endpoint - definitions as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - Documentation/devicetree/bindings/graph.txt + $ref: /schemas/graph.yaml#/properties/ports + properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for DSI input port@1: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for eDP output (panel or connector). required: - port@0 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/bridge/renesas,lvds.yaml b/Documentation/devicetree/bindings/display/bridge/renesas,lvds.yaml index e5b163951b91..acfc327f70a7 100644 --- a/Documentation/devicetree/bindings/display/bridge/renesas,lvds.yaml +++ b/Documentation/devicetree/bindings/display/bridge/renesas,lvds.yaml @@ -49,33 +49,21 @@ properties: maxItems: 1 ports: - type: object - description: | - This device has two video ports. Their connections are modelled using the - OF graph bindings specified in Documentation/devicetree/bindings/graph.txt. - Each port shall have a single endpoint. + $ref: /schemas/graph.yaml#/properties/ports properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Parallel RGB input port port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: LVDS output port required: - port@0 - port@1 - additionalProperties: false - power-domains: maxItems: 1 @@ -83,9 +71,9 @@ properties: $ref: /schemas/types.yaml#/definitions/phandle description: phandle to the companion LVDS encoder. This property is mandatory - for the first LVDS encoder on D3 and E3 SoCs, and shall point to - the second encoder to be used as a companion in dual-link mode. It - shall not be set for any other LVDS encoder. + for the first LVDS encoder on R-Car D3 and E3, and RZ/G2E SoCs, and shall + point to the second encoder to be used as a companion in dual-link mode. + It shall not be set for any other LVDS encoder. required: - compatible diff --git a/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml b/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml index 64e8a1c24b40..6c7b577fd471 100644 --- a/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml +++ b/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml @@ -30,31 +30,21 @@ properties: - ti,ths8135 ports: - type: object - description: | - This device has two video ports. Their connections are modeled using the - OF graph bindings specified in Documentation/devicetree/bindings/graph.txt. + $ref: /schemas/graph.yaml#/properties/ports + properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The bridge input port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The bridge output required: - port@0 - port@1 - additionalProperties: false - enable-gpios: maxItems: 1 description: GPIO controlling bridge enable diff --git a/Documentation/devicetree/bindings/display/bridge/snps,dw-mipi-dsi.yaml b/Documentation/devicetree/bindings/display/bridge/snps,dw-mipi-dsi.yaml index e42cb610f545..3c3e51af154b 100644 --- a/Documentation/devicetree/bindings/display/bridge/snps,dw-mipi-dsi.yaml +++ b/Documentation/devicetree/bindings/display/bridge/snps,dw-mipi-dsi.yaml @@ -47,14 +47,15 @@ properties: const: apb ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Input node to receive pixel data. + port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DSI output node to panel. required: diff --git a/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.yaml b/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.yaml index 3d5ce08a5792..8ae382429d2b 100644 --- a/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.yaml +++ b/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.yaml @@ -25,46 +25,41 @@ properties: const: thine,thc63lvd1024 ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports description: | - This device has four video ports. Their connections are modeled using the - OF graph bindings specified in Documentation/devicetree/bindings/graph.txt. + The device can operate in single or dual input and output modes. - The device can operate in single-link mode or dual-link mode. In - single-link mode, all pixels are received on port@0, and port@1 shall not - contain any endpoint. In dual-link mode, even-numbered pixels are - received on port@0 and odd-numbered pixels on port@1, and both port@0 and - port@1 shall contain endpoints. + When operating in single input mode, all pixels are received on port@0, + and port@1 shall not contain any endpoint. In dual input mode, + even-numbered pixels are received on port@0 and odd-numbered pixels on + port@1, and both port@0 and port@1 shall contain endpoints. + + When operating in single output mode all pixels are output from the first + CMOS/TTL port and port@3 shall not contain any endpoint. In dual output + mode pixels are output from both CMOS/TTL ports and both port@2 and + port@3 shall contain endpoints. properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: First LVDS input port port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Second LVDS input port port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: First digital CMOS/TTL parallel output port@3: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Second digital CMOS/TTL parallel output required: - port@0 - port@2 - additionalProperties: false - oe-gpios: maxItems: 1 description: Output enable GPIO signal, pin name "OE", active high. diff --git a/Documentation/devicetree/bindings/display/bridge/ti,sn65dsi86.yaml b/Documentation/devicetree/bindings/display/bridge/ti,sn65dsi86.yaml index f8622bd0f61e..26932d2e86ab 100644 --- a/Documentation/devicetree/bindings/display/bridge/ti,sn65dsi86.yaml +++ b/Documentation/devicetree/bindings/display/bridge/ti,sn65dsi86.yaml @@ -71,54 +71,26 @@ properties: description: See ../../pwm/pwm.yaml for description of the cell formats. ports: - type: object - additionalProperties: false + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object - additionalProperties: false - + $ref: /schemas/graph.yaml#/properties/port description: Video port for MIPI DSI input - properties: - reg: - const: 0 - - endpoint: - type: object - additionalProperties: false - properties: - remote-endpoint: true - - required: - - reg - port@1: - type: object - additionalProperties: false - + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: Video port for eDP output (panel or connector). properties: - reg: - const: 1 - endpoint: - type: object - additionalProperties: false + $ref: /schemas/graph.yaml#/$defs/endpoint-base + unevaluatedProperties: false properties: - remote-endpoint: true - data-lanes: oneOf: - minItems: 1 @@ -171,12 +143,7 @@ properties: dependencies: lane-polarities: [data-lanes] - required: - - reg - required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 diff --git a/Documentation/devicetree/bindings/display/bridge/ti,tfp410.yaml b/Documentation/devicetree/bindings/display/bridge/ti,tfp410.yaml index 605831c1e836..4c5dd8ec2951 100644 --- a/Documentation/devicetree/bindings/display/bridge/ti,tfp410.yaml +++ b/Documentation/devicetree/bindings/display/bridge/ti,tfp410.yaml @@ -31,23 +31,18 @@ properties: maximum: 7 ports: - description: - A node containing input and output port nodes with endpoint - definitions as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: port@0: + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: DPI input port. - type: object properties: - reg: - const: 0 - endpoint: - type: object + $ref: /schemas/graph.yaml#/$defs/endpoint-base + unevaluatedProperties: false properties: pclk-sample: @@ -67,15 +62,8 @@ properties: default: 24 port@1: + $ref: /schemas/graph.yaml#/properties/port description: DVI output port. - type: object - - properties: - reg: - const: 1 - - endpoint: - type: object required: - port@0 diff --git a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358762.yaml b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358762.yaml index 195025e6803c..5216c27fc0ad 100644 --- a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358762.yaml +++ b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358762.yaml @@ -25,62 +25,20 @@ properties: description: Regulator for 1.2V internal core power. ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object - additionalProperties: false - - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for MIPI DSI input - properties: - reg: - const: 0 - - patternProperties: - endpoint: - type: object - additionalProperties: false - - properties: - remote-endpoint: true - - required: - - reg - port@1: - type: object - additionalProperties: false - - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for MIPI DPI output (panel or connector). - properties: - reg: - const: 1 - - patternProperties: - endpoint: - type: object - additionalProperties: false - - properties: - remote-endpoint: true - - required: - - reg - required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 diff --git a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358768.yaml b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358768.yaml index c036a75db8f7..eacfe7165083 100644 --- a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358768.yaml +++ b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358768.yaml @@ -42,65 +42,30 @@ properties: const: refclk ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object - additionalProperties: false - + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: | Video port for RGB input properties: - reg: - const: 0 - - patternProperties: endpoint: - type: object - additionalProperties: false + $ref: /schemas/graph.yaml#/$defs/endpoint-base + unevaluatedProperties: false properties: data-lines: enum: [ 16, 18, 24 ] - remote-endpoint: true - - required: - - reg - port@1: - type: object - additionalProperties: false - + $ref: /schemas/graph.yaml#/properties/port description: | Video port for DSI output (panel or connector). - properties: - reg: - const: 1 - - patternProperties: - endpoint: - type: object - additionalProperties: false - - properties: - remote-endpoint: true - - required: - - reg - required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 @@ -156,4 +121,3 @@ examples: }; }; }; - diff --git a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml index b5959cc78b8d..10471c6c1ff9 100644 --- a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml +++ b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml @@ -42,31 +42,22 @@ properties: description: Hardware reset, Low active ports: - type: object - description: - A node containing input and output port nodes with endpoint definitions - as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt + $ref: /schemas/graph.yaml#/properties/ports + properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | DSI Input. The remote endpoint phandle should be a reference to a valid mipi_dsi_host device node. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Video port for LVDS output (panel or connector). port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Video port for Dual link LVDS output (panel or connector). diff --git a/Documentation/devicetree/bindings/display/connector/analog-tv-connector.yaml b/Documentation/devicetree/bindings/display/connector/analog-tv-connector.yaml index eebe88fed999..a31ca2d52b86 100644 --- a/Documentation/devicetree/bindings/display/connector/analog-tv-connector.yaml +++ b/Documentation/devicetree/bindings/display/connector/analog-tv-connector.yaml @@ -25,6 +25,7 @@ properties: $ref: /schemas/types.yaml#/definitions/uint32 port: + $ref: /schemas/graph.yaml#/properties/port description: Connection to controller providing analog TV signals required: diff --git a/Documentation/devicetree/bindings/display/connector/dp-connector.yaml b/Documentation/devicetree/bindings/display/connector/dp-connector.yaml new file mode 100644 index 000000000000..22792a79e7ce --- /dev/null +++ b/Documentation/devicetree/bindings/display/connector/dp-connector.yaml @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/connector/dp-connector.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: DisplayPort Connector + +maintainers: + - Tomi Valkeinen + +properties: + compatible: + const: dp-connector + + label: true + + type: + enum: + - full-size + - mini + + hpd-gpios: + description: A GPIO line connected to HPD + maxItems: 1 + + dp-pwr-supply: + description: Power supply for the DP_PWR pin + + port: + $ref: /schemas/graph.yaml#/properties/port + description: Connection to controller providing DP signals + +required: + - compatible + - type + - port + +additionalProperties: false + +examples: + - | + connector { + compatible = "dp-connector"; + label = "dp0"; + type = "full-size"; + + port { + dp_connector_in: endpoint { + remote-endpoint = <&dp_out>; + }; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/display/connector/dvi-connector.yaml b/Documentation/devicetree/bindings/display/connector/dvi-connector.yaml index 71cb9220fa59..93eb14294e68 100644 --- a/Documentation/devicetree/bindings/display/connector/dvi-connector.yaml +++ b/Documentation/devicetree/bindings/display/connector/dvi-connector.yaml @@ -36,6 +36,7 @@ properties: description: the connector has pins for DVI dual-link port: + $ref: /schemas/graph.yaml#/properties/port description: Connection to controller providing DVI signals required: diff --git a/Documentation/devicetree/bindings/display/connector/hdmi-connector.yaml b/Documentation/devicetree/bindings/display/connector/hdmi-connector.yaml index 14d7128af592..83c0d008265b 100644 --- a/Documentation/devicetree/bindings/display/connector/hdmi-connector.yaml +++ b/Documentation/devicetree/bindings/display/connector/hdmi-connector.yaml @@ -37,6 +37,7 @@ properties: maxItems: 1 port: + $ref: /schemas/graph.yaml#/properties/port description: Connection to controller providing HDMI signals required: diff --git a/Documentation/devicetree/bindings/display/connector/vga-connector.yaml b/Documentation/devicetree/bindings/display/connector/vga-connector.yaml index 5782c4bb3252..25f868002000 100644 --- a/Documentation/devicetree/bindings/display/connector/vga-connector.yaml +++ b/Documentation/devicetree/bindings/display/connector/vga-connector.yaml @@ -20,6 +20,7 @@ properties: $ref: /schemas/types.yaml#/definitions/phandle port: + $ref: /schemas/graph.yaml#/properties/port description: Connection to controller providing VGA signals required: diff --git a/Documentation/devicetree/bindings/display/ht16k33.txt b/Documentation/devicetree/bindings/display/ht16k33.txt deleted file mode 100644 index d5a8b070b467..000000000000 --- a/Documentation/devicetree/bindings/display/ht16k33.txt +++ /dev/null @@ -1,40 +0,0 @@ -Holtek ht16k33 RAM mapping 16*8 LED controller driver with keyscan -------------------------------------------------------------------------------- - -Required properties: -- compatible: "holtek,ht16k33" -- reg: I2C slave address of the chip. -- interrupts: Interrupt specification for the key pressed interrupt. -- refresh-rate-hz: Display update interval in HZ. -- debounce-delay-ms: Debouncing interval time in milliseconds. -- linux,keymap: The keymap for keys as described in the binding - document (devicetree/bindings/input/matrix-keymap.txt). - -Optional properties: -- linux,no-autorepeat: Disable keyrepeat. -- default-brightness-level: Initial brightness level [0-15] (default: 15). - -Example: - -&i2c1 { - ht16k33: ht16k33@70 { - compatible = "holtek,ht16k33"; - reg = <0x70>; - refresh-rate-hz = <20>; - debounce-delay-ms = <50>; - interrupt-parent = <&gpio4>; - interrupts = <5 (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_EDGE_RISING)>; - linux,keymap = < - MATRIX_KEY(2, 0, KEY_F6) - MATRIX_KEY(3, 0, KEY_F8) - MATRIX_KEY(4, 0, KEY_F10) - MATRIX_KEY(5, 0, KEY_F4) - MATRIX_KEY(6, 0, KEY_F2) - MATRIX_KEY(2, 1, KEY_F5) - MATRIX_KEY(3, 1, KEY_F7) - MATRIX_KEY(4, 1, KEY_F9) - MATRIX_KEY(5, 1, KEY_F3) - MATRIX_KEY(6, 1, KEY_F1) - >; - }; -}; diff --git a/Documentation/devicetree/bindings/display/imx/nxp,imx8mq-dcss.yaml b/Documentation/devicetree/bindings/display/imx/nxp,imx8mq-dcss.yaml index f1f25aa794d9..0091df9dd73b 100644 --- a/Documentation/devicetree/bindings/display/imx/nxp,imx8mq-dcss.yaml +++ b/Documentation/devicetree/bindings/display/imx/nxp,imx8mq-dcss.yaml @@ -74,7 +74,7 @@ properties: - description: Must be 400 MHz port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to the input port of a HDMI/DP or MIPI display bridge. diff --git a/Documentation/devicetree/bindings/display/ingenic,ipu.yaml b/Documentation/devicetree/bindings/display/ingenic,ipu.yaml index 12064a8e7a92..e679f48a3886 100644 --- a/Documentation/devicetree/bindings/display/ingenic,ipu.yaml +++ b/Documentation/devicetree/bindings/display/ingenic,ipu.yaml @@ -31,9 +31,8 @@ properties: clock-names: const: ipu -patternProperties: - "^ports?$": - description: OF graph bindings (specified in bindings/graph.txt). + port: + $ref: /schemas/graph.yaml#/properties/port required: - compatible diff --git a/Documentation/devicetree/bindings/display/ingenic,lcd.yaml b/Documentation/devicetree/bindings/display/ingenic,lcd.yaml index 768050f30dba..50d2b0a50e8a 100644 --- a/Documentation/devicetree/bindings/display/ingenic,lcd.yaml +++ b/Documentation/devicetree/bindings/display/ingenic,lcd.yaml @@ -39,18 +39,18 @@ properties: minItems: 1 port: - description: OF graph bindings (specified in bindings/graph.txt). + $ref: /schemas/graph.yaml#/properties/port ports: - description: OF graph bindings (specified in bindings/graph.txt). - type: object + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DPI output, to interface with TFT panels. port@8: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Link to the Image Processing Unit (IPU). (See ingenic,ipu.yaml). diff --git a/Documentation/devicetree/bindings/display/intel,keembay-display.yaml b/Documentation/devicetree/bindings/display/intel,keembay-display.yaml index 0a697d45c2ad..bc6622b010ca 100644 --- a/Documentation/devicetree/bindings/display/intel,keembay-display.yaml +++ b/Documentation/devicetree/bindings/display/intel,keembay-display.yaml @@ -36,7 +36,7 @@ properties: maxItems: 1 port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Display output node to DSI. required: diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,disp.txt b/Documentation/devicetree/bindings/display/mediatek/mediatek,disp.txt index ed76332ec01e..93b160df3eec 100644 --- a/Documentation/devicetree/bindings/display/mediatek/mediatek,disp.txt +++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,disp.txt @@ -37,13 +37,14 @@ Required properties (all function blocks): "mediatek,-disp-aal" - adaptive ambient light controller "mediatek,-disp-gamma" - gamma correction "mediatek,-disp-merge" - merge streams from two RDMA sources + "mediatek,-disp-postmask" - control round corner for display frame "mediatek,-disp-split" - split stream to two encoders "mediatek,-disp-ufoe" - data compression engine "mediatek,-dsi" - DSI controller, see mediatek,dsi.txt "mediatek,-dpi" - DPI controller, see mediatek,dpi.txt "mediatek,-disp-mutex" - display mutex "mediatek,-disp-od" - overdrive - the supported chips are mt2701, mt7623, mt2712, mt8167 and mt8173. + the supported chips are mt2701, mt7623, mt2712, mt8167, mt8173, mt8183 and mt8192. - reg: Physical base address and length of the function block register space - interrupts: The interrupt signal from the function block (required, except for merge and split function blocks). @@ -66,6 +67,14 @@ Required properties (DMA function blocks): argument, see Documentation/devicetree/bindings/iommu/mediatek,iommu.txt for details. +Optional properties (RDMA function blocks): +- mediatek,rdma-fifo-size: rdma fifo size may be different even in same SOC, add this + property to the corresponding rdma + the value is the Max value which defined in hardware data sheet. + mediatek,rdma-fifo-size of mt8173-rdma0 is 8K + mediatek,rdma-fifo-size of mt8183-rdma0 is 5K + mediatek,rdma-fifo-size of mt8183-rdma1 is 2K + Examples: mmsys: clock-controller@14000000 { @@ -103,6 +112,7 @@ rdma0: rdma@1400e000 { clocks = <&mmsys CLK_MM_DISP_RDMA0>; iommus = <&iommu M4U_PORT_DISP_RDMA0>; mediatek,larb = <&larb0>; + mediatek,rdma-fifosize = <8192>; }; rdma1: rdma@1400f000 { diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt b/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt index f06f24d405a5..8238a86686be 100644 --- a/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt +++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt @@ -22,23 +22,7 @@ Required properties: MIPI TX Configuration Module ============================ -The MIPI TX configuration module controls the MIPI D-PHY. - -Required properties: -- compatible: "mediatek,-mipi-tx" -- the supported chips are mt2701, 7623, mt8173 and mt8183. -- reg: Physical base address and length of the controller's registers -- clocks: PLL reference clock -- clock-output-names: name of the output clock line to the DSI encoder -- #clock-cells: must be <0>; -- #phy-cells: must be <0>. - -Optional properties: -- drive-strength-microamp: adjust driving current, should be 3000 ~ 6000. And - the step is 200. -- nvmem-cells: A phandle to the calibration data provided by a nvmem device. If - unspecified default values shall be used. -- nvmem-cell-names: Should be "calibration-data" +See phy/mediatek,dsi-phy.yaml Example: diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt b/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt index 6b1c586403e4..b284ca51b913 100644 --- a/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt +++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt @@ -53,23 +53,7 @@ Required properties: HDMI PHY ======== - -The HDMI PHY serializes the HDMI encoder's three channel 10-bit parallel -output and drives the HDMI pads. - -Required properties: -- compatible: "mediatek,-hdmi-phy" -- the supported chips are mt2701, mt7623 and mt8173 -- reg: Physical base address and length of the module's registers -- clocks: PLL reference clock -- clock-names: must contain "pll_ref" -- clock-output-names: must be "hdmitx_dig_cts" on mt8173 -- #phy-cells: must be <0> -- #clock-cells: must be <0> - -Optional properties: -- mediatek,ibias: TX DRV bias current for <1.65Gbps, defaults to 0xa -- mediatek,ibias_up: TX DRV bias current for >1.65Gbps, defaults to 0x1c +See phy/mediatek,hdmi-phy.yaml Example: diff --git a/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml b/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml index 6b7fddc80c41..67682fe77f10 100644 --- a/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml +++ b/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml @@ -37,34 +37,33 @@ properties: panel-timing: true ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: The sink for odd pixels. properties: - reg: - const: 0 - dual-lvds-odd-pixels: true required: - - reg - dual-lvds-odd-pixels port@1: - type: object + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: The sink for even pixels. properties: - reg: - const: 1 - dual-lvds-even-pixels: true required: - - reg - dual-lvds-even-pixels + required: + - port@0 + - port@1 + additionalProperties: false required: diff --git a/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml b/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml index b8b9435e464c..4f92365e888a 100644 --- a/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml +++ b/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml @@ -30,6 +30,7 @@ properties: power supply for LCM (1.8V) dcdc-en-gpios: + maxItems: 1 description: | phandle of the gpio for power ic line Power IC supply enable, High active diff --git a/Documentation/devicetree/bindings/display/panel/mantix,mlaf057we51-x.yaml b/Documentation/devicetree/bindings/display/panel/mantix,mlaf057we51-x.yaml index 51f423297ec8..a4b8569ab81c 100644 --- a/Documentation/devicetree/bindings/display/panel/mantix,mlaf057we51-x.yaml +++ b/Documentation/devicetree/bindings/display/panel/mantix,mlaf057we51-x.yaml @@ -20,6 +20,7 @@ properties: compatible: enum: - mantix,mlaf057we51-x + - ys,ys57pss36bh5gq port: true reg: @@ -37,7 +38,8 @@ properties: reset-gpios: true - 'mantix,tp-rstn-gpios': + mantix,tp-rstn-gpios: + maxItems: 1 description: second reset line that triggers DSI config load backlight: true diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml index 2f5df1d235ae..ef4c0a24512d 100644 --- a/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml +++ b/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml @@ -30,6 +30,7 @@ properties: panel. The novatek,nt36672a compatible shall always be provided as a fallback. reset-gpios: + maxItems: 1 description: phandle of gpio for reset line - This should be 8mA, gpio can be configured using mux, pinctrl, pinctrl-names (active high) diff --git a/Documentation/devicetree/bindings/display/panel/panel-common.yaml b/Documentation/devicetree/bindings/display/panel/panel-common.yaml index cd6dc5461721..5b38dc89cb21 100644 --- a/Documentation/devicetree/bindings/display/panel/panel-common.yaml +++ b/Documentation/devicetree/bindings/display/panel/panel-common.yaml @@ -68,16 +68,7 @@ properties: # Connectivity port: - type: object - - ports: - type: object - description: - Panels receive video data through one or multiple connections. While - the nature of those connections is specific to the panel type, the - connectivity is expressed in a standard fashion using ports as specified - in the device graph bindings defined in - Documentation/devicetree/bindings/graph.txt. + $ref: /schemas/graph.yaml#/properties/port ddc-i2c-bus: $ref: /schemas/types.yaml#/definitions/phandle diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml index 72e4b6d4d5e1..fbd71669248f 100644 --- a/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml +++ b/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml @@ -35,6 +35,8 @@ properties: - boe,tv080wum-nl0 # Innolux P079ZCA 7.85" 768x1024 TFT LCD panel - innolux,p079zca + # Khadas TS050 5" 1080x1920 LCD panel + - khadas,ts050 # Kingdisplay KD097D04 9.7" 1536x2048 TFT LCD panel - kingdisplay,kd097d04 # LG ACX467AKM-7 4.95" 1080×1920 LCD Panel diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml index 3207608d1178..62b0d54d87b7 100644 --- a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml +++ b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml @@ -76,6 +76,8 @@ properties: # BOE OPTOELECTRONICS TECHNOLOGY 10.1" WXGA TFT LCD panel - boe,nv101wxmn51 # BOE NV133FHM-N61 13.3" FHD (1920x1080) TFT LCD Panel + - boe,nv110wtm-n61 + # BOE NV110WTM-N61 11.0" 2160x1440 TFT LCD Panel - boe,nv133fhm-n61 # BOE NV133FHM-N62 13.3" FHD (1920x1080) TFT LCD Panel - boe,nv133fhm-n62 diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml index 1dab80ae1d0a..ea58df49263a 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml @@ -11,6 +11,7 @@ maintainers: allOf: - $ref: panel-common.yaml# + - $ref: /schemas/leds/backlight/common.yaml# properties: compatible: @@ -19,6 +20,8 @@ properties: reg: true reset-gpios: true port: true + default-brightness: true + max-brightness: true vdd3-supply: description: VDD regulator @@ -31,7 +34,6 @@ required: - reset-gpios - vdd3-supply - vci-supply - - port unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.yaml index 4110d003ce1f..008c144257cb 100644 --- a/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.yaml @@ -43,34 +43,24 @@ properties: This soc uses GRF regs to switch the HDMI TX input between vop0 and vop1. ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Port node with two endpoints, numbered 0 and 1, connected respectively to vop0 and vop1. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Port node with one endpoint connected to a hdmi-connector node. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml index ed8148e26e24..6f43d885c9b3 100644 --- a/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml +++ b/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml @@ -70,10 +70,7 @@ properties: - const: dclk port: - type: object - description: - A port node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/port assigned-clocks: maxItems: 2 diff --git a/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml b/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml index 327a14d85df8..679daed4124e 100644 --- a/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml +++ b/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml @@ -51,20 +51,16 @@ properties: Phandle of the regulator that provides the supply voltage. ports: - type: object - description: - A node containing DSI input & output port nodes with endpoint - definitions as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - Documentation/devicetree/bindings/graph.txt + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DSI input port node, connected to the ltdc rgb output port. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DSI output port node, connected to a panel or a bridge input port" diff --git a/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml b/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml index bf8ad916e9b0..d54f9ca207af 100644 --- a/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml +++ b/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml @@ -35,15 +35,13 @@ properties: maxItems: 1 port: - type: object - description: - "Video port for DPI RGB output. + $ref: /schemas/graph.yaml#/properties/port + description: | + Video port for DPI RGB output. ltdc has one video port with up to 2 endpoints: - for external dpi rgb panel or bridge, using gpios. - for internal dpi input of the MIPI DSI host controller. Note: These 2 endpoints cannot be activated simultaneously. - Please refer to the bindings defined in - Documentation/devicetree/bindings/media/video-interfaces.txt." required: - compatible diff --git a/Documentation/devicetree/bindings/display/ste,mcde.txt b/Documentation/devicetree/bindings/display/ste,mcde.txt deleted file mode 100644 index 4c33c692bd5f..000000000000 --- a/Documentation/devicetree/bindings/display/ste,mcde.txt +++ /dev/null @@ -1,104 +0,0 @@ -ST-Ericsson Multi Channel Display Engine MCDE - -The ST-Ericsson MCDE is a display controller with support for compositing -and displaying several channels memory resident graphics data on DSI or -LCD displays or bridges. It is used in the ST-Ericsson U8500 platform. - -Required properties: - -- compatible: must be: - "ste,mcde" -- reg: register base for the main MCDE control registers, should be - 0x1000 in size -- interrupts: the interrupt line for the MCDE -- epod-supply: a phandle to the EPOD regulator -- vana-supply: a phandle to the analog voltage regulator -- clocks: an array of the MCDE clocks in this strict order: - MCDECLK (main MCDE clock), LCDCLK (LCD clock), PLLDSI - (HDMI clock), DSI0ESCLK (DSI0 energy save clock), - DSI1ESCLK (DSI1 energy save clock), DSI2ESCLK (DSI2 energy - save clock) -- clock-names: must be the following array: - "mcde", "lcd", "hdmi" - to match the required clock inputs above. -- #address-cells: should be <1> (for the DSI hosts that will be children) -- #size-cells: should be <1> (for the DSI hosts that will be children) -- ranges: this should always be stated - -Required subnodes: - -The devicetree must specify subnodes for the DSI host adapters. -These must have the following characteristics: - -- compatible: must be: - "ste,mcde-dsi" -- reg: must specify the register range for the DSI host -- vana-supply: phandle to the VANA voltage regulator -- clocks: phandles to the high speed and low power (energy save) clocks - the high speed clock is not present on the third (dsi2) block, so it - should only have the "lp" clock -- clock-names: "hs" for the high speed clock and "lp" for the low power - (energy save) clock -- #address-cells: should be <1> -- #size-cells: should be <0> - -Display panels and bridges will appear as children on the DSI hosts, and -the displays are connected to the DSI hosts using the common binding -for video transmitter interfaces; see -Documentation/devicetree/bindings/media/video-interfaces.txt - -If a DSI host is unused (not connected) it will have no children defined. - -Example: - -mcde@a0350000 { - compatible = "ste,mcde"; - reg = <0xa0350000 0x1000>; - interrupts = ; - epod-supply = <&db8500_b2r2_mcde_reg>; - vana-supply = <&ab8500_ldo_ana_reg>; - clocks = <&prcmu_clk PRCMU_MCDECLK>, /* Main MCDE clock */ - <&prcmu_clk PRCMU_LCDCLK>, /* LCD clock */ - <&prcmu_clk PRCMU_PLLDSI>; /* HDMI clock */ - clock-names = "mcde", "lcd", "hdmi"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - dsi0: dsi@a0351000 { - compatible = "ste,mcde-dsi"; - reg = <0xa0351000 0x1000>; - vana-supply = <&ab8500_ldo_ana_reg>; - clocks = <&prcmu_clk PRCMU_DSI0CLK>, <&prcmu_clk PRCMU_DSI0ESCCLK>; - clock-names = "hs", "lp"; - #address-cells = <1>; - #size-cells = <0>; - - panel { - compatible = "samsung,s6d16d0"; - reg = <0>; - vdd1-supply = <&ab8500_ldo_aux1_reg>; - reset-gpios = <&gpio2 1 GPIO_ACTIVE_LOW>; - }; - - }; - dsi1: dsi@a0352000 { - compatible = "ste,mcde-dsi"; - reg = <0xa0352000 0x1000>; - vana-supply = <&ab8500_ldo_ana_reg>; - clocks = <&prcmu_clk PRCMU_DSI1CLK>, <&prcmu_clk PRCMU_DSI1ESCCLK>; - clock-names = "hs", "lp"; - #address-cells = <1>; - #size-cells = <0>; - }; - dsi2: dsi@a0353000 { - compatible = "ste,mcde-dsi"; - reg = <0xa0353000 0x1000>; - vana-supply = <&ab8500_ldo_ana_reg>; - /* This DSI port only has the Low Power / Energy Save clock */ - clocks = <&prcmu_clk PRCMU_DSI2ESCCLK>; - clock-names = "lp"; - #address-cells = <1>; - #size-cells = <0>; - }; -}; diff --git a/Documentation/devicetree/bindings/display/ste,mcde.yaml b/Documentation/devicetree/bindings/display/ste,mcde.yaml new file mode 100644 index 000000000000..de0c678b3c29 --- /dev/null +++ b/Documentation/devicetree/bindings/display/ste,mcde.yaml @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/ste,mcde.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ST-Ericsson Multi Channel Display Engine MCDE + +maintainers: + - Linus Walleij + +properties: + compatible: + const: ste,mcde + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + description: an array of the MCDE clocks + items: + - description: MCDECLK (main MCDE clock) + - description: LCDCLK (LCD clock) + - description: PLLDSI (HDMI clock) + + clock-names: + items: + - const: mcde + - const: lcd + - const: hdmi + + resets: + maxItems: 1 + + epod-supply: + description: a phandle to the EPOD regulator + + vana-supply: + description: a phandle to the analog voltage regulator + + port: + $ref: /schemas/graph.yaml#/properties/port + description: + A DPI port node + + "#address-cells": + const: 1 + + "#size-cells": + const: 1 + + ranges: true + +patternProperties: + "^dsi@[0-9a-f]+$": + description: subnodes for the three DSI host adapters + type: object + allOf: + - $ref: dsi-controller.yaml# + properties: + compatible: + const: ste,mcde-dsi + + reg: + maxItems: 1 + + vana-supply: + description: a phandle to the analog voltage regulator + + clocks: + description: phandles to the high speed and low power (energy save) clocks + the high speed clock is not present on the third (dsi2) block, so it + should only have the "lp" clock + minItems: 1 + maxItems: 2 + + clock-names: + oneOf: + - items: + - const: hs + - const: lp + - items: + - const: lp + + required: + - compatible + - reg + - vana-supply + - clocks + - clock-names + + unevaluatedProperties: false + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + - epod-supply + - vana-supply + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + + mcde@a0350000 { + compatible = "ste,mcde"; + reg = <0xa0350000 0x1000>; + interrupts = ; + epod-supply = <&db8500_b2r2_mcde_reg>; + vana-supply = <&ab8500_ldo_ana_reg>; + clocks = <&prcmu_clk PRCMU_MCDECLK>, + <&prcmu_clk PRCMU_LCDCLK>, + <&prcmu_clk PRCMU_PLLDSI>; + clock-names = "mcde", "lcd", "hdmi"; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + dsi0: dsi@a0351000 { + compatible = "ste,mcde-dsi"; + reg = <0xa0351000 0x1000>; + vana-supply = <&ab8500_ldo_ana_reg>; + clocks = <&prcmu_clk PRCMU_DSI0CLK>, <&prcmu_clk PRCMU_DSI0ESCCLK>; + clock-names = "hs", "lp"; + #address-cells = <1>; + #size-cells = <0>; + + panel@0 { + compatible = "samsung,s6d16d0"; + reg = <0>; + vdd1-supply = <&ab8500_ldo_aux1_reg>; + reset-gpios = <&gpio2 1 GPIO_ACTIVE_LOW>; + }; + }; + + dsi1: dsi@a0352000 { + compatible = "ste,mcde-dsi"; + reg = <0xa0352000 0x1000>; + vana-supply = <&ab8500_ldo_ana_reg>; + clocks = <&prcmu_clk PRCMU_DSI1CLK>, <&prcmu_clk PRCMU_DSI1ESCCLK>; + clock-names = "hs", "lp"; + #address-cells = <1>; + #size-cells = <0>; + }; + + dsi2: dsi@a0353000 { + compatible = "ste,mcde-dsi"; + reg = <0xa0353000 0x1000>; + vana-supply = <&ab8500_ldo_ana_reg>; + /* This DSI port only has the Low Power / Energy Save clock */ + clocks = <&prcmu_clk PRCMU_DSI2ESCCLK>; + clock-names = "lp"; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml index 4dc30738ee57..781c1868b0b8 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml @@ -74,30 +74,19 @@ properties: type: boolean ports: - type: object - description: - Ports as described in Documentation/devicetree/bindings/graph.txt + $ref: /schemas/graph.yaml#/properties/ports + properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The DSS OLDI output port node form video port 1 port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The DSS DPI output port node from video port 2 - required: - - "#address-cells" - - "#size-cells" - ti,am65x-oldi-io-ctrl: $ref: "/schemas/types.yaml#/definitions/phandle-array" maxItems: 1 diff --git a/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml index c9a947d55fa4..2986f9acc9f0 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml @@ -107,40 +107,29 @@ properties: type: boolean ports: - type: object - description: - Ports as described in Documentation/devicetree/bindings/graph.txt + $ref: /schemas/graph.yaml#/properties/ports + properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The output port node form video port 1 port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The output port node from video port 2 port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The output port node from video port 3 port@3: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The output port node from video port 4 - required: - - "#address-cells" - - "#size-cells" - max-memory-bandwidth: $ref: /schemas/types.yaml#/definitions/uint32 description: diff --git a/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml index 8f87b82c6695..7ce7bbad5780 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml @@ -54,9 +54,8 @@ properties: description: phandle to the associated power domain port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: - Port as described in Documentation/devicetree/bindings/graph.txt. The DSS DPI output port node max-memory-bandwidth: diff --git a/Documentation/devicetree/bindings/dma/ingenic,dma.yaml b/Documentation/devicetree/bindings/dma/ingenic,dma.yaml index 6a2043721b95..ac4d59494fc8 100644 --- a/Documentation/devicetree/bindings/dma/ingenic,dma.yaml +++ b/Documentation/devicetree/bindings/dma/ingenic,dma.yaml @@ -17,6 +17,8 @@ properties: enum: - ingenic,jz4740-dma - ingenic,jz4725b-dma + - ingenic,jz4760-dma + - ingenic,jz4760b-dma - ingenic,jz4770-dma - ingenic,jz4780-dma - ingenic,x1000-dma diff --git a/Documentation/devicetree/bindings/dma/intel,ldma.yaml b/Documentation/devicetree/bindings/dma/intel,ldma.yaml new file mode 100644 index 000000000000..a5c4be783593 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/intel,ldma.yaml @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/intel,ldma.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Lightning Mountain centralized DMA controllers. + +maintainers: + - chuanhua.lei@intel.com + - mallikarjunax.reddy@intel.com + +allOf: + - $ref: "dma-controller.yaml#" + +properties: + compatible: + enum: + - intel,lgm-cdma + - intel,lgm-dma2tx + - intel,lgm-dma1rx + - intel,lgm-dma1tx + - intel,lgm-dma0tx + - intel,lgm-dma3 + - intel,lgm-toe-dma30 + - intel,lgm-toe-dma31 + + reg: + maxItems: 1 + + "#dma-cells": + const: 3 + description: + The first cell is the peripheral's DMA request line. + The second cell is the peripheral's (port) number corresponding to the channel. + The third cell is the burst length of the channel. + + dma-channels: + minimum: 1 + maximum: 16 + + dma-channel-mask: + maxItems: 1 + + clocks: + maxItems: 1 + + resets: + maxItems: 1 + + reset-names: + items: + - const: ctrl + + interrupts: + maxItems: 1 + + intel,dma-poll-cnt: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + DMA descriptor polling counter is used to control the poling mechanism + for the descriptor fetching for all channels. + + intel,dma-byte-en: + type: boolean + description: + DMA byte enable is only valid for DMA write(RX). + Byte enable(1) means DMA write will be based on the number of dwords + instead of the whole burst. + + intel,dma-drb: + type: boolean + description: + DMA descriptor read back to make sure data and desc synchronization. + + intel,dma-dburst-wr: + type: boolean + description: + Enable RX dynamic burst write. When it is enabled, the DMA does RX dynamic burst; + if it is disabled, the DMA RX will still support programmable fixed burst size of 2,4,8,16. + It only applies to RX DMA and memcopy DMA. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + dma0: dma-controller@e0e00000 { + compatible = "intel,lgm-cdma"; + reg = <0xe0e00000 0x1000>; + #dma-cells = <3>; + dma-channels = <16>; + dma-channel-mask = <0xFFFF>; + interrupt-parent = <&ioapic1>; + interrupts = <82 1>; + resets = <&rcu0 0x30 0>; + reset-names = "ctrl"; + clocks = <&cgu0 80>; + intel,dma-poll-cnt = <4>; + intel,dma-byte-en; + intel,dma-drb; + }; + - | + dma3: dma-controller@ec800000 { + compatible = "intel,lgm-dma3"; + reg = <0xec800000 0x1000>; + clocks = <&cgu0 71>; + resets = <&rcu0 0x10 9>; + #dma-cells = <3>; + intel,dma-poll-cnt = <16>; + intel,dma-byte-en; + intel,dma-dburst-wr; + }; diff --git a/Documentation/devicetree/bindings/dma/owl-dma.yaml b/Documentation/devicetree/bindings/dma/owl-dma.yaml index 256d62af2c64..93b4847554fb 100644 --- a/Documentation/devicetree/bindings/dma/owl-dma.yaml +++ b/Documentation/devicetree/bindings/dma/owl-dma.yaml @@ -8,8 +8,8 @@ title: Actions Semi Owl SoCs DMA controller description: | The OWL DMA is a general-purpose direct memory access controller capable of - supporting 10 and 12 independent DMA channels for S700 and S900 SoCs - respectively. + supporting 10 independent DMA channels for the Actions Semi S700 SoC and 12 + independent DMA channels for the S500 and S900 SoC variants. maintainers: - Manivannan Sadhasivam @@ -20,8 +20,9 @@ allOf: properties: compatible: enum: - - actions,s900-dma + - actions,s500-dma - actions,s700-dma + - actions,s900-dma reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml index c07eb6f2fc8d..7f2a54bc732d 100644 --- a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml +++ b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml @@ -14,34 +14,37 @@ allOf: properties: compatible: - items: - - enum: - - renesas,dmac-r8a7742 # RZ/G1H - - renesas,dmac-r8a7743 # RZ/G1M - - renesas,dmac-r8a7744 # RZ/G1N - - renesas,dmac-r8a7745 # RZ/G1E - - renesas,dmac-r8a77470 # RZ/G1C - - renesas,dmac-r8a774a1 # RZ/G2M - - renesas,dmac-r8a774b1 # RZ/G2N - - renesas,dmac-r8a774c0 # RZ/G2E - - renesas,dmac-r8a774e1 # RZ/G2H - - renesas,dmac-r8a7790 # R-Car H2 - - renesas,dmac-r8a7791 # R-Car M2-W - - renesas,dmac-r8a7792 # R-Car V2H - - renesas,dmac-r8a7793 # R-Car M2-N - - renesas,dmac-r8a7794 # R-Car E2 - - renesas,dmac-r8a7795 # R-Car H3 - - renesas,dmac-r8a7796 # R-Car M3-W - - renesas,dmac-r8a77961 # R-Car M3-W+ - - renesas,dmac-r8a77965 # R-Car M3-N - - renesas,dmac-r8a77970 # R-Car V3M - - renesas,dmac-r8a77980 # R-Car V3H - - renesas,dmac-r8a77990 # R-Car E3 - - renesas,dmac-r8a77995 # R-Car D3 - - const: renesas,rcar-dmac + oneOf: + - items: + - enum: + - renesas,dmac-r8a7742 # RZ/G1H + - renesas,dmac-r8a7743 # RZ/G1M + - renesas,dmac-r8a7744 # RZ/G1N + - renesas,dmac-r8a7745 # RZ/G1E + - renesas,dmac-r8a77470 # RZ/G1C + - renesas,dmac-r8a774a1 # RZ/G2M + - renesas,dmac-r8a774b1 # RZ/G2N + - renesas,dmac-r8a774c0 # RZ/G2E + - renesas,dmac-r8a774e1 # RZ/G2H + - renesas,dmac-r8a7790 # R-Car H2 + - renesas,dmac-r8a7791 # R-Car M2-W + - renesas,dmac-r8a7792 # R-Car V2H + - renesas,dmac-r8a7793 # R-Car M2-N + - renesas,dmac-r8a7794 # R-Car E2 + - renesas,dmac-r8a7795 # R-Car H3 + - renesas,dmac-r8a7796 # R-Car M3-W + - renesas,dmac-r8a77961 # R-Car M3-W+ + - renesas,dmac-r8a77965 # R-Car M3-N + - renesas,dmac-r8a77970 # R-Car V3M + - renesas,dmac-r8a77980 # R-Car V3H + - renesas,dmac-r8a77990 # R-Car E3 + - renesas,dmac-r8a77995 # R-Car D3 + - const: renesas,rcar-dmac - reg: - maxItems: 1 + - items: + - const: renesas,dmac-r8a779a0 # R-Car V3U + + reg: true interrupts: minItems: 9 @@ -110,6 +113,23 @@ required: - power-domains - resets +if: + properties: + compatible: + contains: + enum: + - renesas,dmac-r8a779a0 +then: + properties: + reg: + items: + - description: Base register block + - description: Channel register block +else: + properties: + reg: + maxItems: 1 + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/dma/sirfsoc-dma.txt b/Documentation/devicetree/bindings/dma/sirfsoc-dma.txt deleted file mode 100644 index ccd52d6a231a..000000000000 --- a/Documentation/devicetree/bindings/dma/sirfsoc-dma.txt +++ /dev/null @@ -1,44 +0,0 @@ -* CSR SiRFSoC DMA controller - -See dma.txt first - -Required properties: -- compatible: Should be "sirf,prima2-dmac", "sirf,atlas7-dmac" or - "sirf,atlas7-dmac-v2" -- reg: Should contain DMA registers location and length. -- interrupts: Should contain one interrupt shared by all channel -- #dma-cells: must be <1>. used to represent the number of integer - cells in the dmas property of client device. -- clocks: clock required - -Example: - -Controller: -dmac0: dma-controller@b00b0000 { - compatible = "sirf,prima2-dmac"; - reg = <0xb00b0000 0x10000>; - interrupts = <12>; - clocks = <&clks 24>; - #dma-cells = <1>; -}; - - -Client: -Fill the specific dma request line in dmas. In the below example, spi0 read -channel request line is 9 of the 2nd dma controller, while write channel uses -4 of the 2nd dma controller; spi1 read channel request line is 12 of the 1st -dma controller, while write channel uses 13 of the 1st dma controller: - -spi0: spi@b00d0000 { - compatible = "sirf,prima2-spi"; - dmas = <&dmac1 9>, - <&dmac1 4>; - dma-names = "rx", "tx"; -}; - -spi1: spi@b0170000 { - compatible = "sirf,prima2-spi"; - dmas = <&dmac0 12>, - <&dmac0 13>; - dma-names = "rx", "tx"; -}; diff --git a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.txt b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.txt deleted file mode 100644 index dbe160400adc..000000000000 --- a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.txt +++ /dev/null @@ -1,39 +0,0 @@ -Synopsys DesignWare AXI DMA Controller - -Required properties: -- compatible: "snps,axi-dma-1.01a" -- reg: Address range of the DMAC registers. This should include - all of the per-channel registers. -- interrupt: Should contain the DMAC interrupt number. -- dma-channels: Number of channels supported by hardware. -- snps,dma-masters: Number of AXI masters supported by the hardware. -- snps,data-width: Maximum AXI data width supported by hardware. - (0 - 8bits, 1 - 16bits, 2 - 32bits, ..., 6 - 512bits) -- snps,priority: Priority of channel. Array size is equal to the number of - dma-channels. Priority value must be programmed within [0:dma-channels-1] - range. (0 - minimum priority) -- snps,block-size: Maximum block size supported by the controller channel. - Array size is equal to the number of dma-channels. - -Optional properties: -- snps,axi-max-burst-len: Restrict master AXI burst length by value specified - in this property. If this property is missing the maximum AXI burst length - supported by DMAC is used. [1:256] - -Example: - -dmac: dma-controller@80000 { - compatible = "snps,axi-dma-1.01a"; - reg = <0x80000 0x400>; - clocks = <&core_clk>, <&cfgr_clk>; - clock-names = "core-clk", "cfgr-clk"; - interrupt-parent = <&intc>; - interrupts = <27>; - - dma-channels = <4>; - snps,dma-masters = <2>; - snps,data-width = <3>; - snps,block-size = <4096 4096 4096 4096>; - snps,priority = <0 1 2 3>; - snps,axi-max-burst-len = <16>; -}; diff --git a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml new file mode 100644 index 000000000000..79e241498e25 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/snps,dw-axi-dmac.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Synopsys DesignWare AXI DMA Controller + +maintainers: + - Eugeniy Paltsev + - Jee Heng Sia + +description: + Synopsys DesignWare AXI DMA Controller DT Binding + +allOf: + - $ref: "dma-controller.yaml#" + +properties: + compatible: + enum: + - snps,axi-dma-1.01a + - intel,kmb-axi-dma + + reg: + minItems: 1 + items: + - description: Address range of the DMAC registers + - description: Address range of the DMAC APB registers + + reg-names: + items: + - const: axidma_ctrl_regs + - const: axidma_apb_regs + + interrupts: + maxItems: 1 + + clocks: + items: + - description: Bus Clock + - description: Module Clock + + clock-names: + items: + - const: core-clk + - const: cfgr-clk + + '#dma-cells': + const: 1 + + dma-channels: + minimum: 1 + maximum: 8 + + snps,dma-masters: + description: | + Number of AXI masters supported by the hardware. + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [1, 2] + + snps,data-width: + description: | + AXI data width supported by hardware. + (0 - 8bits, 1 - 16bits, 2 - 32bits, ..., 6 - 512bits) + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3, 4, 5, 6] + + snps,priority: + description: | + Channel priority specifier associated with the DMA channels. + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 8 + + snps,block-size: + description: | + Channel block size specifier associated with the DMA channels. + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 8 + + snps,axi-max-burst-len: + description: | + Restrict master AXI burst length by value specified in this property. + If this property is missing the maximum AXI burst length supported by + DMAC is used. + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 256 + +required: + - compatible + - reg + - clocks + - clock-names + - interrupts + - '#dma-cells' + - dma-channels + - snps,dma-masters + - snps,data-width + - snps,priority + - snps,block-size + +additionalProperties: false + +examples: + - | + #include + #include + /* example with snps,dw-axi-dmac */ + dmac: dma-controller@80000 { + compatible = "snps,axi-dma-1.01a"; + reg = <0x80000 0x400>; + clocks = <&core_clk>, <&cfgr_clk>; + clock-names = "core-clk", "cfgr-clk"; + interrupt-parent = <&intc>; + interrupts = <27>; + #dma-cells = <1>; + dma-channels = <4>; + snps,dma-masters = <2>; + snps,data-width = <3>; + snps,block-size = <4096 4096 4096 4096>; + snps,priority = <0 1 2 3>; + snps,axi-max-burst-len = <16>; + }; diff --git a/Documentation/devicetree/bindings/dma/ste-coh901318.txt b/Documentation/devicetree/bindings/dma/ste-coh901318.txt deleted file mode 100644 index 091ad057e9cf..000000000000 --- a/Documentation/devicetree/bindings/dma/ste-coh901318.txt +++ /dev/null @@ -1,32 +0,0 @@ -ST-Ericsson COH 901 318 DMA Controller - -This is a DMA controller which has begun as a fork of the -ARM PL08x PrimeCell VHDL code. - -Required properties: -- compatible: should be "stericsson,coh901318" -- reg: register locations and length -- interrupts: the single DMA IRQ -- #dma-cells: must be set to <1>, as the channels on the - COH 901 318 are simple and identified by a single number -- dma-channels: the number of DMA channels handled - -Example: - -dmac: dma-controller@c00020000 { - compatible = "stericsson,coh901318"; - reg = <0xc0020000 0x1000>; - interrupt-parent = <&vica>; - interrupts = <2>; - #dma-cells = <1>; - dma-channels = <40>; -}; - -Consumers example: - -uart0: serial@c0013000 { - compatible = "..."; - (...) - dmas = <&dmac 17 &dmac 18>; - dma-names = "tx", "rx"; -}; diff --git a/Documentation/devicetree/bindings/dma/zxdma.txt b/Documentation/devicetree/bindings/dma/zxdma.txt deleted file mode 100644 index 0ab80f69e566..000000000000 --- a/Documentation/devicetree/bindings/dma/zxdma.txt +++ /dev/null @@ -1,38 +0,0 @@ -* ZTE ZX296702 DMA controller - -Required properties: -- compatible: Should be "zte,zx296702-dma" -- reg: Should contain DMA registers location and length. -- interrupts: Should contain one interrupt shared by all channel -- #dma-cells: see dma.txt, should be 1, para number -- dma-channels: physical channels supported -- dma-requests: virtual channels supported, each virtual channel - have specific request line -- clocks: clock required - -Example: - -Controller: - dma: dma-controller@09c00000{ - compatible = "zte,zx296702-dma"; - reg = <0x09c00000 0x1000>; - clocks = <&topclk ZX296702_DMA_ACLK>; - interrupts = ; - #dma-cells = <1>; - dma-channels = <24>; - dma-requests = <24>; - }; - -Client: -Use specific request line passing from dmax -For example, spdif0 tx channel request line is 4 - spdif0: spdif0@b004000 { - #sound-dai-cells = <0>; - compatible = "zte,zx296702-spdif"; - reg = <0x0b004000 0x1000>; - clocks = <&lsp0clk ZX296702_SPDIF0_DIV>; - clock-names = "tx"; - interrupts = ; - dmas = <&dma 4>; - dma-names = "tx"; - } diff --git a/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml b/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml index 4cc011230153..7afc9f2be13a 100644 --- a/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml +++ b/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml @@ -21,7 +21,7 @@ properties: - fsl,imx8mp-dsp reg: - description: Should contain register location and length + maxItems: 1 clocks: items: diff --git a/Documentation/devicetree/bindings/eeprom/at24.yaml b/Documentation/devicetree/bindings/eeprom/at24.yaml index d5117c638b75..021d8ae42da3 100644 --- a/Documentation/devicetree/bindings/eeprom/at24.yaml +++ b/Documentation/devicetree/bindings/eeprom/at24.yaml @@ -95,9 +95,6 @@ properties: pattern: spd$ # These are special cases that don't conform to the above pattern. # Each requires a standard at24 model as fallback. - - items: - - const: rohm,br24t01 - - const: atmel,24c01 - items: - const: nxp,se97b - const: atmel,24c02 @@ -113,6 +110,12 @@ properties: - items: - const: renesas,r1ex24128 - const: atmel,24c128 + - items: + - const: rohm,br24g01 + - const: atmel,24c01 + - items: + - const: rohm,br24t01 + - const: atmel,24c01 label: description: Descriptive name of the EEPROM. diff --git a/Documentation/devicetree/bindings/eeprom/at25.yaml b/Documentation/devicetree/bindings/eeprom/at25.yaml index 121a601db22e..6a2dc8b3ed14 100644 --- a/Documentation/devicetree/bindings/eeprom/at25.yaml +++ b/Documentation/devicetree/bindings/eeprom/at25.yaml @@ -39,8 +39,7 @@ properties: - const: atmel,at25 reg: - description: - Chip select number. + maxItems: 1 spi-max-frequency: true diff --git a/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml b/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml index 4b0f414486d2..d5cfa32ea52d 100644 --- a/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml +++ b/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml @@ -19,6 +19,7 @@ properties: const: nxp,ptn5150 int-gpios: + maxItems: 1 deprecated: true description: GPIO pin (input) connected to the PTN5150's INTB pin. @@ -31,6 +32,7 @@ properties: maxItems: 1 vbus-gpios: + maxItems: 1 description: GPIO pin (output) used to control VBUS. If skipped, no such control takes place. diff --git a/Documentation/devicetree/bindings/gpio/gpio-atlas7.txt b/Documentation/devicetree/bindings/gpio/gpio-atlas7.txt deleted file mode 100644 index d7e123fc90b5..000000000000 --- a/Documentation/devicetree/bindings/gpio/gpio-atlas7.txt +++ /dev/null @@ -1,50 +0,0 @@ -CSR SiRFatlas7 GPIO controller bindings - -Required properties: -- compatible : "sirf,atlas7-gpio" -- reg : Address range of the pinctrl registers -- interrupts : Interrupts used by every GPIO group -- gpio-banks : How many gpio banks on this controller -- gpio-controller : Indicates this device is a GPIO controller -- interrupt-controller : Marks the device node as an interrupt controller - -The GPIO controller also acts as an interrupt controller. It uses the default -two cells specifier as described in Documentation/devicetree/bindings/ -interrupt-controller/interrupts.txt. - -Example: - - gpio_0: gpio_mediam@17040000 { - compatible = "sirf,atlas7-gpio"; - reg = <0x17040000 0x1000>; - interrupts = <0 13 0>, <0 14 0>; - - #gpio-cells = <2>; - #interrupt-cells = <2>; - - gpio-controller; - interrupt-controller; - - gpio-banks = <2>; - gpio-ranges = <&pinctrl 0 0 0>, - <&pinctrl 32 0 0>; - gpio-ranges-group-names = "lvds_gpio_grp", - "uart_nand_gpio_grp"; - }; - - leds { - compatible = "gpio-leds"; - - led1 { - gpios = <&gpio_1 15 0>; - ... - }; - - led2 { - gpios = <&gpio_2 34 0>; - ... - }; - }; - -Please refer to gpio.txt in this directory for details of the common -gpio properties used by devices. diff --git a/Documentation/devicetree/bindings/gpio/gpio-davinci.txt b/Documentation/devicetree/bindings/gpio/gpio-davinci.txt index cd91d61eac31..696ea46227d1 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-davinci.txt +++ b/Documentation/devicetree/bindings/gpio/gpio-davinci.txt @@ -7,6 +7,7 @@ Required Properties: "ti,k2g-gpio", "ti,keystone-gpio": for 66AK2G "ti,am654-gpio", "ti,keystone-gpio": for TI K3 AM654 "ti,j721e-gpio", "ti,keystone-gpio": for J721E SoCs + "ti,am64-gpio", "ti,keystone-gpio": for AM64 SoCs - reg: Physical base address of the controller and the size of memory mapped registers. diff --git a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml index f5ee23c2df60..b6a6e742b66d 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml +++ b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml @@ -32,6 +32,7 @@ properties: - maxim,max7327 - nxp,pca6416 - nxp,pca9505 + - nxp,pca9506 - nxp,pca9534 - nxp,pca9535 - nxp,pca9536 @@ -70,7 +71,7 @@ properties: gpio-line-names: minItems: 1 - maxItems: 32 + maxItems: 40 interrupts: maxItems: 1 @@ -81,6 +82,7 @@ properties: const: 2 reset-gpios: + maxItems: 1 description: GPIO specification for the RESET input. This is an active low signal to the PCA953x. Not valid for Maxim MAX732x devices. diff --git a/Documentation/devicetree/bindings/gpio/gpio-stericsson-coh901.txt b/Documentation/devicetree/bindings/gpio/gpio-stericsson-coh901.txt deleted file mode 100644 index fd665b44d767..000000000000 --- a/Documentation/devicetree/bindings/gpio/gpio-stericsson-coh901.txt +++ /dev/null @@ -1,7 +0,0 @@ -ST-Ericsson COH 901 571/3 GPIO controller - -Required properties: -- compatible: Compatible property value should be "stericsson,gpio-coh901" -- reg: Physical base address of the controller and length of memory mapped - region. -- interrupts: the 0...n interrupts assigned to the different GPIO ports/banks. diff --git a/Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml b/Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml index 1f2ef408bb43..fe1e1c63ffe3 100644 --- a/Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml @@ -46,7 +46,7 @@ examples: #include gpio: gpio@207800 { - compatible = "mstar,msc313e-gpio"; + compatible = "mstar,msc313-gpio"; #gpio-cells = <2>; reg = <0x207800 0x200>; gpio-controller; diff --git a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml index 5026662e4508..f2541739ee3b 100644 --- a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml @@ -48,6 +48,9 @@ properties: - renesas,gpio-r8a77995 # R-Car D3 - const: renesas,rcar-gen3-gpio # R-Car Gen3 or RZ/G2 + - items: + - const: renesas,gpio-r8a779a0 # R-Car V3U + reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/gpio/sifive,gpio.yaml b/Documentation/devicetree/bindings/gpio/sifive,gpio.yaml index a0efd8dc2538..c2902aac2514 100644 --- a/Documentation/devicetree/bindings/gpio/sifive,gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/sifive,gpio.yaml @@ -13,7 +13,10 @@ maintainers: properties: compatible: items: - - const: sifive,fu540-c000-gpio + - enum: + - sifive,fu540-c000-gpio + - sifive,fu740-c000-gpio + - canaan,k210-gpiohs - const: sifive,gpio0 reg: @@ -21,9 +24,9 @@ properties: interrupts: description: - interrupt mapping one per GPIO. Maximum 16 GPIOs. + Interrupt mapping, one per GPIO. Maximum 32 GPIOs. minItems: 1 - maxItems: 16 + maxItems: 32 interrupt-controller: true @@ -36,6 +39,14 @@ properties: "#gpio-cells": const: 2 + ngpios: + description: + The number of GPIOs available on the controller implementation. + It is 16 for the SiFive SoCs and 32 for the Canaan K210. + minimum: 1 + maximum: 32 + default: 16 + gpio-controller: true required: @@ -44,10 +55,20 @@ required: - interrupts - interrupt-controller - "#interrupt-cells" - - clocks - "#gpio-cells" - gpio-controller +if: + properties: + compatible: + contains: + enum: + - sifive,fu540-c000-gpio + - sifive,fu740-c000-gpio +then: + required: + - clocks + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/gpio/toshiba,gpio-visconti.yaml b/Documentation/devicetree/bindings/gpio/toshiba,gpio-visconti.yaml new file mode 100644 index 000000000000..9ad470e01953 --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/toshiba,gpio-visconti.yaml @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/gpio/toshiba,gpio-visconti.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Toshiba Visconti ARM SoCs GPIO controller + +maintainers: + - Nobuhiro Iwamatsu + +properties: + compatible: + items: + - const: toshiba,gpio-tmpv7708 + + reg: + maxItems: 1 + + "#gpio-cells": + const: 2 + + gpio-ranges: true + + gpio-controller: true + + interrupt-controller: true + + "#interrupt-cells": + const: 2 + + interrupts: + description: + interrupt mapping one per GPIO. + minItems: 16 + maxItems: 16 + +required: + - compatible + - reg + - "#gpio-cells" + - gpio-ranges + - gpio-controller + - interrupt-controller + - "#interrupt-cells" + - interrupt-parent + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + gpio: gpio@28020000 { + compatible = "toshiba,gpio-tmpv7708"; + reg = <0 0x28020000 0 0x1000>; + #gpio-cells = <0x2>; + gpio-ranges = <&pmux 0 0 32>; + gpio-controller; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&gic>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/gpio/zx296702-gpio.txt b/Documentation/devicetree/bindings/gpio/zx296702-gpio.txt deleted file mode 100644 index 0dab156fcf41..000000000000 --- a/Documentation/devicetree/bindings/gpio/zx296702-gpio.txt +++ /dev/null @@ -1,24 +0,0 @@ -ZTE ZX296702 GPIO controller - -Required properties: -- compatible : "zte,zx296702-gpio" -- #gpio-cells : Should be two. The first cell is the pin number and the - second cell is used to specify optional parameters: - - bit 0 specifies polarity (0 for normal, 1 for inverted) -- gpio-controller : Marks the device node as a GPIO controller. -- interrupts : Interrupt mapping for GPIO IRQ. -- gpio-ranges : Interaction with the PINCTRL subsystem. - -gpio1: gpio@b008040 { - compatible = "zte,zx296702-gpio"; - reg = <0xb008040 0x40>; - gpio-controller; - #gpio-cells = <2>; - gpio-ranges = < &pmx0 0 54 2 &pmx0 2 59 14>; - interrupts = ; - interrupt-parent = <&intc>; - interrupt-controller; - #interrupt-cells = <2>; - clock-names = "gpio_pclk"; - clocks = <&lsp0clk ZX296702_GPIO_CLK>; -}; diff --git a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt deleted file mode 100644 index b2df82b44625..000000000000 --- a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt +++ /dev/null @@ -1,33 +0,0 @@ -Broadcom V3D GPU - -Only the Broadcom V3D 3.x and newer GPUs are covered by this binding. -For V3D 2.x, see brcm,bcm-vc4.txt. - -Required properties: -- compatible: Should be "brcm,7268-v3d" or "brcm,7278-v3d" -- reg: Physical base addresses and lengths of the register areas -- reg-names: Names for the register areas. The "hub" and "core0" - register areas are always required. The "gca" register area - is required if the GCA cache controller is present. The - "bridge" register area is required if an external reset - controller is not present. -- interrupts: The interrupt numbers. The first interrupt is for the hub, - while the following interrupts are separate interrupt lines - for the cores (if they don't share the hub's interrupt). - See bindings/interrupt-controller/interrupts.txt - -Optional properties: -- clocks: The core clock the unit runs on -- resets: The reset line for v3d, if not using a mapping of the bridge - See bindings/reset/reset.txt - -v3d { - compatible = "brcm,7268-v3d"; - reg = <0xf1204000 0x100>, - <0xf1200000 0x4000>, - <0xf1208000 0x4000>, - <0xf1204100 0x100>; - reg-names = "bridge", "hub", "core0", "gca"; - interrupts = <0 78 4>, - <0 77 4>; -}; diff --git a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml new file mode 100644 index 000000000000..9d72264fa90a --- /dev/null +++ b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/gpu/brcm,bcm-v3d.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom V3D GPU Bindings + +maintainers: + - Eric Anholt + - Nicolas Saenz Julienne + +properties: + $nodename: + pattern: '^gpu@[a-f0-9]+$' + + compatible: + enum: + - brcm,7268-v3d + - brcm,7278-v3d + + reg: + items: + - description: hub register (required) + - description: core0 register (required) + - description: GCA cache controller register (if GCA controller present) + - description: bridge register (if no external reset controller) + minItems: 2 + + reg-names: + items: + - const: hub + - const: core0 + - enum: [ bridge, gca ] + - enum: [ bridge, gca ] + minItems: 2 + maxItems: 4 + + interrupts: + items: + - description: hub interrupt (required) + - description: core interrupts (if it doesn't share the hub's interrupt) + minItems: 1 + + clocks: + maxItems: 1 + + resets: + maxItems: 1 + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - reg-names + - interrupts + +additionalProperties: false + +examples: + - | + gpu@f1200000 { + compatible = "brcm,7268-v3d"; + reg = <0xf1200000 0x4000>, + <0xf1208000 0x4000>, + <0xf1204000 0x100>, + <0xf1204100 0x100>; + reg-names = "hub", "core0", "bridge", "gca"; + interrupts = <0 78 4>, + <0 77 4>; + }; + +... diff --git a/Documentation/devicetree/bindings/graph.txt b/Documentation/devicetree/bindings/graph.txt index 0415e2c53ba0..14733b5cb61e 100644 --- a/Documentation/devicetree/bindings/graph.txt +++ b/Documentation/devicetree/bindings/graph.txt @@ -1,128 +1 @@ -Common bindings for device graphs - -General concept ---------------- - -The hierarchical organisation of the device tree is well suited to describe -control flow to devices, but there can be more complex connections between -devices that work together to form a logical compound device, following an -arbitrarily complex graph. -There already is a simple directed graph between devices tree nodes using -phandle properties pointing to other nodes to describe connections that -can not be inferred from device tree parent-child relationships. The device -tree graph bindings described herein abstract more complex devices that can -have multiple specifiable ports, each of which can be linked to one or more -ports of other devices. - -These common bindings do not contain any information about the direction or -type of the connections, they just map their existence. Specific properties -may be described by specialized bindings depending on the type of connection. - -To see how this binding applies to video pipelines, for example, see -Documentation/devicetree/bindings/media/video-interfaces.txt. -Here the ports describe data interfaces, and the links between them are -the connecting data buses. A single port with multiple connections can -correspond to multiple devices being connected to the same physical bus. - -Organisation of ports and endpoints ------------------------------------ - -Ports are described by child 'port' nodes contained in the device node. -Each port node contains an 'endpoint' subnode for each remote device port -connected to this port. If a single port is connected to more than one -remote device, an 'endpoint' child node must be provided for each link. -If more than one port is present in a device node or there is more than one -endpoint at a port, or a port node needs to be associated with a selected -hardware interface, a common scheme using '#address-cells', '#size-cells' -and 'reg' properties is used to number the nodes. - -device { - ... - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - #address-cells = <1>; - #size-cells = <0>; - reg = <0>; - - endpoint@0 { - reg = <0>; - ... - }; - endpoint@1 { - reg = <1>; - ... - }; - }; - - port@1 { - reg = <1>; - - endpoint { ... }; - }; -}; - -All 'port' nodes can be grouped under an optional 'ports' node, which -allows to specify #address-cells, #size-cells properties for the 'port' -nodes independently from any other child device nodes a device might -have. - -device { - ... - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - ... - endpoint@0 { ... }; - endpoint@1 { ... }; - }; - - port@1 { ... }; - }; -}; - -Links between endpoints ------------------------ - -Each endpoint should contain a 'remote-endpoint' phandle property that points -to the corresponding endpoint in the port of the remote device. In turn, the -remote endpoint should contain a 'remote-endpoint' property. If it has one, it -must not point to anything other than the local endpoint. Two endpoints with -their 'remote-endpoint' phandles pointing at each other form a link between the -containing ports. - -device-1 { - port { - device_1_output: endpoint { - remote-endpoint = <&device_2_input>; - }; - }; -}; - -device-2 { - port { - device_2_input: endpoint { - remote-endpoint = <&device_1_output>; - }; - }; -}; - -Required properties -------------------- - -If there is more than one 'port' or more than one 'endpoint' node or 'reg' -property present in the port and/or endpoint nodes then the following -properties are required in a relevant parent node: - - - #address-cells : number of cells required to define port/endpoint - identifier, should be 1. - - #size-cells : should be zero. - -Optional endpoint properties ----------------------------- - -- remote-endpoint: phandle to an 'endpoint' subnode of a remote device node. - +This file has moved to graph.yaml in dt-schema repo diff --git a/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml b/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml index ac35491a6f65..ae1b37dbee75 100644 --- a/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml +++ b/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml @@ -13,6 +13,7 @@ properties: compatible: enum: - ti,omap4-hwspinlock # for OMAP44xx, OMAP54xx, AM33xx, AM43xx, DRA7xx SoCs + - ti,am64-hwspinlock # for K3 AM64x SoCs - ti,am654-hwspinlock # for K3 AM65x, J721E and J7200 SoCs reg: diff --git a/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml b/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml new file mode 100644 index 000000000000..3bc8e73dfbf0 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- + +$id: http://devicetree.org/schemas/hwmon/ti,tps23861.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: TI TPS23861 PoE PSE + +maintainers: + - Robert Marko + +description: | + The TPS23861 is a IEEE 802.3at Quad Port Power-over-Ethernet PSE Controller. + + Datasheets: + https://www.ti.com/lit/gpn/tps23861 + + +properties: + compatible: + enum: + - ti,tps23861 + + reg: + maxItems: 1 + + shunt-resistor-micro-ohms: + description: The value of curent sense resistor in microohms. + default: 255000 + minimum: 250000 + maximum: 255000 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + tps23861@30 { + compatible = "ti,tps23861"; + reg = <0x30>; + shunt-resistor-micro-ohms = <255000>; + }; + }; diff --git a/Documentation/devicetree/bindings/i2c/i2c-sirf.txt b/Documentation/devicetree/bindings/i2c/i2c-sirf.txt deleted file mode 100644 index 2701eefb00f7..000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-sirf.txt +++ /dev/null @@ -1,19 +0,0 @@ -I2C for SiRFprimaII platforms - -Required properties : -- compatible : Must be "sirf,prima2-i2c" -- reg: physical base address of the controller and length of memory mapped - region. -- interrupts: interrupt number to the cpu. - -Optional properties: -- clock-frequency : Constains desired I2C/HS-I2C bus clock frequency in Hz. - The absence of the property indicates the default frequency 100 kHz. - -Examples : - -i2c0: i2c@b00e0000 { - compatible = "sirf,prima2-i2c"; - reg = <0xb00e0000 0x10000>; - interrupts = <24>; -}; diff --git a/Documentation/devicetree/bindings/i2c/i2c-stu300.txt b/Documentation/devicetree/bindings/i2c/i2c-stu300.txt deleted file mode 100644 index bd81a482634f..000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-stu300.txt +++ /dev/null @@ -1,15 +0,0 @@ -ST Microelectronics DDC I2C - -Required properties : -- compatible : Must be "st,ddci2c" -- reg: physical base address of the controller and length of memory mapped - region. -- interrupts: interrupt number to the cpu. -- #address-cells = <1>; -- #size-cells = <0>; - -Optional properties: -- Child nodes conforming to i2c bus binding - -Examples : - diff --git a/Documentation/devicetree/bindings/i2c/i2c-zx2967.txt b/Documentation/devicetree/bindings/i2c/i2c-zx2967.txt deleted file mode 100644 index cb806d1ae4c9..000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-zx2967.txt +++ /dev/null @@ -1,22 +0,0 @@ -ZTE zx2967 I2C controller - -Required properties: - - compatible: must be "zte,zx296718-i2c" - - reg: physical address and length of the device registers - - interrupts: a single interrupt specifier - - clocks: clock for the device - - #address-cells: should be <1> - - #size-cells: should be <0> - - clock-frequency: the desired I2C bus clock frequency. - -Examples: - - i2c@112000 { - compatible = "zte,zx296718-i2c"; - reg = <0x00112000 0x1000>; - interrupts = ; - clocks = <&osc24m>; - #address-cells = <1> - #size-cells = <0>; - clock-frequency = <1600000>; - }; diff --git a/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml b/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml index 5b5ae402f97a..eb72dd571def 100644 --- a/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml @@ -18,21 +18,14 @@ properties: - const: allwinner,sun4i-a10-i2c - const: allwinner,sun6i-a31-i2c - items: - - const: allwinner,sun8i-a23-i2c + - enum: + - allwinner,sun8i-a23-i2c + - allwinner,sun8i-a83t-i2c + - allwinner,sun50i-a64-i2c + - allwinner,sun50i-a100-i2c + - allwinner,sun50i-h6-i2c + - allwinner,sun50i-h616-i2c - const: allwinner,sun6i-a31-i2c - - items: - - const: allwinner,sun8i-a83t-i2c - - const: allwinner,sun6i-a31-i2c - - items: - - const: allwinner,sun50i-a64-i2c - - const: allwinner,sun6i-a31-i2c - - items: - - const: allwinner,sun50i-a100-i2c - - const: allwinner,sun6i-a31-i2c - - items: - - const: allwinner,sun50i-h6-i2c - - const: allwinner,sun6i-a31-i2c - - const: marvell,mv64xxx-i2c - const: marvell,mv78230-i2c - const: marvell,mv78230-a0-i2c diff --git a/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml b/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml index e3ef2d36f372..128444942aec 100644 --- a/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml @@ -17,7 +17,7 @@ maintainers: properties: compatible: - const: nuvoton,npcm7xx-i2c + const: nuvoton,npcm750-i2c reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/i2c/renesas,i2c.txt b/Documentation/devicetree/bindings/i2c/renesas,i2c.txt index 96d869ac3839..5762d2d1ab9c 100644 --- a/Documentation/devicetree/bindings/i2c/renesas,i2c.txt +++ b/Documentation/devicetree/bindings/i2c/renesas,i2c.txt @@ -26,6 +26,7 @@ Required properties: "renesas,i2c-r8a77980" if the device is a part of a R8A77980 SoC. "renesas,i2c-r8a77990" if the device is a part of a R8A77990 SoC. "renesas,i2c-r8a77995" if the device is a part of a R8A77995 SoC. + "renesas,i2c-r8a779a0" if the device is a part of a R8A779A0 SoC. "renesas,rcar-gen1-i2c" for a generic R-Car Gen1 compatible device. "renesas,rcar-gen2-i2c" for a generic R-Car Gen2 or RZ/G1 compatible device. diff --git a/Documentation/devicetree/bindings/i3c/i3c.txt b/Documentation/devicetree/bindings/i3c/i3c.txt deleted file mode 100644 index 4ffe059f0fec..000000000000 --- a/Documentation/devicetree/bindings/i3c/i3c.txt +++ /dev/null @@ -1,140 +0,0 @@ -Generic device tree bindings for I3C busses -=========================================== - -This document describes generic bindings that should be used to describe I3C -busses in a device tree. - -Required properties -------------------- - -- #address-cells - should be <3>. Read more about addresses below. -- #size-cells - should be <0>. -- compatible - name of the I3C master controller driving the I3C bus - -For other required properties e.g. to describe register sets, -clocks, etc. check the binding documentation of the specific driver. -The node describing an I3C bus should be named i3c-master. - -Optional properties -------------------- - -These properties may not be supported by all I3C master drivers. Each I3C -master bindings should specify which of them are supported. - -- i3c-scl-hz: frequency of the SCL signal used for I3C transfers. - When undefined the core sets it to 12.5MHz. - -- i2c-scl-hz: frequency of the SCL signal used for I2C transfers. - When undefined, the core looks at LVR (Legacy Virtual Register) - values of I2C devices described in the device tree to determine - the maximum I2C frequency. - -I2C devices -=========== - -Each I2C device connected to the bus should be described in a subnode. All -properties described in Documentation/devicetree/bindings/i2c/i2c.txt are -valid here, but several new properties have been added. - -New constraint on existing properties: --------------------------------------- -- reg: contains 3 cells - + first cell : still encoding the I2C address. 10 bit addressing is not - supported. Devices with 10 bit address can't be properly passed through - DEFSLVS command. - - + second cell: shall be 0 - - + third cell: shall encode the I3C LVR (Legacy Virtual Register) - bit[31:8]: unused/ignored - bit[7:5]: I2C device index. Possible values - * 0: I2C device has a 50 ns spike filter - * 1: I2C device does not have a 50 ns spike filter but supports high - frequency on SCL - * 2: I2C device does not have a 50 ns spike filter and is not tolerant - to high frequencies - * 3-7: reserved - - bit[4]: tell whether the device operates in FM (Fast Mode) or FM+ mode - * 0: FM+ mode - * 1: FM mode - - bit[3:0]: device type - * 0-15: reserved - -The I2C node unit-address should always match the first cell of the reg -property: @. - -I3C devices -=========== - -All I3C devices are supposed to support DAA (Dynamic Address Assignment), and -are thus discoverable. So, by default, I3C devices do not have to be described -in the device tree. -This being said, one might want to attach extra resources to these devices, -and those resources may have to be described in the device tree, which in turn -means we have to describe I3C devices. - -Another use case for describing an I3C device in the device tree is when this -I3C device has a static I2C address and we want to assign it a specific I3C -dynamic address before the DAA takes place (so that other devices on the bus -can't take this dynamic address). - -The I3C device should be names @,, -where device-type is describing the type of device connected on the bus -(gpio-controller, sensor, ...). - -Required properties -------------------- -- reg: contains 3 cells - + first cell : encodes the static I2C address. Should be 0 if the device does - not have one (0 is not a valid I2C address). - - + second and third cells: should encode the ProvisionalID. The second cell - contains the manufacturer ID left-shifted by 1. - The third cell contains ORing of the part ID - left-shifted by 16, the instance ID left-shifted - by 12 and the extra information. This encoding is - following the PID definition provided by the I3C - specification. - -Optional properties -------------------- -- assigned-address: dynamic address to be assigned to this device. This - property is only valid if the I3C device has a static - address (first cell of the reg property != 0). - - -Example: - - i3c-master@d040000 { - compatible = "cdns,i3c-master"; - clocks = <&coreclock>, <&i3csysclock>; - clock-names = "pclk", "sysclk"; - interrupts = <3 0>; - reg = <0x0d040000 0x1000>; - #address-cells = <3>; - #size-cells = <0>; - i2c-scl-hz = <100000>; - - /* I2C device. */ - nunchuk: nunchuk@52 { - compatible = "nintendo,nunchuk"; - reg = <0x52 0x0 0x10>; - }; - - /* I3C device with a static I2C address. */ - thermal_sensor: sensor@68,39200144004 { - reg = <0x68 0x392 0x144004>; - assigned-address = <0xa>; - }; - - /* - * I3C device without a static I2C address but requiring - * resources described in the DT. - */ - sensor@0,39200154004 { - reg = <0x0 0x392 0x154004>; - clocks = <&clock_provider 0>; - }; - }; diff --git a/Documentation/devicetree/bindings/i3c/i3c.yaml b/Documentation/devicetree/bindings/i3c/i3c.yaml new file mode 100644 index 000000000000..52042aa44d19 --- /dev/null +++ b/Documentation/devicetree/bindings/i3c/i3c.yaml @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i3c/i3c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: I3C bus binding + +maintainers: + - Alexandre Belloni + - Miquel Raynal + +description: | + I3C busses can be described with a node for the primary I3C controller device + and a set of child nodes for each I2C or I3C slave on the bus. Each of them + may, during the life of the bus, request mastership. + +properties: + $nodename: + pattern: "^i3c-master@[0-9a-f]+$" + + "#address-cells": + const: 3 + description: | + Each I2C device connected to the bus should be described in a subnode. + + All I3C devices are supposed to support DAA (Dynamic Address Assignment), + and are thus discoverable. So, by default, I3C devices do not have to be + described in the device tree. This being said, one might want to attach + extra resources to these devices, and those resources may have to be + described in the device tree, which in turn means we have to describe + I3C devices. + + Another use case for describing an I3C device in the device tree is when + this I3C device has a static I2C address and we want to assign it a + specific I3C dynamic address before the DAA takes place (so that other + devices on the bus can't take this dynamic address). + + "#size-cells": + const: 0 + + i3c-scl-hz: + description: | + Frequency of the SCL signal used for I3C transfers. When undefined, the + default value should be 12.5MHz. + + May not be supported by all controllers. + + i2c-scl-hz: + description: | + Frequency of the SCL signal used for I2C transfers. When undefined, the + default should be to look at LVR (Legacy Virtual Register) values of + I2C devices described in the device tree to determine the maximum I2C + frequency. + + May not be supported by all controllers. + +required: + - "#address-cells" + - "#size-cells" + +patternProperties: + "@[0-9a-f]+$": + type: object + description: | + I2C child, should be named: @ + + All properties described in Documentation/devicetree/bindings/i2c/i2c.txt + are valid here, except the reg property whose content is changed. + + properties: + compatible: + description: + Compatible of the I2C device. + + reg: + items: + - items: + - description: | + I2C address. 10 bit addressing is not supported. Devices with + 10-bit address can't be properly passed through DEFSLVS + command. + minimum: 0 + maximum: 0x7f + - const: 0 + - description: | + Shall encode the I3C LVR (Legacy Virtual Register): + bit[31:8]: unused/ignored + bit[7:5]: I2C device index. Possible values: + * 0: I2C device has a 50 ns spike filter + * 1: I2C device does not have a 50 ns spike filter but + supports high frequency on SCL + * 2: I2C device does not have a 50 ns spike filter and is + not tolerant to high frequencies + * 3-7: reserved + bit[4]: tell whether the device operates in FM (Fast Mode) + or FM+ mode: + * 0: FM+ mode + * 1: FM mode + bit[3:0]: device type + * 0-15: reserved + + required: + - compatible + - reg + + "@[0-9a-f]+,[0-9a-f]+$": + type: object + description: | + I3C child, should be named: @, + + properties: + reg: + items: + - items: + - description: | + Encodes the static I2C address. Should be 0 if the device does + not have one (0 is not a valid I2C address). + minimum: 0 + maximum: 0x7f + - description: | + First half of the Provisional ID (following the PID + definition provided by the I3C specification). + + Contains the manufacturer ID left-shifted by 1. + - description: | + Second half of the Provisional ID (following the PID + definition provided by the I3C specification). + + Contains the ORing of the part ID left-shifted by 16, + the instance ID left-shifted by 12 and extra information. + + assigned-address: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0x1 + maximum: 0xff + description: | + Dynamic address to be assigned to this device. This property is only + valid if the I3C device has a static address (first cell of the reg + property != 0). + + required: + - reg + +additionalProperties: true + +examples: + - | + i3c-master@d040000 { + compatible = "cdns,i3c-master"; + clocks = <&coreclock>, <&i3csysclock>; + clock-names = "pclk", "sysclk"; + interrupts = <3 0>; + reg = <0x0d040000 0x1000>; + #address-cells = <3>; + #size-cells = <0>; + i2c-scl-hz = <100000>; + + /* I2C device. */ + nunchuk: nunchuk@52 { + compatible = "nintendo,nunchuk"; + reg = <0x52 0x0 0x10>; + }; + + /* I3C device with a static I2C address. */ + thermal_sensor: sensor@68,39200144004 { + reg = <0x68 0x392 0x144004>; + assigned-address = <0xa>; + }; + + /* + * I3C device without a static I2C address but requiring + * resources described in the DT. + */ + sensor@0,39200154004 { + reg = <0x0 0x392 0x154004>; + clocks = <&clock_provider 0>; + }; + }; diff --git a/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml b/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml index 07a7b10163a3..04da001fc6ec 100644 --- a/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml +++ b/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml @@ -9,6 +9,9 @@ title: MIPI I3C HCI Device Tree Bindings maintainers: - Nicolas Pitre +allOf: + - $ref: /schemas/i3c/i3c.yaml# + description: | MIPI I3C Host Controller Interface @@ -36,12 +39,14 @@ required: - reg - interrupts -additionalProperties: false +unevaluatedProperties: false examples: - | - i3c@a0000000 { + i3c-master@a0000000 { compatible = "mipi-i3c-hci"; reg = <0xa0000000 0x2000>; interrupts = <89>; + #address-cells = <3>; + #size-cells = <0>; }; diff --git a/Documentation/devicetree/bindings/i3c/silvaco,i3c-master.yaml b/Documentation/devicetree/bindings/i3c/silvaco,i3c-master.yaml new file mode 100644 index 000000000000..adb5165505aa --- /dev/null +++ b/Documentation/devicetree/bindings/i3c/silvaco,i3c-master.yaml @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i3c/silvaco,i3c-master.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Silvaco I3C master + +maintainers: + - Conor Culhane + +allOf: + - $ref: "i3c.yaml#" + +properties: + compatible: + const: silvaco,i3c-master-v1 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: system clock + - description: bus clock + - description: other (slower) events clock + + clock-names: + items: + - const: pclk + - const: fast_clk + - const: slow_clk + + resets: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clock-names + - clocks + +additionalProperties: true + +examples: + - | + i3c-master@a0000000 { + compatible = "silvaco,i3c-master"; + clocks = <&zynqmp_clk 71>, <&fclk>, <&sclk>; + clock-names = "pclk", "fast_clk", "slow_clk"; + interrupt-parent = <&gic>; + interrupts = <0 89 4>; + reg = <0xa0000000 0x1000>; + #address-cells = <3>; + #size-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml index 924477dfb833..a85a28145ef6 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml @@ -40,6 +40,7 @@ properties: ADC reference voltage supply adi,sync-in-gpios: + maxItems: 1 description: Enables synchronization of multiple devices that require simultaneous sampling. A pulse is always required if the configuration is changed @@ -76,6 +77,7 @@ patternProperties: properties: reg: + maxItems: 1 description: | The channel number. diff --git a/Documentation/devicetree/bindings/iio/adc/aspeed,ast2400-adc.yaml b/Documentation/devicetree/bindings/iio/adc/aspeed,ast2400-adc.yaml index 7f534a933e92..a726b6c2ab65 100644 --- a/Documentation/devicetree/bindings/iio/adc/aspeed,ast2400-adc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/aspeed,ast2400-adc.yaml @@ -23,6 +23,7 @@ properties: maxItems: 1 clocks: + maxItems: 1 description: Input clock used to derive the sample clock. Expected to be the SoC's APB clock. diff --git a/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml b/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml index 2716d4e95329..0bd2fc0356c8 100644 --- a/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml +++ b/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml @@ -20,7 +20,7 @@ properties: description: Power supply for the reference voltage reg: - description: spi chipselect number according to the usual spi bindings + maxItems: 1 spi-max-frequency: description: maximal spi bus frequency supported diff --git a/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml index 95cc705b961b..74a4a9d95798 100644 --- a/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml @@ -68,6 +68,7 @@ patternProperties: properties: reg: + maxItems: 1 description: | ADC channel number. See include/dt-bindings/iio/qcom,spmi-vadc.h diff --git a/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.yaml b/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.yaml index 517e32976c30..a58334c3bb76 100644 --- a/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.yaml @@ -41,6 +41,8 @@ properties: maxItems: 2 clocks: + minItems: 1 + maxItems: 2 description: | Core can use up to two clocks, depending on part used: - "adc" clock: for the analog circuitry, common to all ADCs. diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml similarity index 77% rename from Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml rename to Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml index 8065228e5df8..56b0cda0f30a 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml @@ -1,16 +1,16 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/iio/dac/adi,ad5686.yaml# +$id: http://devicetree.org/schemas/iio/dac/adi,ad5696.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Analog Devices AD5686 and similar multi-channel DACs +title: Analog Devices AD5696 and similar multi-channel DACs maintainers: - Michael Auchter description: | - Binding for Analog Devices AD5686 and similar multi-channel DACs + Binding for Analog Devices AD5696 and similar multi-channel DACs properties: compatible: @@ -48,8 +48,8 @@ examples: #address-cells = <1>; #size-cells = <0>; - ad5686: dac@0 { - compatible = "adi,ad5686"; + ad5696: dac@0 { + compatible = "adi,ad5696"; reg = <0>; vcc-supply = <&dac_vref>; }; diff --git a/Documentation/devicetree/bindings/iio/health/ti,afe4404.yaml b/Documentation/devicetree/bindings/iio/health/ti,afe4404.yaml index 3b4d6c48b8bb..c0e815d9999e 100644 --- a/Documentation/devicetree/bindings/iio/health/ti,afe4404.yaml +++ b/Documentation/devicetree/bindings/iio/health/ti,afe4404.yaml @@ -11,7 +11,7 @@ maintainers: properties: compatible: - const: ti,afe4403 + const: ti,afe4404 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8975.yaml b/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8975.yaml index a25590a16ba7..a0a1ffe017df 100644 --- a/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8975.yaml +++ b/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8975.yaml @@ -47,6 +47,7 @@ properties: description: an optional 3x3 mounting rotation matrix. reset-gpios: + maxItems: 1 description: | an optional pin needed for AK09911 to set the reset state. This should be usually active low diff --git a/Documentation/devicetree/bindings/iio/potentiometer/adi,ad5272.yaml b/Documentation/devicetree/bindings/iio/potentiometer/adi,ad5272.yaml index 1aee9f9be951..0ebb6725a1af 100644 --- a/Documentation/devicetree/bindings/iio/potentiometer/adi,ad5272.yaml +++ b/Documentation/devicetree/bindings/iio/potentiometer/adi,ad5272.yaml @@ -25,6 +25,7 @@ properties: maxItems: 1 reset-gpios: + maxItems: 1 description: Active low signal to the AD5272 RESET input. diff --git a/Documentation/devicetree/bindings/input/goodix,gt7375p.yaml b/Documentation/devicetree/bindings/input/goodix,gt7375p.yaml new file mode 100644 index 000000000000..fe1c5016f7f3 --- /dev/null +++ b/Documentation/devicetree/bindings/input/goodix,gt7375p.yaml @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/goodix,gt7375p.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Goodix GT7375P touchscreen + +maintainers: + - Douglas Anderson + +description: + Supports the Goodix GT7375P touchscreen. + This touchscreen uses the i2c-hid protocol but has some non-standard + power sequencing required. + +properties: + compatible: + items: + - const: goodix,gt7375p + + reg: + enum: + - 0x5d + - 0x14 + + interrupts: + maxItems: 1 + + reset-gpios: + true + + vdd-supply: + description: The 3.3V supply to the touchscreen. + +required: + - compatible + - reg + - interrupts + - reset-gpios + - vdd-supply + +additionalProperties: false + +examples: + - | + #include + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + ap_ts: touchscreen@5d { + compatible = "goodix,gt7375p"; + reg = <0x5d>; + + interrupt-parent = <&tlmm>; + interrupts = <9 IRQ_TYPE_LEVEL_LOW>; + + reset-gpios = <&tlmm 8 GPIO_ACTIVE_LOW>; + vdd-supply = <&pp3300_ts>; + }; + }; diff --git a/Documentation/devicetree/bindings/input/google,cros-ec-keyb.yaml b/Documentation/devicetree/bindings/input/google,cros-ec-keyb.yaml index 8e50c14a9d77..5377b232fa10 100644 --- a/Documentation/devicetree/bindings/input/google,cros-ec-keyb.yaml +++ b/Documentation/devicetree/bindings/input/google,cros-ec-keyb.yaml @@ -31,6 +31,17 @@ properties: if the EC does not have its own logic or hardware for this. type: boolean + function-row-physmap: + minItems: 1 + maxItems: 15 + description: | + An ordered u32 array describing the rows/columns (in the scan matrix) + of top row keys from physical left (KEY_F1) to right. Each entry + encodes the row/column as: + (((row) & 0xFF) << 24) | (((column) & 0xFF) << 16) + where the lower 16 bits are reserved. This property is specified only + when the keyboard has a custom design for the top row keys. + required: - compatible @@ -38,11 +49,24 @@ unevaluatedProperties: false examples: - | + #include cros-ec-keyb { compatible = "google,cros-ec-keyb"; keypad,num-rows = <8>; keypad,num-columns = <13>; google,needs-ghost-filter; + function-row-physmap = < + MATRIX_KEY(0x00, 0x02, 0) /* T1 */ + MATRIX_KEY(0x03, 0x02, 0) /* T2 */ + MATRIX_KEY(0x02, 0x02, 0) /* T3 */ + MATRIX_KEY(0x01, 0x02, 0) /* T4 */ + MATRIX_KEY(0x03, 0x04, 0) /* T5 */ + MATRIX_KEY(0x02, 0x04, 0) /* T6 */ + MATRIX_KEY(0x01, 0x04, 0) /* T7 */ + MATRIX_KEY(0x02, 0x09, 0) /* T8 */ + MATRIX_KEY(0x01, 0x09, 0) /* T9 */ + MATRIX_KEY(0x00, 0x04, 0) /* T10 */ + >; /* * Keymap entries take the form of 0xRRCCKKKK where * RR=Row CC=Column KKKK=Key Code diff --git a/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml b/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml index a792d6377b1d..a9b53c2e6f0a 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml +++ b/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml @@ -29,6 +29,7 @@ properties: description: touchscreen can be used as a wakeup source. reset-gpios: + maxItems: 1 description: reset gpio the chip is connected to. vcc33-supply: diff --git a/Documentation/devicetree/bindings/interconnect/qcom,qcs404.yaml b/Documentation/devicetree/bindings/interconnect/qcom,qcs404.yaml deleted file mode 100644 index 3fbb8785fbc9..000000000000 --- a/Documentation/devicetree/bindings/interconnect/qcom,qcs404.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/interconnect/qcom,qcs404.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Qualcomm QCS404 Network-On-Chip interconnect - -maintainers: - - Georgi Djakov - -description: | - The Qualcomm QCS404 interconnect providers support adjusting the - bandwidth requirements between the various NoC fabrics. - -properties: - reg: - maxItems: 1 - - compatible: - enum: - - qcom,qcs404-bimc - - qcom,qcs404-pcnoc - - qcom,qcs404-snoc - - '#interconnect-cells': - const: 1 - - clock-names: - items: - - const: bus - - const: bus_a - - clocks: - items: - - description: Bus Clock - - description: Bus A Clock - -required: - - compatible - - reg - - '#interconnect-cells' - - clock-names - - clocks - -additionalProperties: false - -examples: - - | - #include - - bimc: interconnect@400000 { - reg = <0x00400000 0x80000>; - compatible = "qcom,qcs404-bimc"; - #interconnect-cells = <1>; - clock-names = "bus", "bus_a"; - clocks = <&rpmcc RPM_SMD_BIMC_CLK>, - <&rpmcc RPM_SMD_BIMC_A_CLK>; - }; - - pnoc: interconnect@500000 { - reg = <0x00500000 0x15080>; - compatible = "qcom,qcs404-pcnoc"; - #interconnect-cells = <1>; - clock-names = "bus", "bus_a"; - clocks = <&rpmcc RPM_SMD_PNOC_CLK>, - <&rpmcc RPM_SMD_PNOC_A_CLK>; - }; - - snoc: interconnect@580000 { - reg = <0x00580000 0x23080>; - compatible = "qcom,qcs404-snoc"; - #interconnect-cells = <1>; - clock-names = "bus", "bus_a"; - clocks = <&rpmcc RPM_SMD_SNOC_CLK>, - <&rpmcc RPM_SMD_SNOC_A_CLK>; - }; diff --git a/Documentation/devicetree/bindings/interconnect/qcom,msm8916.yaml b/Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml similarity index 77% rename from Documentation/devicetree/bindings/interconnect/qcom,msm8916.yaml rename to Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml index e1009ae4e8f7..983d71fb5399 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,msm8916.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml @@ -1,27 +1,35 @@ # SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/interconnect/qcom,msm8916.yaml# +$id: http://devicetree.org/schemas/interconnect/qcom,rpm.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Qualcomm MSM8916 Network-On-Chip interconnect +title: Qualcomm RPM Network-On-Chip Interconnect maintainers: - Georgi Djakov description: | - The Qualcomm MSM8916 interconnect providers support adjusting the - bandwidth requirements between the various NoC fabrics. + RPM interconnect providers support system bandwidth requirements through + RPM processor. The provider is able to communicate with the RPM through + the RPM shared memory device. properties: + reg: + maxItems: 1 + compatible: enum: - qcom,msm8916-bimc - qcom,msm8916-pcnoc - qcom,msm8916-snoc - - reg: - maxItems: 1 + - qcom,msm8939-bimc + - qcom,msm8939-pcnoc + - qcom,msm8939-snoc + - qcom,msm8939-snoc-mm + - qcom,qcs404-bimc + - qcom,qcs404-pcnoc + - qcom,qcs404-snoc '#interconnect-cells': const: 1 diff --git a/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml index 30c2a092d2d3..799e73cdb90b 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml @@ -45,6 +45,10 @@ properties: - qcom,sdm845-mem-noc - qcom,sdm845-mmss-noc - qcom,sdm845-system-noc + - qcom,sdx55-ipa-virt + - qcom,sdx55-mc-virt + - qcom,sdx55-mem-noc + - qcom,sdx55-system-noc - qcom,sm8150-aggre1-noc - qcom,sm8150-aggre2-noc - qcom,sm8150-camnoc-noc @@ -69,7 +73,7 @@ properties: - qcom,sm8250-system-noc '#interconnect-cells': - const: 1 + enum: [ 1, 2 ] qcom,bcm-voters: $ref: /schemas/types.yaml#/definitions/phandle-array diff --git a/Documentation/devicetree/bindings/interrupt-controller/fsl,intmux.yaml b/Documentation/devicetree/bindings/interrupt-controller/fsl,intmux.yaml index 43c6effbb5bd..1d6e0f64a807 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/fsl,intmux.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/fsl,intmux.yaml @@ -31,7 +31,7 @@ properties: The 1st cell is hw interrupt number, the 2nd cell is channel index. clocks: - description: ipg clock. + maxItems: 1 clock-names: const: ipg diff --git a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml index b9a61c9f7530..08d5a57ce00f 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml @@ -8,10 +8,11 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: SiFive Platform-Level Interrupt Controller (PLIC) description: - SiFive SOCs include an implementation of the Platform-Level Interrupt Controller - (PLIC) high-level specification in the RISC-V Privileged Architecture - specification. The PLIC connects all external interrupts in the system to all - hart contexts in the system, via the external interrupt source in each hart. + SiFive SoCs and other RISC-V SoCs include an implementation of the + Platform-Level Interrupt Controller (PLIC) high-level specification in + the RISC-V Privileged Architecture specification. The PLIC connects all + external interrupts in the system to all hart contexts in the system, via + the external interrupt source in each hart. A hart context is a privilege mode in a hardware execution thread. For example, in an 4 core system with 2-way SMT, you have 8 harts and probably at least two @@ -42,7 +43,9 @@ maintainers: properties: compatible: items: - - const: sifive,fu540-c000-plic + - enum: + - sifive,fu540-c000-plic + - canaan,k210-plic - const: sifive,plic-1.0.0 reg: diff --git a/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml b/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml index 2a5b29567926..6d3e68eb2e8b 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml @@ -36,6 +36,8 @@ properties: Reference to a phandle of a hardware spinlock provider node. interrupts: + minItems: 1 + maxItems: 96 description: Interrupts references to primary interrupt controller diff --git a/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.yaml index c2ce215501a5..9731dd4421a1 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.yaml @@ -33,6 +33,9 @@ description: | corresponding PRUSS node. The node should be named "interrupt-controller". properties: + $nodename: + pattern: "^interrupt-controller@[0-9a-f]+$" + compatible: enum: - ti,pruss-intc diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 3b63f2ae24db..6ba161dea4d8 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -34,9 +34,11 @@ properties: items: - enum: - qcom,sc7180-smmu-500 + - qcom,sc8180x-smmu-500 - qcom,sdm845-smmu-500 - qcom,sm8150-smmu-500 - qcom,sm8250-smmu-500 + - qcom,sm8350-smmu-500 - const: arm,mmu-500 - description: Qcom Adreno GPUs implementing "arm,smmu-v2" items: diff --git a/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt b/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt deleted file mode 100644 index ac949f7fe3d4..000000000000 --- a/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt +++ /dev/null @@ -1,105 +0,0 @@ -* Mediatek IOMMU Architecture Implementation - - Some Mediatek SOCs contain a Multimedia Memory Management Unit (M4U), and -this M4U have two generations of HW architecture. Generation one uses flat -pagetable, and only supports 4K size page mapping. Generation two uses the -ARM Short-Descriptor translation table format for address translation. - - About the M4U Hardware Block Diagram, please check below: - - EMI (External Memory Interface) - | - m4u (Multimedia Memory Management Unit) - | - +--------+ - | | - gals0-rx gals1-rx (Global Async Local Sync rx) - | | - | | - gals0-tx gals1-tx (Global Async Local Sync tx) - | | Some SoCs may have GALS. - +--------+ - | - SMI Common(Smart Multimedia Interface Common) - | - +----------------+------- - | | - | gals-rx There may be GALS in some larbs. - | | - | | - | gals-tx - | | - SMI larb0 SMI larb1 ... SoCs have several SMI local arbiter(larb). - (display) (vdec) - | | - | | - +-----+-----+ +----+----+ - | | | | | | - | | |... | | | ... There are different ports in each larb. - | | | | | | -OVL0 RDMA0 WDMA0 MC PP VLD - - As above, The Multimedia HW will go through SMI and M4U while it -access EMI. SMI is a bridge between m4u and the Multimedia HW. It contain -smi local arbiter and smi common. It will control whether the Multimedia -HW should go though the m4u for translation or bypass it and talk -directly with EMI. And also SMI help control the power domain and clocks for -each local arbiter. - Normally we specify a local arbiter(larb) for each multimedia HW -like display, video decode, and camera. And there are different ports -in each larb. Take a example, There are many ports like MC, PP, VLD in the -video decode local arbiter, all these ports are according to the video HW. - In some SoCs, there may be a GALS(Global Async Local Sync) module between -smi-common and m4u, and additional GALS module between smi-larb and -smi-common. GALS can been seen as a "asynchronous fifo" which could help -synchronize for the modules in different clock frequency. - -Required properties: -- compatible : must be one of the following string: - "mediatek,mt2701-m4u" for mt2701 which uses generation one m4u HW. - "mediatek,mt2712-m4u" for mt2712 which uses generation two m4u HW. - "mediatek,mt6779-m4u" for mt6779 which uses generation two m4u HW. - "mediatek,mt7623-m4u", "mediatek,mt2701-m4u" for mt7623 which uses - generation one m4u HW. - "mediatek,mt8167-m4u" for mt8167 which uses generation two m4u HW. - "mediatek,mt8173-m4u" for mt8173 which uses generation two m4u HW. - "mediatek,mt8183-m4u" for mt8183 which uses generation two m4u HW. -- reg : m4u register base and size. -- interrupts : the interrupt of m4u. -- clocks : must contain one entry for each clock-names. -- clock-names : Only 1 optional clock: - - "bclk": the block clock of m4u. - Here is the list which require this "bclk": - - mt2701, mt2712, mt7623 and mt8173. - Note that m4u use the EMI clock which always has been enabled before kernel - if there is no this "bclk". -- mediatek,larbs : List of phandle to the local arbiters in the current Socs. - Refer to bindings/memory-controllers/mediatek,smi-larb.txt. It must sort - according to the local arbiter index, like larb0, larb1, larb2... -- iommu-cells : must be 1. This is the mtk_m4u_id according to the HW. - Specifies the mtk_m4u_id as defined in - dt-binding/memory/mt2701-larb-port.h for mt2701, mt7623 - dt-binding/memory/mt2712-larb-port.h for mt2712, - dt-binding/memory/mt6779-larb-port.h for mt6779, - dt-binding/memory/mt8167-larb-port.h for mt8167, - dt-binding/memory/mt8173-larb-port.h for mt8173, and - dt-binding/memory/mt8183-larb-port.h for mt8183. - -Example: - iommu: iommu@10205000 { - compatible = "mediatek,mt8173-m4u"; - reg = <0 0x10205000 0 0x1000>; - interrupts = ; - clocks = <&infracfg CLK_INFRA_M4U>; - clock-names = "bclk"; - mediatek,larbs = <&larb0 &larb1 &larb2 &larb3 &larb4 &larb5>; - #iommu-cells = <1>; - }; - -Example for a client device: - display { - compatible = "mediatek,mt8173-disp"; - iommus = <&iommu M4U_PORT_DISP_OVL0>, - <&iommu M4U_PORT_DISP_RDMA0>; - ... - }; diff --git a/Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml b/Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml new file mode 100644 index 000000000000..0f26fe14c8e2 --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/mediatek,iommu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek IOMMU Architecture Implementation + +maintainers: + - Yong Wu + +description: |+ + Some MediaTek SOCs contain a Multimedia Memory Management Unit (M4U), and + this M4U have two generations of HW architecture. Generation one uses flat + pagetable, and only supports 4K size page mapping. Generation two uses the + ARM Short-Descriptor translation table format for address translation. + + About the M4U Hardware Block Diagram, please check below: + + EMI (External Memory Interface) + | + m4u (Multimedia Memory Management Unit) + | + +--------+ + | | + gals0-rx gals1-rx (Global Async Local Sync rx) + | | + | | + gals0-tx gals1-tx (Global Async Local Sync tx) + | | Some SoCs may have GALS. + +--------+ + | + SMI Common(Smart Multimedia Interface Common) + | + +----------------+------- + | | + | gals-rx There may be GALS in some larbs. + | | + | | + | gals-tx + | | + SMI larb0 SMI larb1 ... SoCs have several SMI local arbiter(larb). + (display) (vdec) + | | + | | + +-----+-----+ +----+----+ + | | | | | | + | | |... | | | ... There are different ports in each larb. + | | | | | | + OVL0 RDMA0 WDMA0 MC PP VLD + + As above, The Multimedia HW will go through SMI and M4U while it + access EMI. SMI is a bridge between m4u and the Multimedia HW. It contain + smi local arbiter and smi common. It will control whether the Multimedia + HW should go though the m4u for translation or bypass it and talk + directly with EMI. And also SMI help control the power domain and clocks for + each local arbiter. + + Normally we specify a local arbiter(larb) for each multimedia HW + like display, video decode, and camera. And there are different ports + in each larb. Take a example, There are many ports like MC, PP, VLD in the + video decode local arbiter, all these ports are according to the video HW. + + In some SoCs, there may be a GALS(Global Async Local Sync) module between + smi-common and m4u, and additional GALS module between smi-larb and + smi-common. GALS can been seen as a "asynchronous fifo" which could help + synchronize for the modules in different clock frequency. + +properties: + compatible: + oneOf: + - enum: + - mediatek,mt2701-m4u # generation one + - mediatek,mt2712-m4u # generation two + - mediatek,mt6779-m4u # generation two + - mediatek,mt8167-m4u # generation two + - mediatek,mt8173-m4u # generation two + - mediatek,mt8183-m4u # generation two + - mediatek,mt8192-m4u # generation two + + - description: mt7623 generation one + items: + - const: mediatek,mt7623-m4u + - const: mediatek,mt2701-m4u + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: bclk is the block clock. + + clock-names: + items: + - const: bclk + + mediatek,larbs: + $ref: /schemas/types.yaml#/definitions/phandle-array + minItems: 1 + maxItems: 32 + description: | + List of phandle to the local arbiters in the current Socs. + Refer to bindings/memory-controllers/mediatek,smi-larb.yaml. It must sort + according to the local arbiter index, like larb0, larb1, larb2... + + '#iommu-cells': + const: 1 + description: | + This is the mtk_m4u_id according to the HW. Specifies the mtk_m4u_id as + defined in + dt-binding/memory/mt2701-larb-port.h for mt2701 and mt7623, + dt-binding/memory/mt2712-larb-port.h for mt2712, + dt-binding/memory/mt6779-larb-port.h for mt6779, + dt-binding/memory/mt8167-larb-port.h for mt8167, + dt-binding/memory/mt8173-larb-port.h for mt8173, + dt-binding/memory/mt8183-larb-port.h for mt8183, + dt-binding/memory/mt8192-larb-port.h for mt8192. + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - mediatek,larbs + - '#iommu-cells' + +allOf: + - if: + properties: + compatible: + contains: + enum: + - mediatek,mt2701-m4u + - mediatek,mt2712-m4u + - mediatek,mt8173-m4u + - mediatek,mt8192-m4u + + then: + required: + - clocks + + - if: + properties: + compatible: + enum: + - mediatek,mt8192-m4u + + then: + required: + - power-domains + +additionalProperties: false + +examples: + - | + #include + #include + + iommu: iommu@10205000 { + compatible = "mediatek,mt8173-m4u"; + reg = <0x10205000 0x1000>; + interrupts = ; + clocks = <&infracfg CLK_INFRA_M4U>; + clock-names = "bclk"; + mediatek,larbs = <&larb0 &larb1 &larb2 + &larb3 &larb4 &larb5>; + #iommu-cells = <1>; + }; + + - | + #include + + /* Example for a client device */ + display { + compatible = "mediatek,mt8173-disp"; + iommus = <&iommu M4U_PORT_DISP_OVL0>, + <&iommu M4U_PORT_DISP_RDMA0>; + }; diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml index cde1afa8dfd6..dda44976acc1 100644 --- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml @@ -76,7 +76,6 @@ required: - compatible - reg - '#iommu-cells' - - power-domains oneOf: - required: @@ -86,6 +85,17 @@ oneOf: additionalProperties: false +allOf: + - if: + properties: + compatible: + not: + contains: + const: renesas,ipmmu-vmsa + then: + required: + - power-domains + examples: - | #include @@ -93,7 +103,7 @@ examples: #include ipmmu_mx: iommu@fe951000 { - compatible = "renasas,ipmmu-r8a7791", "renasas,ipmmu-vmsa"; + compatible = "renesas,ipmmu-r8a7791", "renesas,ipmmu-vmsa"; reg = <0xfe951000 0x1000>; interrupts = , ; diff --git a/Documentation/devicetree/bindings/leds/leds-lgm.yaml b/Documentation/devicetree/bindings/leds/leds-lgm.yaml new file mode 100644 index 000000000000..32bbf146c01d --- /dev/null +++ b/Documentation/devicetree/bindings/leds/leds-lgm.yaml @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/leds/leds-lgm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel Lightning Mountain (LGM) SoC LED Serial Shift Output (SSO) Controller driver + +maintainers: + - Zhu, Yi Xin + - Amireddy Mallikarjuna reddy + +properties: + compatible: + const: intel,lgm-ssoled + + gpio-controller: true + + '#gpio-cells': + const: 2 + + ngpios: + minimum: 0 + maximum: 32 + description: + Number of GPIOs this controller provides. + + intel,sso-update-rate-hz: + description: + Blink frequency for SOUTs in Hz. + + led-controller: + type: object + description: + This sub-node must contain a sub-node for each leds. + + additionalProperties: false + + patternProperties: + "^led@[0-23]$": + type: object + + properties: + reg: + description: Index of the LED. + minimum: 0 + maximum: 2 + + intel,sso-hw-trigger: + type: boolean + description: This property indicates Hardware driven/control LED. + + intel,sso-hw-blink: + type: boolean + description: This property indicates Enable LED blink by Hardware. + + intel,sso-blink-rate-hz: + description: LED HW blink frequency. + + retain-state-suspended: + type: boolean + description: The suspend state of LED can be retained. + + retain-state-shutdown: + type: boolean + description: Retain the state of the LED on shutdown. + +required: + - compatible + - reg + - clocks + - clock-names + - "#gpio-cells" + - gpio-controller + +additionalProperties: false + +examples: + - | + #include + #include + + ssogpio: ssogpio@e0d40000 { + compatible = "intel,sso-led"; + reg = <0xE0D40000 0x2E4>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <32>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ledc>; + clocks = <&cgu0 LGM_GCLK_LEDC0>, <&afeclk>; + clock-names = "sso", "fpid"; + intel,sso-update-rate-hz = <250000>; + + led-controller { + #address-cells = <1>; + #size-cells = <0>; + + led@0 { + reg = <0>; + function = "gphy"; + color = ; + led-gpio = <&ssogpio 0 0>; + }; + + led@23 { + reg = <23>; + function = LED_FUNCTION_POWER; + color = ; + led-gpio = <&ssogpio 23 0>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/leds/ti,tca6507.yaml b/Documentation/devicetree/bindings/leds/ti,tca6507.yaml index 94c307c98762..32c600387895 100644 --- a/Documentation/devicetree/bindings/leds/ti,tca6507.yaml +++ b/Documentation/devicetree/bindings/leds/ti,tca6507.yaml @@ -69,6 +69,7 @@ patternProperties: if: patternProperties: "^gpio@[0-6]$": + type: object properties: compatible: contains: diff --git a/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt b/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt index 5fe80c1c19fc..12371f5c6cd9 100644 --- a/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt +++ b/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt @@ -28,6 +28,9 @@ SoCs has each of these instances form a cluster and combine multiple clusters into a single IP block present within the Main NavSS. The interrupt lines from all these clusters are multiplexed and routed to different processor subsystems over a limited number of common interrupt output lines of an Interrupt Router. +The AM64x SoCS also uses a single IP block comprising of multiple clusters, +but the number of clusters are smaller, and the interrupt output lines are +connected directly to various processors. Mailbox Device Node: ==================== @@ -42,6 +45,7 @@ Required properties: "ti,omap4-mailbox" for OMAP44xx, OMAP54xx, AM33xx, AM43xx and DRA7xx SoCs "ti,am654-mailbox" for K3 AM65x and J721E SoCs + "ti,am64-mailbox" for K3 AM64x SoCs - reg: Contains the mailbox register address range (base address and length) - interrupts: Contains the interrupt information for the mailbox diff --git a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml index ffd09b664ff5..5dc1173d03fd 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml @@ -24,6 +24,7 @@ properties: - qcom,msm8998-apcs-hmss-global - qcom,qcs404-apcs-apps-global - qcom,sc7180-apss-shared + - qcom,sc8180x-apss-shared - qcom,sdm660-apcs-hmss-global - qcom,sdm845-apss-shared - qcom,sm8150-apss-shared @@ -33,9 +34,11 @@ properties: clocks: description: phandles to the parent clocks of the clock driver + minItems: 2 items: - description: primary pll parent of the clock driver - description: auxiliary parent + - description: reference clock '#mbox-cells': const: 1 @@ -44,9 +47,11 @@ properties: const: 0 clock-names: + minItems: 2 items: - const: pll - const: aux + - const: ref required: - compatible @@ -55,6 +60,35 @@ required: additionalProperties: false +allOf: + - if: + properties: + compatible: + enum: + - qcom,ipq6018-apcs-apps-global + - qcom,ipq8074-apcs-apps-global + - qcom,msm8916-apcs-kpss-global + - qcom,msm8994-apcs-kpss-global + - qcom,msm8996-apcs-hmss-global + - qcom,msm8998-apcs-hmss-global + - qcom,qcs404-apcs-apps-global + - qcom,sc7180-apss-shared + - qcom,sdm660-apcs-hmss-global + - qcom,sdm845-apss-shared + - qcom,sm8150-apss-shared + then: + properties: + clocks: + maxItems: 2 + - if: + properties: + compatible: + enum: + - qcom,sdx55-apcs-gcc + then: + properties: + clocks: + maxItems: 3 examples: # Example apcs with msm8996 diff --git a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml index 2f7058f7760c..c34303b87a5b 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml @@ -53,6 +53,7 @@ properties: maxItems: 1 memory-region: + maxItems: 1 description: CMA pool to use for buffers allocation instead of the default CMA pool. diff --git a/Documentation/devicetree/bindings/media/i2c/imx219.yaml b/Documentation/devicetree/bindings/media/i2c/imx219.yaml index 012c0565d8ae..5fc96944b448 100644 --- a/Documentation/devicetree/bindings/media/i2c/imx219.yaml +++ b/Documentation/devicetree/bindings/media/i2c/imx219.yaml @@ -40,6 +40,7 @@ properties: Digital core voltage supply, 1.2 volts reset-gpios: + maxItems: 1 description: |- Reference to the GPIO connected to the xclr pin, if any. Must be released (set high) after all supplies are applied. diff --git a/Documentation/devicetree/bindings/memory-controllers/exynos-srom.yaml b/Documentation/devicetree/bindings/memory-controllers/exynos-srom.yaml index 637e24f0f73b..c6e44f47ce7c 100644 --- a/Documentation/devicetree/bindings/memory-controllers/exynos-srom.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/exynos-srom.yaml @@ -28,6 +28,8 @@ properties: const: 1 ranges: + minItems: 1 + maxItems: 4 description: | Reflects the memory layout with four integer values per bank. Format: 0 diff --git a/Documentation/devicetree/bindings/mfd/bd9571mwv.txt b/Documentation/devicetree/bindings/mfd/bd9571mwv.txt index 8c4678650d1a..1d6413e96c37 100644 --- a/Documentation/devicetree/bindings/mfd/bd9571mwv.txt +++ b/Documentation/devicetree/bindings/mfd/bd9571mwv.txt @@ -1,7 +1,7 @@ -* ROHM BD9571MWV Power Management Integrated Circuit (PMIC) bindings +* ROHM BD9571MWV/BD9574MWF Power Management Integrated Circuit (PMIC) bindings Required properties: - - compatible : Should be "rohm,bd9571mwv". + - compatible : Should be "rohm,bd9571mwv" or "rohm,bd9574mwf". - reg : I2C slave address. - interrupts : The interrupt line the device is connected to. - interrupt-controller : Marks the device node as an interrupt controller. diff --git a/Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml b/Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml new file mode 100644 index 000000000000..c24ad45cabb5 --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/canaan,k210-sysctl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Canaan Kendryte K210 System Controller Device Tree Bindings + +maintainers: + - Damien Le Moal + +description: + Canaan Inc. Kendryte K210 SoC system controller which provides a + register map for controlling the clocks, reset signals and pin power + domains of the SoC. + +properties: + compatible: + items: + - const: canaan,k210-sysctl + - const: syscon + - const: simple-mfd + + clocks: + maxItems: 1 + description: + System controller Advanced Power Bus (APB) interface clock source. + + clock-names: + items: + - const: pclk + + reg: + maxItems: 1 + + clock-controller: + # Child node + type: object + $ref: "../clock/canaan,k210-clk.yaml" + description: + Clock controller for the SoC clocks. This child node definition + should follow the bindings specified in + Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml. + + reset-controller: + # Child node + type: object + $ref: "../reset/canaan,k210-rst.yaml" + description: + Reset controller for the SoC. This child node definition + should follow the bindings specified in + Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml. + + syscon-reboot: + # Child node + type: object + $ref: "../power/reset/syscon-reboot.yaml" + description: + Reboot method for the SoC. This child node definition + should follow the bindings specified in + Documentation/devicetree/bindings/power/reset/syscon-reboot.yaml. + +required: + - compatible + - clocks + - reg + - clock-controller + +additionalProperties: false + +examples: + - | + #include + #include + + clocks { + in0: oscllator { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <26000000>; + }; + }; + + sysctl: syscon@50440000 { + compatible = "canaan,k210-sysctl", + "syscon", "simple-mfd"; + reg = <0x50440000 0x100>; + clocks = <&sysclk K210_CLK_APB1>; + clock-names = "pclk"; + + sysclk: clock-controller { + #clock-cells = <1>; + compatible = "canaan,k210-clk"; + clocks = <&in0>; + }; + + sysrst: reset-controller { + compatible = "canaan,k210-rst"; + #reset-cells = <1>; + }; + + reboot: syscon-reboot { + compatible = "syscon-reboot"; + regmap = <&sysctl>; + offset = <48>; + mask = <1>; + value = <1>; + }; + }; diff --git a/Documentation/devicetree/bindings/mfd/ene-kb930.yaml b/Documentation/devicetree/bindings/mfd/ene-kb930.yaml new file mode 100644 index 000000000000..06ed9ec8f4bb --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/ene-kb930.yaml @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/ene-kb930.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ENE KB930 Embedded Controller bindings + +description: | + This binding describes the ENE KB930 Embedded Controller attached to an + I2C bus. + +maintainers: + - Dmitry Osipenko + +properties: + compatible: + items: + - enum: + - acer,a500-iconia-ec # Acer A500 Iconia tablet device + - const: ene,kb930 + reg: + maxItems: 1 + + monitored-battery: true + power-supplies: true + system-power-controller: true + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + battery: battery-cell { + compatible = "simple-battery"; + charge-full-design-microamp-hours = <3260000>; + energy-full-design-microwatt-hours = <24000000>; + operating-range-celsius = <0 40>; + }; + + mains: ac-adapter { + compatible = "gpio-charger"; + charger-type = "mains"; + gpios = <&gpio 125 0>; + }; + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + embedded-controller@58 { + compatible = "acer,a500-iconia-ec", "ene,kb930"; + reg = <0x58>; + + system-power-controller; + + monitored-battery = <&battery>; + power-supplies = <&mains>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml b/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml index d08e8fe76446..5a1e8d21f7a0 100644 --- a/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml +++ b/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml @@ -83,8 +83,9 @@ properties: 2 - scaled voltage based on an optional resistor divider and optional offset 3 - pre-scaled 16-bit voltage value + 4 - fan tach input to report RPM's $ref: /schemas/types.yaml#/definitions/uint32 - enum: [0, 1, 2, 3] + enum: [0, 1, 2, 3, 4] gw,voltage-divider-ohms: description: Values of resistors for divider on raw ADC input diff --git a/Documentation/devicetree/bindings/mfd/iqs62x.yaml b/Documentation/devicetree/bindings/mfd/iqs62x.yaml index 541b06d80e73..044cd7542c2b 100644 --- a/Documentation/devicetree/bindings/mfd/iqs62x.yaml +++ b/Documentation/devicetree/bindings/mfd/iqs62x.yaml @@ -93,7 +93,7 @@ examples: pwmleds { compatible = "pwm-leds"; - panel { + led-1 { pwms = <&iqs620a_pwm 0 1000000>; max-brightness = <255>; }; diff --git a/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt b/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt index a8ebb4621f79..7b636b7a8311 100644 --- a/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt +++ b/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt @@ -4,6 +4,7 @@ Required properties: - compatible : shall be one of: "atmel,at93c46d" "eeprom-93xx46" + "microchip,93lc46b" - data-size : number of data bits per word (either 8 or 16) Optional properties: diff --git a/Documentation/devicetree/bindings/misc/fsl,dpaa2-console.yaml b/Documentation/devicetree/bindings/misc/fsl,dpaa2-console.yaml index 271a3eafc054..8cc951feb7df 100644 --- a/Documentation/devicetree/bindings/misc/fsl,dpaa2-console.yaml +++ b/Documentation/devicetree/bindings/misc/fsl,dpaa2-console.yaml @@ -15,6 +15,7 @@ properties: const: "fsl,dpaa2-console" reg: + maxItems: 1 description: A standard property. Specifies the region where the MCFBA (MC firmware base address) register can be found. diff --git a/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml b/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml index e82c9a07b6fb..e75b3a8ba816 100644 --- a/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml +++ b/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml @@ -26,6 +26,8 @@ properties: - const: allwinner,sun9i-a80-mmc - const: allwinner,sun50i-a64-emmc - const: allwinner,sun50i-a64-mmc + - const: allwinner,sun50i-a100-emmc + - const: allwinner,sun50i-a100-mmc - items: - const: allwinner,sun8i-a83t-mmc - const: allwinner,sun7i-a20-mmc @@ -47,6 +49,12 @@ properties: - items: - const: allwinner,sun50i-h6-mmc - const: allwinner,sun50i-a64-mmc + - items: + - const: allwinner,sun50i-h616-emmc + - const: allwinner,sun50i-a100-emmc + - items: + - const: allwinner,sun50i-h616-mmc + - const: allwinner,sun50i-a100-mmc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml b/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml new file mode 100644 index 000000000000..47595cb483be --- /dev/null +++ b/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mmc/arm,pl18x.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ARM PrimeCell MultiMedia Card Interface (MMCI) PL180 and PL181 + +maintainers: + - Linus Walleij + - Ulf Hansson + +description: + The ARM PrimeCells MMCI PL180 and PL181 provides an interface for + reading and writing to MultiMedia and SD cards alike. Over the years + vendors have use the VHDL code from ARM to create derivative MMC/SD/SDIO + host controllers with very similar characteristics. + +allOf: + - $ref: /schemas/arm/primecell.yaml# + - $ref: mmc-controller.yaml# + +# We need a select here so we don't match all nodes with 'arm,primecell' +select: + properties: + compatible: + contains: + enum: + - arm,pl180 + - arm,pl181 + - arm,pl18x + required: + - compatible + +properties: + compatible: + oneOf: + - description: The first version of the block, simply called + PL180 and found in the ARM Integrator IM/PD1 logic module. + items: + - const: arm,pl180 + - const: arm,primecell + - description: The improved version of the block, found in the + ARM Versatile and later reference designs. Further revisions + exist but get detected at runtime by reading some magic numbers + in the PrimeCell ID registers. + items: + - const: arm,pl181 + - const: arm,primecell + - description: Wildcard entry that will let the operating system + inspect the PrimeCell ID registers to determine which hardware + variant of PL180 or PL181 this is. + items: + - const: arm,pl18x + - const: arm,primecell + + clocks: + description: One or two clocks, the "apb_pclk" and the "MCLK" + which is the core block clock. The names are not compulsory. + minItems: 1 + maxItems: 2 + + power-domains: true + + resets: + maxItems: 1 + + reg: + description: the MMIO memory window must be exactly 4KB (0x1000) and the + layout should provide the PrimeCell ID registers so that the device can + be discovered. On ST Micro variants, a second register window may be + defined if a delay block is present and used for tuning. + + interrupts: + description: The first interrupt is the command interrupt and corresponds + to the event at the end of a command. The second interrupt is the + PIO (polled I/O) interrupt and occurs when the FIFO needs to be + emptied as part of a bulk read from the card. Some variants have these + two interrupts wired into the same line (logic OR) and in that case + only one interrupt may be provided. + minItems: 1 + maxItems: 2 + + st,sig-dir-dat0: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, bus signal direction pins used for + DAT[0]. + + st,sig-dir-dat2: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, bus signal direction pins used for + DAT[2]. + + st,sig-dir-dat31: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, bus signal direction pins used for + DAT[3] and DAT[1]. + + st,sig-dir-dat74: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, bus signal direction pins used for + DAT[7] and DAT[4]. + + st,sig-dir-cmd: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, CMD signal direction used for + pin CMD. + + st,sig-pin-fbclk: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, feedback clock FBCLK signal pin + in use. + + st,sig-dir: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, signal direction polarity used for + pins CMD, DAT[0], DAT[1], DAT[2] and DAT[3]. + + st,neg-edge: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, data and command phase relation, + generated on the sd clock falling edge. + + st,use-ckin: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, use CKIN pin from an external + driver to sample the receive data (for example with a voltage switch + transceiver). + + st,cmd-gpios: + maxItems: 1 + description: + The GPIO matching the CMD pin. + + st,ck-gpios: + maxItems: 1 + description: + The GPIO matching the CK pin. + + st,ckin-gpios: + maxItems: 1 + description: + The GPIO matching the CKIN pin. + +dependencies: + st,cmd-gpios: [ "st,use-ckin" ] + st,ck-gpios: [ "st,use-ckin" ] + st,ckin-gpios: [ "st,use-ckin" ] + +unevaluatedProperties: false + +required: + - compatible + - reg + - interrupts + +examples: + - | + #include + #include + + mmc@5000 { + compatible = "arm,pl180", "arm,primecell"; + reg = <0x5000 0x1000>; + interrupts-extended = <&vic 22 &sic 1>; + clocks = <&xtal24mhz>, <&pclk>; + clock-names = "mclk", "apb_pclk"; + }; + + mmc@80126000 { + compatible = "arm,pl18x", "arm,primecell"; + reg = <0x80126000 0x1000>; + interrupts = <0 60 IRQ_TYPE_LEVEL_HIGH>; + dmas = <&dma 29 0 0x2>, <&dma 29 0 0x0>; + dma-names = "rx", "tx"; + clocks = <&prcc_kclk 1 5>, <&prcc_pclk 1 5>; + clock-names = "sdi", "apb_pclk"; + max-frequency = <100000000>; + bus-width = <4>; + cap-sd-highspeed; + cap-mmc-highspeed; + cd-gpios = <&gpio2 31 0x4>; + st,sig-dir-dat0; + st,sig-dir-dat2; + st,sig-dir-cmd; + st,sig-pin-fbclk; + vmmc-supply = <&ab8500_ldo_aux3_reg>; + vqmmc-supply = <&vmmci>; + }; + + mmc@101f6000 { + compatible = "arm,pl18x", "arm,primecell"; + reg = <0x101f6000 0x1000>; + clocks = <&sdiclk>, <&pclksdi>; + clock-names = "mclk", "apb_pclk"; + interrupt-parent = <&vica>; + interrupts = <22>; + max-frequency = <400000>; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + full-pwr-cycle; + st,sig-dir-dat0; + st,sig-dir-dat2; + st,sig-dir-dat31; + st,sig-dir-cmd; + st,sig-pin-fbclk; + vmmc-supply = <&vmmc_regulator>; + }; + + mmc@52007000 { + compatible = "arm,pl18x", "arm,primecell"; + arm,primecell-periphid = <0x10153180>; + reg = <0x52007000 0x1000>; + interrupts = <49>; + interrupt-names = "cmd_irq"; + clocks = <&rcc 0>; + clock-names = "apb_pclk"; + resets = <&rcc 1>; + cap-sd-highspeed; + cap-mmc-highspeed; + max-frequency = <120000000>; + }; diff --git a/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt b/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt index ed1456f5c94d..c51a62d751dc 100644 --- a/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt +++ b/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt @@ -12,6 +12,7 @@ Required Properties: - "marvell,armada-3700-sdhci": For controllers on Armada-3700 SoC. Must provide a second register area and marvell,pad-type. - "marvell,armada-ap806-sdhci": For controllers on Armada AP806. + - "marvell,armada-ap807-sdhci": For controllers on Armada AP807. - "marvell,armada-cp110-sdhci": For controllers on Armada CP110. - clocks: diff --git a/Documentation/devicetree/bindings/mmc/mmc-controller.yaml b/Documentation/devicetree/bindings/mmc/mmc-controller.yaml index e674bba52ee9..e141330c1114 100644 --- a/Documentation/devicetree/bindings/mmc/mmc-controller.yaml +++ b/Documentation/devicetree/bindings/mmc/mmc-controller.yaml @@ -40,6 +40,7 @@ properties: There is no card detection available; polling must be used. cd-gpios: + maxItems: 1 description: The card detection will be done using the GPIO provided. @@ -104,6 +105,7 @@ properties: line. Not used in combination with eMMC or SDIO. wp-gpios: + maxItems: 1 description: GPIO to use for the write-protect detection. diff --git a/Documentation/devicetree/bindings/mmc/mmci.txt b/Documentation/devicetree/bindings/mmc/mmci.txt deleted file mode 100644 index 4ec921e4bf34..000000000000 --- a/Documentation/devicetree/bindings/mmc/mmci.txt +++ /dev/null @@ -1,74 +0,0 @@ -* ARM PrimeCell MultiMedia Card Interface (MMCI) PL180/1 - -The ARM PrimeCell MMCI PL180 and PL181 provides an interface for -reading and writing to MultiMedia and SD cards alike. - -This file documents differences between the core properties described -by mmc.txt and the properties used by the mmci driver. Using "st" as -the prefix for a property, indicates support by the ST Micro variant. - -Required properties: -- compatible : contains "arm,pl18x", "arm,primecell". -- vmmc-supply : phandle to the regulator device tree node, mentioned - as the VCC/VDD supply in the eMMC/SD specs. - -Optional properties: -- arm,primecell-periphid : contains the PrimeCell Peripheral ID, it overrides - the ID provided by the HW -- resets : phandle to internal reset line. - Should be defined for sdmmc variant. -- vqmmc-supply : phandle to the regulator device tree node, mentioned - as the VCCQ/VDD_IO supply in the eMMC/SD specs. -specific for ux500 variant: -- st,sig-dir-dat0 : bus signal direction pin used for DAT[0]. -- st,sig-dir-dat2 : bus signal direction pin used for DAT[2]. -- st,sig-dir-dat31 : bus signal direction pin used for DAT[3] and DAT[1]. -- st,sig-dir-dat74 : bus signal direction pin used for DAT[4] to DAT[7]. -- st,sig-dir-cmd : cmd signal direction pin used for CMD. -- st,sig-pin-fbclk : feedback clock signal pin used. - -specific for sdmmc variant: -- reg : a second base register may be defined if a delay - block is present and used for tuning. -- st,sig-dir : signal direction polarity used for cmd, dat0 dat123. -- st,neg-edge : data & command phase relation, generated on - sd clock falling edge. -- st,use-ckin : use ckin pin from an external driver to sample - the receive data (example: with voltage - switch transceiver). - -Deprecated properties: -- mmc-cap-mmc-highspeed : indicates whether MMC is high speed capable. -- mmc-cap-sd-highspeed : indicates whether SD is high speed capable. - -Example: - -sdi0_per1@80126000 { - compatible = "arm,pl18x", "arm,primecell"; - reg = <0x80126000 0x1000>; - interrupts = <0 60 IRQ_TYPE_LEVEL_HIGH>; - - dmas = <&dma 29 0 0x2>, /* Logical - DevToMem */ - <&dma 29 0 0x0>; /* Logical - MemToDev */ - dma-names = "rx", "tx"; - - clocks = <&prcc_kclk 1 5>, <&prcc_pclk 1 5>; - clock-names = "sdi", "apb_pclk"; - - max-frequency = <100000000>; - bus-width = <4>; - cap-sd-highspeed; - cap-mmc-highspeed; - cd-gpios = <&gpio2 31 0x4>; // 95 - st,sig-dir-dat0; - st,sig-dir-dat2; - st,sig-dir-cmd; - st,sig-pin-fbclk; - - vmmc-supply = <&ab8500_ldo_aux3_reg>; - vqmmc-supply = <&vmmci>; - - pinctrl-names = "default", "sleep"; - pinctrl-0 = <&sdi0_default_mode>; - pinctrl-1 = <&sdi0_sleep_mode>; -}; diff --git a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml index 6bbf29b5c239..1118b6fa93c9 100644 --- a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml +++ b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml @@ -59,6 +59,7 @@ properties: - renesas,sdhi-r8a77980 # R-Car V3H - renesas,sdhi-r8a77990 # R-Car E3 - renesas,sdhi-r8a77995 # R-Car D3 + - renesas,sdhi-r8a779a0 # R-Car V3U - const: renesas,rcar-gen3-sdhi # R-Car Gen3 or RZ/G2 reg: @@ -123,7 +124,7 @@ required: if: properties: compatible: - items: + contains: enum: - renesas,sdhi-r7s72100 - renesas,sdhi-r7s9210 diff --git a/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml b/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml index 1ae945434c53..3a79e39253d2 100644 --- a/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml +++ b/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml @@ -15,12 +15,19 @@ allOf: properties: compatible: - enum: - - ti,am654-sdhci-5.1 - - ti,j721e-sdhci-8bit - - ti,j721e-sdhci-4bit - - ti,j7200-sdhci-8bit - - ti,j721e-sdhci-4bit + oneOf: + - const: ti,am654-sdhci-5.1 + - const: ti,j721e-sdhci-8bit + - const: ti,j721e-sdhci-4bit + - const: ti,j721e-sdhci-4bit + - const: ti,am64-sdhci-8bit + - const: ti,am64-sdhci-4bit + - items: + - const: ti,j7200-sdhci-8bit + - const: ti,j721e-sdhci-8bit + - items: + - const: ti,j7200-sdhci-4bit + - const: ti,j721e-sdhci-4bit reg: maxItems: 2 diff --git a/Documentation/devicetree/bindings/mmc/sdhci-msm.txt b/Documentation/devicetree/bindings/mmc/sdhci-msm.txt index 3b602fd6180b..4c7fa6a4ed15 100644 --- a/Documentation/devicetree/bindings/mmc/sdhci-msm.txt +++ b/Documentation/devicetree/bindings/mmc/sdhci-msm.txt @@ -17,10 +17,11 @@ Required properties: "qcom,msm8916-sdhci", "qcom,sdhci-msm-v4" "qcom,msm8992-sdhci", "qcom,sdhci-msm-v4" "qcom,msm8996-sdhci", "qcom,sdhci-msm-v4" - "qcom,sm8250-sdhci", "qcom,sdhci-msm-v5" - "qcom,sdm845-sdhci", "qcom,sdhci-msm-v5" "qcom,qcs404-sdhci", "qcom,sdhci-msm-v5" "qcom,sc7180-sdhci", "qcom,sdhci-msm-v5"; + "qcom,sdm845-sdhci", "qcom,sdhci-msm-v5" + "qcom,sdx55-sdhci", "qcom,sdhci-msm-v5"; + "qcom,sm8250-sdhci", "qcom,sdhci-msm-v5" NOTE that some old device tree files may be floating around that only have the string "qcom,sdhci-msm-v4" without the SoC compatible string but doing that should be considered a deprecated practice. @@ -30,10 +31,12 @@ Required properties: - SD Core register map (required for controllers earlier than msm-v5) - CQE register map (Optional, CQE support is present on SDHC instance meant for eMMC and version v4.2 and above) + - Inline Crypto Engine register map (optional) - reg-names: When CQE register map is supplied, below reg-names are required - "hc" for Host controller register map - "core" for SD core register map - "cqhci" for CQE register map + - "ice" for Inline Crypto Engine register map (optional) - interrupts: Should contain an interrupt-specifiers for the interrupts: - Host controller interrupt (required) - pinctrl-names: Should contain only one value - "default". @@ -46,6 +49,7 @@ Required properties: "xo" - TCXO clock (optional) "cal" - reference clock for RCLK delay calibration (optional) "sleep" - sleep clock for RCLK delay calibration (optional) + "ice" - clock for Inline Crypto Engine (optional) - qcom,ddr-config: Certain chipsets and platforms require particular settings for the DDR_CONFIG register. Use this field to specify the register diff --git a/Documentation/devicetree/bindings/mmc/sdhci-sirf.txt b/Documentation/devicetree/bindings/mmc/sdhci-sirf.txt deleted file mode 100644 index dd6ed464bcb8..000000000000 --- a/Documentation/devicetree/bindings/mmc/sdhci-sirf.txt +++ /dev/null @@ -1,18 +0,0 @@ -* SiRFprimII/marco/atlas6 SDHCI Controller - -This file documents differences between the core properties in mmc.txt -and the properties used by the sdhci-sirf driver. - -Required properties: -- compatible: sirf,prima2-sdhc - -Optional properties: -- cd-gpios: card detect gpio, with zero flags. - -Example: - - sd0: sdhci@56000000 { - compatible = "sirf,prima2-sdhc"; - reg = <0xcd000000 0x100000>; - cd-gpios = <&gpio 6 0>; - }; diff --git a/Documentation/devicetree/bindings/mmc/zx-dw-mshc.txt b/Documentation/devicetree/bindings/mmc/zx-dw-mshc.txt deleted file mode 100644 index 0f59bd5361f5..000000000000 --- a/Documentation/devicetree/bindings/mmc/zx-dw-mshc.txt +++ /dev/null @@ -1,31 +0,0 @@ -* ZTE specific extensions to the Synopsys Designware Mobile Storage - Host Controller - -The Synopsys designware mobile storage host controller is used to interface -a SoC with storage medium such as eMMC or SD/MMC cards. This file documents -differences between the core Synopsys dw mshc controller properties described -by synopsys-dw-mshc.txt and the properties used by the ZTE specific -extensions to the Synopsys Designware Mobile Storage Host Controller. - -Required Properties: - -* compatible: should be - - "zte,zx296718-dw-mshc": for ZX SoCs - -Example: - - mmc1: mmc@1110000 { - compatible = "zte,zx296718-dw-mshc"; - reg = <0x01110000 0x1000>; - interrupts = ; - fifo-depth = <32>; - data-addr = <0x200>; - fifo-watermark-aligned; - bus-width = <4>; - clock-frequency = <50000000>; - clocks = <&topcrm SD0_AHB>, <&topcrm SD0_WCLK>; - clock-names = "biu", "ciu"; - max-frequency = <50000000>; - cap-sdio-irq; - cap-sd-highspeed; - }; diff --git a/Documentation/devicetree/bindings/mtd/jedec,spi-nor.txt b/Documentation/devicetree/bindings/mtd/jedec,spi-nor.txt deleted file mode 100644 index f03be904d3c2..000000000000 --- a/Documentation/devicetree/bindings/mtd/jedec,spi-nor.txt +++ /dev/null @@ -1,91 +0,0 @@ -* SPI NOR flash: ST M25Pxx (and similar) serial flash chips - -Required properties: -- #address-cells, #size-cells : Must be present if the device has sub-nodes - representing partitions. -- compatible : May include a device-specific string consisting of the - manufacturer and name of the chip. A list of supported chip - names follows. - Must also include "jedec,spi-nor" for any SPI NOR flash that can - be identified by the JEDEC READ ID opcode (0x9F). - - Supported chip names: - at25df321a - at25df641 - at26df081a - mr25h128 - mr25h256 - mr25h10 - mr25h40 - mx25l4005a - mx25l1606e - mx25l6405d - mx25l12805d - mx25l25635e - n25q064 - n25q128a11 - n25q128a13 - n25q512a - s25fl256s1 - s25fl512s - s25sl12801 - s25fl008k - s25fl064k - sst25vf040b - m25p40 - m25p80 - m25p16 - m25p32 - m25p64 - m25p128 - w25x80 - w25x32 - w25q32 - w25q64 - w25q32dw - w25q80bl - w25q128 - w25q256 - - The following chip names have been used historically to - designate quirky versions of flash chips that do not support the - JEDEC READ ID opcode (0x9F): - m25p05-nonjedec - m25p10-nonjedec - m25p20-nonjedec - m25p40-nonjedec - m25p80-nonjedec - m25p16-nonjedec - m25p32-nonjedec - m25p64-nonjedec - m25p128-nonjedec - -- reg : Chip-Select number -- spi-max-frequency : Maximum frequency of the SPI bus the chip can operate at - -Optional properties: -- m25p,fast-read : Use the "fast read" opcode to read data from the chip instead - of the usual "read" opcode. This opcode is not supported by - all chips and support for it can not be detected at runtime. - Refer to your chips' datasheet to check if this is supported - by your chip. -- broken-flash-reset : Some flash devices utilize stateful addressing modes - (e.g., for 32-bit addressing) which need to be managed - carefully by a system. Because these sorts of flash don't - have a standardized software reset command, and because some - systems don't toggle the flash RESET# pin upon system reset - (if the pin even exists at all), there are systems which - cannot reboot properly if the flash is left in the "wrong" - state. This boolean flag can be used on such systems, to - denote the absence of a reliable reset mechanism. - -Example: - - flash: m25p80@0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "spansion,m25p80", "jedec,spi-nor"; - reg = <0>; - spi-max-frequency = <40000000>; - m25p,fast-read; - }; diff --git a/Documentation/devicetree/bindings/mtd/jedec,spi-nor.yaml b/Documentation/devicetree/bindings/mtd/jedec,spi-nor.yaml new file mode 100644 index 000000000000..5e7e5349f9a1 --- /dev/null +++ b/Documentation/devicetree/bindings/mtd/jedec,spi-nor.yaml @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mtd/jedec,spi-nor.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: SPI NOR flash ST M25Pxx (and similar) serial flash chips + +maintainers: + - Rob Herring + +properties: + compatible: + oneOf: + - items: + - pattern: "^((((micron|spansion|st),)?\ + (m25p(40|80|16|32|64|128)|\ + n25q(32b|064|128a11|128a13|256a|512a|164k)))|\ + atmel,at25df(321a|641|081a)|\ + everspin,mr25h(10|40|128|256)|\ + (mxicy|macronix),mx25l(4005a|1606e|6405d|8005|12805d|25635e)|\ + (mxicy|macronix),mx25u(4033|4035)|\ + (spansion,)?s25fl(128s|256s1|512s|008k|064k|164k)|\ + (sst|microchip),sst25vf(016b|032b|040b)|\ + (sst,)?sst26wf016b|\ + (sst,)?sst25wf(040b|080)|\ + winbond,w25x(80|32)|\ + (winbond,)?w25q(16|32(w|dw)?|64(dw)?|80bl|128(fw)?|256))$" + - const: jedec,spi-nor + - items: + - enum: + - issi,is25lp016d + - micron,mt25qu02g + - mxicy,mx25r1635f + - mxicy,mx25u6435f + - mxicy,mx25v8035f + - spansion,s25sl12801 + - spansion,s25fs512s + - const: jedec,spi-nor + - const: jedec,spi-nor + description: + Must also include "jedec,spi-nor" for any SPI NOR flash that can be + identified by the JEDEC READ ID opcode (0x9F). + + reg: + maxItems: 1 + + spi-max-frequency: true + spi-rx-bus-width: true + spi-tx-bus-width: true + + m25p,fast-read: + type: boolean + description: + Use the "fast read" opcode to read data from the chip instead of the usual + "read" opcode. This opcode is not supported by all chips and support for + it can not be detected at runtime. Refer to your chips' datasheet to check + if this is supported by your chip. + + broken-flash-reset: + type: boolean + description: + Some flash devices utilize stateful addressing modes (e.g., for 32-bit + addressing) which need to be managed carefully by a system. Because these + sorts of flash don't have a standardized software reset command, and + because some systems don't toggle the flash RESET# pin upon system reset + (if the pin even exists at all), there are systems which cannot reboot + properly if the flash is left in the "wrong" state. This boolean flag can + be used on such systems, to denote the absence of a reliable reset + mechanism. + + label: true + + partitions: + type: object + + '#address-cells': true + '#size-cells': true + +patternProperties: + # Note: use 'partitions' node for new users + '^partition@': + type: object + +additionalProperties: false + +examples: + - | + spi { + #address-cells = <1>; + #size-cells = <0>; + + flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "spansion,m25p80", "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <40000000>; + m25p,fast-read; + }; + }; +... diff --git a/Documentation/devicetree/bindings/net/can/rcar_canfd.txt b/Documentation/devicetree/bindings/net/can/rcar_canfd.txt index 22cf2a889b2c..248c4ed97a0a 100644 --- a/Documentation/devicetree/bindings/net/can/rcar_canfd.txt +++ b/Documentation/devicetree/bindings/net/can/rcar_canfd.txt @@ -97,7 +97,7 @@ E.g. below enables Channel 0 alone in the board using External clock as fCAN clock. &canfd { - pinctrl-0 = <&canfd0_pins &can_clk_pins>; + pinctrl-0 = <&canfd0_pins>, <&can_clk_pins>; pinctrl-names = "default"; status = "okay"; diff --git a/Documentation/devicetree/bindings/net/ethernet-controller.yaml b/Documentation/devicetree/bindings/net/ethernet-controller.yaml index f599c1d9c961..4b7d1e5d003c 100644 --- a/Documentation/devicetree/bindings/net/ethernet-controller.yaml +++ b/Documentation/devicetree/bindings/net/ethernet-controller.yaml @@ -206,6 +206,11 @@ properties: Indicates that full-duplex is used. When absent, half duplex is assumed. + pause: + $ref: /schemas/types.yaml#definitions/flag + description: + Indicates that pause should be enabled. + asym-pause: $ref: /schemas/types.yaml#/definitions/flag description: diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml index 3fae9a5f0c6a..783b9e32cf66 100644 --- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml +++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml @@ -72,6 +72,7 @@ properties: dma-coherent: true clocks: + maxItems: 1 description: CPSWxG NUSS functional clock clock-names: diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml index ce43a1c58a57..4317eba503ca 100644 --- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml +++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml @@ -59,6 +59,7 @@ properties: - const: cpts clocks: + maxItems: 1 description: CPTS reference clock clock-names: diff --git a/Documentation/devicetree/bindings/nvmem/rmem.yaml b/Documentation/devicetree/bindings/nvmem/rmem.yaml new file mode 100644 index 000000000000..1d85a0a30846 --- /dev/null +++ b/Documentation/devicetree/bindings/nvmem/rmem.yaml @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/nvmem/rmem.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Reserved Memory Based nvmem Device + +maintainers: + - Nicolas Saenz Julienne + +allOf: + - $ref: "nvmem.yaml#" + +properties: + compatible: + items: + - enum: + - raspberrypi,bootloader-config + - const: nvmem-rmem + + no-map: + $ref: /schemas/types.yaml#/definitions/flag + description: + Avoid creating a virtual mapping of the region as part of the OS' + standard mapping of system memory. + +required: + - compatible + - no-map + +unevaluatedProperties: false + +examples: + - | + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + + blconfig: nvram@10000000 { + compatible = "raspberrypi,bootloader-config", "nvmem-rmem"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x10000000 0x1000>; + no-map; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml index 807694b4f41f..f90557f6deb8 100644 --- a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml @@ -14,6 +14,7 @@ properties: items: - enum: - brcm,bcm2711-pcie # The Raspberry Pi 4 + - brcm,bcm4908-pcie - brcm,bcm7211-pcie # Broadcom STB version of RPi4 - brcm,bcm7278-pcie # Broadcom 7278 Arm - brcm,bcm7216-pcie # Broadcom 7216 Arm @@ -63,15 +64,6 @@ properties: aspm-no-l0s: true - resets: - description: for "brcm,bcm7216-pcie", must be a valid reset - phandle pointing to the RESCAL reset controller provider node. - $ref: "/schemas/types.yaml#/definitions/phandle" - - reset-names: - items: - - const: rescal - brcm,scb-sizes: description: u64 giving the 64bit PCIe memory viewport size of a memory controller. There may be up to @@ -98,12 +90,39 @@ required: allOf: - $ref: /schemas/pci/pci-bus.yaml# + - if: + properties: + compatible: + contains: + const: brcm,bcm4908-pcie + then: + properties: + resets: + items: + - description: reset controller handling the PERST# signal + + reset-names: + items: + - const: perst + + required: + - resets + - reset-names - if: properties: compatible: contains: const: brcm,bcm7216-pcie then: + properties: + resets: + items: + - description: phandle pointing to the RESCAL reset controller + + reset-names: + items: + - const: rescal + required: - resets - reset-names diff --git a/Documentation/devicetree/bindings/pci/layerscape-pci.txt b/Documentation/devicetree/bindings/pci/layerscape-pci.txt index daa99f7d4c3f..6d898dd4a8e2 100644 --- a/Documentation/devicetree/bindings/pci/layerscape-pci.txt +++ b/Documentation/devicetree/bindings/pci/layerscape-pci.txt @@ -26,6 +26,7 @@ Required properties: "fsl,ls1046a-pcie-ep", "fsl,ls-pcie-ep" "fsl,ls1088a-pcie-ep", "fsl,ls-pcie-ep" "fsl,ls2088a-pcie-ep", "fsl,ls-pcie-ep" + "fsl,lx2160ar2-pcie-ep", "fsl,ls-pcie-ep" - reg: base addresses and lengths of the PCIe controller register blocks. - interrupts: A list of interrupt outputs of the controller. Must contain an entry for each entry in the interrupt-names property. diff --git a/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml new file mode 100644 index 000000000000..04251d71f56b --- /dev/null +++ b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pci/microchip,pcie-host.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip PCIe Root Port Bridge Controller Device Tree Bindings + +maintainers: + - Daire McNamara + +allOf: + - $ref: /schemas/pci/pci-bus.yaml# + +properties: + compatible: + const: microchip,pcie-host-1.0 # PolarFire + + reg: + maxItems: 2 + + reg-names: + items: + - const: cfg + - const: apb + + interrupts: + minItems: 1 + maxItems: 2 + items: + - description: PCIe host controller + - description: builtin MSI controller + + interrupt-names: + minItems: 1 + maxItems: 2 + items: + - const: pcie + - const: msi + + ranges: + maxItems: 1 + + msi-controller: + description: Identifies the node as an MSI controller. + + msi-parent: + description: MSI controller the device is capable of using. + +required: + - reg + - reg-names + - "#interrupt-cells" + - interrupts + - interrupt-map-mask + - interrupt-map + - msi-controller + +unevaluatedProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + pcie0: pcie@2030000000 { + compatible = "microchip,pcie-host-1.0"; + reg = <0x0 0x70000000 0x0 0x08000000>, + <0x0 0x43000000 0x0 0x00010000>; + reg-names = "cfg", "apb"; + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + interrupts = <119>; + interrupt-map-mask = <0x0 0x0 0x0 0x7>; + interrupt-map = <0 0 0 1 &pcie_intc0 0>, + <0 0 0 2 &pcie_intc0 1>, + <0 0 0 3 &pcie_intc0 2>, + <0 0 0 4 &pcie_intc0 3>; + interrupt-parent = <&plic0>; + msi-parent = <&pcie0>; + msi-controller; + bus-range = <0x00 0x7f>; + ranges = <0x03000000 0x0 0x78000000 0x0 0x78000000 0x0 0x04000000>; + pcie_intc0: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie.txt b/Documentation/devicetree/bindings/pci/qcom,pcie.txt index 3b55310390a0..0da458a051b6 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie.txt +++ b/Documentation/devicetree/bindings/pci/qcom,pcie.txt @@ -132,8 +132,8 @@ - "master_bus" AXI Master clock - "slave_bus" AXI Slave clock --clock-names: - Usage: required for sdm845 and sm8250 +- clock-names: + Usage: required for sdm845 Value type: Definition: Should contain the following entries - "aux" Auxiliary clock @@ -144,6 +144,19 @@ - "tbu" PCIe TBU clock - "pipe" PIPE clock +- clock-names: + Usage: required for sm8250 + Value type: + Definition: Should contain the following entries + - "aux" Auxiliary clock + - "cfg" Configuration clock + - "bus_master" Master AXI clock + - "bus_slave" Slave AXI clock + - "slave_q2a" Slave Q2A clock + - "tbu" PCIe TBU clock + - "ddrss_sf_tbu" PCIe SF TBU clock + - "pipe" PIPE clock + - resets: Usage: required Value type: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml index 94ac23687b7e..77606c899fe2 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml @@ -51,9 +51,11 @@ properties: - const: usb2_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml index fd6e126fcf18..078af52b16ed 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml @@ -50,9 +50,11 @@ properties: - const: usb1_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml index 7670411002c9..e632140722a2 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml @@ -50,9 +50,11 @@ properties: - const: usb3_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml index 9b319381d1ad..5bad9b06e2e7 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml @@ -45,9 +45,11 @@ properties: - const: usb1_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml index b0ed01bbf3db..922b4665e00d 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml @@ -54,9 +54,11 @@ properties: - const: usb2_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml index b0674406f8aa..a94019efc2f3 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml @@ -50,9 +50,11 @@ properties: - const: usb1_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml index 48dc9c834a9b..33f3ddc0492d 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml @@ -56,9 +56,11 @@ properties: - const: usb2_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml index 60c344585276..f80431060803 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml @@ -62,9 +62,11 @@ properties: - const: usb3_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml index a2bb36790fbd..d947e50a49d2 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml @@ -56,9 +56,11 @@ properties: - const: usb2_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml index eadfd0c9493c..a2836c296cc4 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml @@ -42,9 +42,11 @@ properties: const: usb0_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml index ded7d6f0a119..2eb493fa64fd 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml @@ -22,7 +22,8 @@ properties: clocks: anyOf: - - description: Main PHY Clock + - maxItems: 1 + description: Main PHY Clock - items: - description: Main PHY clock @@ -39,20 +40,16 @@ properties: - const: hsic_480M resets: - anyOf: + minItems: 1 + items: - description: Normal USB PHY reset - - - items: - - description: Normal USB PHY reset - - description: HSIC Reset + - description: HSIC Reset reset-names: - oneOf: + minItems: 1 + items: - const: phy - - - items: - - const: phy - - const: hsic + - const: hsic phy_type: const: hsic diff --git a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.txt b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.txt deleted file mode 100644 index 698aacbdcfc4..000000000000 --- a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.txt +++ /dev/null @@ -1,86 +0,0 @@ -Broadcom STB USB PHY - -Required properties: -- compatible: should be one of - "brcm,brcmstb-usb-phy" - "brcm,bcm7216-usb-phy" - "brcm,bcm7211-usb-phy" - -- reg and reg-names properties requirements are specific to the - compatible string. - "brcm,brcmstb-usb-phy": - - reg: 1 or 2 offset and length pairs. One for the base CTRL registers - and an optional pair for systems with USB 3.x support - - reg-names: not specified - "brcm,bcm7216-usb-phy": - - reg: 3 offset and length pairs for CTRL, XHCI_EC and XHCI_GBL - registers - - reg-names: "ctrl", "xhci_ec", "xhci_gbl" - "brcm,bcm7211-usb-phy": - - reg: 5 offset and length pairs for CTRL, XHCI_EC, XHCI_GBL, - USB_PHY and USB_MDIO registers and an optional pair - for the BDC registers - - reg-names: "ctrl", "xhci_ec", "xhci_gbl", "usb_phy", "usb_mdio", "bdc_ec" - -- #phy-cells: Shall be 1 as it expects one argument for setting - the type of the PHY. Possible values are: - - PHY_TYPE_USB2 for USB1.1/2.0 PHY - - PHY_TYPE_USB3 for USB3.x PHY - -Optional Properties: -- clocks : clock phandles. -- clock-names: String, clock name. -- interrupts: wakeup interrupt -- interrupt-names: "wakeup" -- brcm,ipp: Boolean, Invert Port Power. - Possible values are: 0 (Don't invert), 1 (Invert) -- brcm,ioc: Boolean, Invert Over Current detection. - Possible values are: 0 (Don't invert), 1 (Invert) -- dr_mode: String, PHY Device mode. - Possible values are: "host", "peripheral ", "drd" or "typec-pd" - If this property is not defined, the phy will default to "host" mode. -- brcm,syscon-piarbctl: phandle to syscon for handling config registers -NOTE: one or both of the following two properties must be set -- brcm,has-xhci: Boolean indicating the phy has an XHCI phy. -- brcm,has-eohci: Boolean indicating the phy has an EHCI/OHCI phy. - - -Example: - -usbphy_0: usb-phy@f0470200 { - reg = <0xf0470200 0xb8>, - <0xf0471940 0x6c0>; - compatible = "brcm,brcmstb-usb-phy"; - #phy-cells = <1>; - dr_mode = "host" - brcm,ioc = <1>; - brcm,ipp = <1>; - brcm,has-xhci; - brcm,has-eohci; - clocks = <&usb20>, <&usb30>; - clock-names = "sw_usb", "sw_usb3"; -}; - -usb-phy@29f0200 { - reg = <0x29f0200 0x200>, - <0x29c0880 0x30>, - <0x29cc100 0x534>, - <0x2808000 0x24>, - <0x2980080 0x8>; - reg-names = "ctrl", - "xhci_ec", - "xhci_gbl", - "usb_phy", - "usb_mdio"; - brcm,ioc = <0x0>; - brcm,ipp = <0x0>; - compatible = "brcm,bcm7211-usb-phy"; - interrupts = <0x30>; - interrupt-parent = <&vpu_intr1_nosec_intc>; - interrupt-names = "wake"; - #phy-cells = <0x1>; - brcm,has-xhci; - syscon-piarbctl = <&syscon_piarbctl>; - clocks = <&scmi_clk 256>; - clock-names = "sw_usb"; -}; diff --git a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml new file mode 100644 index 000000000000..0497368d1fca --- /dev/null +++ b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/brcm,brcmstb-usb-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom STB USB PHY + +description: Broadcom's PHY that handles EHCI/OHCI and/or XHCI + +maintainers: + - Al Cooper + - RafaÅ‚ MiÅ‚ecki + +properties: + compatible: + enum: + - brcm,bcm4908-usb-phy + - brcm,bcm7211-usb-phy + - brcm,bcm7216-usb-phy + - brcm,brcmstb-usb-phy + + reg: + minItems: 1 + maxItems: 6 + items: + - description: the base CTRL register + - description: XHCI EC register + - description: XHCI GBL register + - description: USB PHY register + - description: USB MDIO register + - description: BDC register + + reg-names: + minItems: 1 + maxItems: 6 + items: + - const: ctrl + - const: xhci_ec + - const: xhci_gbl + - const: usb_phy + - const: usb_mdio + - const: bdc_ec + + clocks: + minItems: 1 + maxItems: 2 + + clock-names: + minItems: 1 + maxItems: 2 + items: + - const: sw_usb + - const: sw_usb3 + + interrupts: + description: wakeup interrupt + + interrupt-names: + const: wake + + brcm,ipp: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Invert Port Power + minimum: 0 + maximum: 1 + + brcm,ioc: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Invert Over Current detection + minimum: 0 + maximum: 1 + + dr_mode: + description: PHY Device mode. If this property is not defined, the PHY will + default to "host" mode. + enum: + - host + - peripheral + - drd + - typec-pd + + brcm,syscon-piarbctl: + description: phandle to syscon for handling config registers + $ref: /schemas/types.yaml#/definitions/phandle + + brcm,has-xhci: + description: Indicates the PHY has an XHCI PHY. + type: boolean + + brcm,has-eohci: + description: Indicates the PHY has an EHCI/OHCI PHY. + type: boolean + + "#phy-cells": + description: | + Cell allows setting the type of the PHY. Possible values are: + - PHY_TYPE_USB2 for USB1.1/2.0 PHY + - PHY_TYPE_USB3 for USB3.x PHY + const: 1 + +required: + - reg + - "#phy-cells" + +anyOf: + - required: + - brcm,has-xhci + - required: + - brcm,has-eohci + +allOf: + - if: + properties: + compatible: + contains: + enum: + - const: brcm,bcm4908-usb-phy + - const: brcm,brcmstb-usb-phy + then: + properties: + reg: + minItems: 1 + maxItems: 2 + - if: + properties: + compatible: + contains: + const: brcm,bcm7211-usb-phy + then: + properties: + reg: + minItems: 5 + maxItems: 6 + reg-names: + minItems: 5 + maxItems: 6 + - if: + properties: + compatible: + contains: + const: brcm,bcm7216-usb-phy + then: + properties: + reg: + minItems: 3 + maxItems: 3 + reg-names: + minItems: 3 + maxItems: 3 + +additionalProperties: false + +examples: + - | + #include + + usb-phy@f0470200 { + compatible = "brcm,brcmstb-usb-phy"; + reg = <0xf0470200 0xb8>, + <0xf0471940 0x6c0>; + #phy-cells = <1>; + dr_mode = "host"; + brcm,ioc = <1>; + brcm,ipp = <1>; + brcm,has-xhci; + brcm,has-eohci; + clocks = <&usb20>, <&usb30>; + clock-names = "sw_usb", "sw_usb3"; + }; + - | + #include + + usb-phy@29f0200 { + compatible = "brcm,bcm7211-usb-phy"; + reg = <0x29f0200 0x200>, + <0x29c0880 0x30>, + <0x29cc100 0x534>, + <0x2808000 0x24>, + <0x2980080 0x8>; + reg-names = "ctrl", + "xhci_ec", + "xhci_gbl", + "usb_phy", + "usb_mdio"; + brcm,ioc = <0x0>; + brcm,ipp = <0x0>; + interrupts = <0x30>; + interrupt-parent = <&vpu_intr1_nosec_intc>; + interrupt-names = "wake"; + #phy-cells = <0x1>; + brcm,has-xhci; + brcm,syscon-piarbctl = <&syscon_piarbctl>; + clocks = <&scmi_clk 256>; + clock-names = "sw_usb"; + }; diff --git a/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml b/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml index 58c3ef8004ad..04edda504ab6 100644 --- a/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml +++ b/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml @@ -99,8 +99,7 @@ patternProperties: if: properties: compatible: - items: - const: brcm,iproc-ns2-sata-phy + const: brcm,iproc-ns2-sata-phy then: properties: reg: diff --git a/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml new file mode 100644 index 000000000000..71d4acea1f66 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,dsi-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek MIPI Display Serial Interface (DSI) PHY binding + +maintainers: + - Chun-Kuang Hu + - Philipp Zabel + - Chunfeng Yun + +description: The MIPI DSI PHY supports up to 4-lane output. + +properties: + $nodename: + pattern: "^dsi-phy@[0-9a-f]+$" + + compatible: + enum: + - mediatek,mt2701-mipi-tx + - mediatek,mt7623-mipi-tx + - mediatek,mt8173-mipi-tx + - mediatek,mt8183-mipi-tx + + reg: + maxItems: 1 + + clocks: + items: + - description: PLL reference clock + + clock-output-names: + maxItems: 1 + + "#phy-cells": + const: 0 + + "#clock-cells": + const: 0 + + nvmem-cells: + maxItems: 1 + description: A phandle to the calibration data provided by a nvmem device, + if unspecified, default values shall be used. + + nvmem-cell-names: + items: + - const: calibration-data + + drive-strength-microamp: + description: adjust driving current + multipleOf: 200 + minimum: 2000 + maximum: 6000 + default: 4600 + +required: + - compatible + - reg + - clocks + - clock-output-names + - "#phy-cells" + - "#clock-cells" + +additionalProperties: false + +examples: + - | + #include + dsi-phy@10215000 { + compatible = "mediatek,mt8173-mipi-tx"; + reg = <0x10215000 0x1000>; + clocks = <&clk26m>; + clock-output-names = "mipi_tx0_pll"; + drive-strength-microamp = <4000>; + nvmem-cells= <&mipi_tx_calibration>; + nvmem-cell-names = "calibration-data"; + #clock-cells = <0>; + #phy-cells = <0>; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/mediatek,hdmi-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,hdmi-phy.yaml new file mode 100644 index 000000000000..4752517a1446 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,hdmi-phy.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,hdmi-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek High Definition Multimedia Interface (HDMI) PHY binding + +maintainers: + - Chun-Kuang Hu + - Philipp Zabel + - Chunfeng Yun + +description: | + The HDMI PHY serializes the HDMI encoder's three channel 10-bit parallel + output and drives the HDMI pads. + +properties: + $nodename: + pattern: "^hdmi-phy@[0-9a-f]+$" + + compatible: + enum: + - mediatek,mt2701-hdmi-phy + - mediatek,mt7623-hdmi-phy + - mediatek,mt8173-hdmi-phy + + reg: + maxItems: 1 + + clocks: + items: + - description: PLL reference clock + + clock-names: + items: + - const: pll_ref + + clock-output-names: + items: + - const: hdmitx_dig_cts + + "#phy-cells": + const: 0 + + "#clock-cells": + const: 0 + + mediatek,ibias: + description: + TX DRV bias current for < 1.65Gbps + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 63 + default: 0xa + + mediatek,ibias_up: + description: + TX DRV bias current for >= 1.65Gbps + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 63 + default: 0x1c + +required: + - compatible + - reg + - clocks + - clock-names + - clock-output-names + - "#phy-cells" + - "#clock-cells" + +additionalProperties: false + +examples: + - | + #include + hdmi_phy: hdmi-phy@10209100 { + compatible = "mediatek,mt8173-hdmi-phy"; + reg = <0x10209100 0x24>; + clocks = <&apmixedsys CLK_APMIXED_HDMI_REF>; + clock-names = "pll_ref"; + clock-output-names = "hdmitx_dig_cts"; + mediatek,ibias = <0xa>; + mediatek,ibias_up = <0x1c>; + #clock-cells = <0>; + #phy-cells = <0>; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml b/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml new file mode 100644 index 000000000000..602e6ff45785 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml @@ -0,0 +1,260 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,tphy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek T-PHY Controller Device Tree Bindings + +maintainers: + - Chunfeng Yun + +description: | + The T-PHY controller supports physical layer functionality for a number of + controllers on MediaTek SoCs, includes USB2.0, USB3.0, PCIe and SATA. + + Layout differences of banks between T-PHY V1 (mt8173/mt2701) and + T-PHY V2 (mt2712) when works on USB mode: + ----------------------------------- + Version 1: + port offset bank + shared 0x0000 SPLLC + 0x0100 FMREG + u2 port0 0x0800 U2PHY_COM + u3 port0 0x0900 U3PHYD + 0x0a00 U3PHYD_BANK2 + 0x0b00 U3PHYA + 0x0c00 U3PHYA_DA + u2 port1 0x1000 U2PHY_COM + u3 port1 0x1100 U3PHYD + 0x1200 U3PHYD_BANK2 + 0x1300 U3PHYA + 0x1400 U3PHYA_DA + u2 port2 0x1800 U2PHY_COM + ... + + Version 2: + port offset bank + u2 port0 0x0000 MISC + 0x0100 FMREG + 0x0300 U2PHY_COM + u3 port0 0x0700 SPLLC + 0x0800 CHIP + 0x0900 U3PHYD + 0x0a00 U3PHYD_BANK2 + 0x0b00 U3PHYA + 0x0c00 U3PHYA_DA + u2 port1 0x1000 MISC + 0x1100 FMREG + 0x1300 U2PHY_COM + u3 port1 0x1700 SPLLC + 0x1800 CHIP + 0x1900 U3PHYD + 0x1a00 U3PHYD_BANK2 + 0x1b00 U3PHYA + 0x1c00 U3PHYA_DA + u2 port2 0x2000 MISC + ... + + SPLLC shared by u3 ports and FMREG shared by u2 ports on V1 are put back + into each port; a new bank MISC for u2 ports and CHIP for u3 ports are + added on V2. + +properties: + $nodename: + pattern: "^t-phy@[0-9a-f]+$" + + compatible: + oneOf: + - items: + - enum: + - mediatek,mt2701-tphy + - mediatek,mt7623-tphy + - mediatek,mt7622-tphy + - mediatek,mt8516-tphy + - const: mediatek,generic-tphy-v1 + - items: + - enum: + - mediatek,mt2712-tphy + - mediatek,mt7629-tphy + - mediatek,mt8183-tphy + - const: mediatek,generic-tphy-v2 + - const: mediatek,mt2701-u3phy + deprecated: true + - const: mediatek,mt2712-u3phy + deprecated: true + - const: mediatek,mt8173-u3phy + + reg: + description: + Register shared by multiple ports, exclude port's private register. + It is needed for T-PHY V1, such as mt2701 and mt8173, but not for + T-PHY V2, such as mt2712. + maxItems: 1 + + "#address-cells": + enum: [1, 2] + + "#size-cells": + enum: [1, 2] + + # Used with non-empty value if optional 'reg' is not provided. + # The format of the value is an arbitrary number of triplets of + # (child-bus-address, parent-bus-address, length). + ranges: true + + mediatek,src-ref-clk-mhz: + description: + Frequency of reference clock for slew rate calibrate + default: 26 + + mediatek,src-coef: + description: + Coefficient for slew rate calibrate, depends on SoC process + $ref: /schemas/types.yaml#/definitions/uint32 + default: 28 + +# Required child node: +patternProperties: + "^usb-phy@[0-9a-f]+$": + type: object + description: + A sub-node is required for each port the controller provides. + Address range information including the usual 'reg' property + is used inside these nodes to describe the controller's topology. + + properties: + reg: + maxItems: 1 + + clocks: + minItems: 1 + maxItems: 2 + items: + - description: Reference clock, (HS is 48Mhz, SS/P is 24~27Mhz) + - description: Reference clock of analog phy + description: + Uses both clocks if the clock of analog and digital phys are + separated, otherwise uses "ref" clock only if needed. + + clock-names: + minItems: 1 + maxItems: 2 + items: + - const: ref + - const: da_ref + + "#phy-cells": + const: 1 + description: | + The cells contain the following arguments. + + - description: The PHY type + enum: + - PHY_TYPE_USB2 + - PHY_TYPE_USB3 + - PHY_TYPE_PCIE + - PHY_TYPE_SATA + + # The following optional vendor properties are only for debug or HQA test + mediatek,eye-src: + description: + The value of slew rate calibrate (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-vrt: + description: + The selection of VRT reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-term: + description: + The selection of HS_TX TERM reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,intr: + description: + The selection of internal resistor (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 31 + + mediatek,discth: + description: + The selection of disconnect threshold (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 15 + + mediatek,bc12: + description: + Specify the flag to enable BC1.2 if support it + type: boolean + + required: + - reg + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - "#address-cells" + - "#size-cells" + - ranges + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + usb@11271000 { + compatible = "mediatek,mt8173-mtu3", "mediatek,mtu3"; + reg = <0x11271000 0x3000>, <0x11280700 0x0100>; + reg-names = "mac", "ippc"; + phys = <&u2port0 PHY_TYPE_USB2>, + <&u3port0 PHY_TYPE_USB3>, + <&u2port1 PHY_TYPE_USB2>; + interrupts = ; + clocks = <&topckgen CLK_TOP_USB30_SEL>; + clock-names = "sys_ck"; + }; + + t-phy@11290000 { + compatible = "mediatek,mt8173-u3phy"; + reg = <0x11290000 0x800>; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + u2port0: usb-phy@11290800 { + reg = <0x11290800 0x100>; + clocks = <&apmixedsys CLK_APMIXED_REF2USB_TX>, <&clk48m>; + clock-names = "ref", "da_ref"; + #phy-cells = <1>; + }; + + u3port0: usb-phy@11290900 { + reg = <0x11290900 0x700>; + clocks = <&clk26m>; + clock-names = "ref"; + #phy-cells = <1>; + }; + + u2port1: usb-phy@11291000 { + reg = <0x11291000 0x100>; + #phy-cells = <1>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/mediatek,ufs-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,ufs-phy.yaml new file mode 100644 index 000000000000..3a9be82e7f13 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,ufs-phy.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,ufs-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Universal Flash Storage (UFS) M-PHY binding + +maintainers: + - Stanley Chu + - Chunfeng Yun + +description: | + UFS M-PHY nodes are defined to describe on-chip UFS M-PHY hardware macro. + Each UFS M-PHY node should have its own node. + To bind UFS M-PHY with UFS host controller, the controller node should + contain a phandle reference to UFS M-PHY node. + +properties: + $nodename: + pattern: "^ufs-phy@[0-9a-f]+$" + + compatible: + const: mediatek,mt8183-ufsphy + + reg: + maxItems: 1 + + clocks: + items: + - description: Unipro core control clock. + - description: M-PHY core control clock. + + clock-names: + items: + - const: unipro + - const: mp + + "#phy-cells": + const: 0 + +required: + - compatible + - reg + - "#phy-cells" + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + #include + ufsphy: ufs-phy@11fa0000 { + compatible = "mediatek,mt8183-ufsphy"; + reg = <0x11fa0000 0xc000>; + clocks = <&infracfg CLK_INFRA_UNIPRO_SCK>, + <&infracfg CLK_INFRA_UFS_MP_SAP_BCLK>; + clock-names = "unipro", "mp"; + #phy-cells = <0>; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml b/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml new file mode 100644 index 000000000000..598fd2b95c29 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,xsphy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek XS-PHY Controller Device Tree Bindings + +maintainers: + - Chunfeng Yun + +description: | + The XS-PHY controller supports physical layer functionality for USB3.1 + GEN2 controller on MediaTek SoCs. + + Banks layout of xsphy + ---------------------------------- + port offset bank + u2 port0 0x0000 MISC + 0x0100 FMREG + 0x0300 U2PHY_COM + u2 port1 0x1000 MISC + 0x1100 FMREG + 0x1300 U2PHY_COM + u2 port2 0x2000 MISC + ... + u31 common 0x3000 DIG_GLB + 0x3100 PHYA_GLB + u31 port0 0x3400 DIG_LN_TOP + 0x3500 DIG_LN_TX0 + 0x3600 DIG_LN_RX0 + 0x3700 DIG_LN_DAIF + 0x3800 PHYA_LN + u31 port1 0x3a00 DIG_LN_TOP + 0x3b00 DIG_LN_TX0 + 0x3c00 DIG_LN_RX0 + 0x3d00 DIG_LN_DAIF + 0x3e00 PHYA_LN + ... + DIG_GLB & PHYA_GLB are shared by U31 ports. + +properties: + $nodename: + pattern: "^xs-phy@[0-9a-f]+$" + + compatible: + items: + - enum: + - mediatek,mt3611-xsphy + - mediatek,mt3612-xsphy + - const: mediatek,xsphy + + reg: + description: + Register shared by multiple U3 ports, exclude port's private register, + if only U2 ports provided, shouldn't use the property. + maxItems: 1 + + "#address-cells": + enum: [1, 2] + + "#size-cells": + enum: [1, 2] + + ranges: true + + mediatek,src-ref-clk-mhz: + description: + Frequency of reference clock for slew rate calibrate + default: 26 + + mediatek,src-coef: + description: + Coefficient for slew rate calibrate, depends on SoC process + $ref: /schemas/types.yaml#/definitions/uint32 + default: 17 + +# Required child node: +patternProperties: + "^usb-phy@[0-9a-f]+$": + type: object + description: + A sub-node is required for each port the controller provides. + Address range information including the usual 'reg' property + is used inside these nodes to describe the controller's topology. + + properties: + reg: + maxItems: 1 + + clocks: + items: + - description: Reference clock, (HS is 48Mhz, SS/P is 24~27Mhz) + + clock-names: + items: + - const: ref + + "#phy-cells": + const: 1 + description: | + The cells contain the following arguments. + + - description: The PHY type + enum: + - PHY_TYPE_USB2 + - PHY_TYPE_USB3 + + # The following optional vendor properties are only for debug or HQA test + mediatek,eye-src: + description: + The value of slew rate calibrate (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-vrt: + description: + The selection of VRT reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-term: + description: + The selection of HS_TX TERM reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,efuse-intr: + description: + The selection of Internal Resistor (U2/U3 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 63 + + mediatek,efuse-tx-imp: + description: + The selection of TX Impedance (U3 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 31 + + mediatek,efuse-rx-imp: + description: + The selection of RX Impedance (U3 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 31 + + required: + - reg + - clocks + - clock-names + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - "#address-cells" + - "#size-cells" + - ranges + +additionalProperties: false + +examples: + - | + #include + + u3phy: xs-phy@11c40000 { + compatible = "mediatek,mt3611-xsphy", "mediatek,xsphy"; + reg = <0x11c43000 0x0200>; + mediatek,src-ref-clk-mhz = <26>; + mediatek,src-coef = <17>; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + u2port0: usb-phy@11c40000 { + reg = <0x11c40000 0x0400>; + clocks = <&clk48m>; + clock-names = "ref"; + mediatek,eye-src = <4>; + #phy-cells = <1>; + }; + + u3port0: usb-phy@11c43000 { + reg = <0x11c43400 0x0500>; + clocks = <&clk26m>; + clock-names = "ref"; + mediatek,efuse-intr = <28>; + #phy-cells = <1>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-tphy.txt b/Documentation/devicetree/bindings/phy/phy-mtk-tphy.txt deleted file mode 100644 index dd75b676b71d..000000000000 --- a/Documentation/devicetree/bindings/phy/phy-mtk-tphy.txt +++ /dev/null @@ -1,162 +0,0 @@ -MediaTek T-PHY binding --------------------------- - -T-phy controller supports physical layer functionality for a number of -controllers on MediaTek SoCs, such as, USB2.0, USB3.0, PCIe, and SATA. - -Required properties (controller (parent) node): - - compatible : should be one of - "mediatek,generic-tphy-v1" - "mediatek,generic-tphy-v2" - "mediatek,mt2701-u3phy" (deprecated) - "mediatek,mt2712-u3phy" (deprecated) - "mediatek,mt8173-u3phy"; - make use of "mediatek,generic-tphy-v1" on mt2701 instead and - "mediatek,generic-tphy-v2" on mt2712 instead. - -- #address-cells: the number of cells used to represent physical - base addresses. -- #size-cells: the number of cells used to represent the size of an address. -- ranges: the address mapping relationship to the parent, defined with - - empty value: if optional 'reg' is used. - - non-empty value: if optional 'reg' is not used. should set - the child's base address to 0, the physical address - within parent's address space, and the length of - the address map. - -Required nodes : a sub-node is required for each port the controller - provides. Address range information including the usual - 'reg' property is used inside these nodes to describe - the controller's topology. - -Optional properties (controller (parent) node): - - reg : offset and length of register shared by multiple ports, - exclude port's private register. It is needed on mt2701 - and mt8173, but not on mt2712. - - mediatek,src-ref-clk-mhz : frequency of reference clock for slew rate - calibrate - - mediatek,src-coef : coefficient for slew rate calibrate, depends on - SoC process - -Required properties (port (child) node): -- reg : address and length of the register set for the port. -- #phy-cells : should be 1 (See second example) - cell after port phandle is phy type from: - - PHY_TYPE_USB2 - - PHY_TYPE_USB3 - - PHY_TYPE_PCIE - - PHY_TYPE_SATA - -Optional properties (PHY_TYPE_USB2 port (child) node): -- clocks : a list of phandle + clock-specifier pairs, one for each - entry in clock-names -- clock-names : may contain - "ref": 48M reference clock for HighSpeed (digital) phy; and 26M - reference clock for SuperSpeed (digital) phy, sometimes is - 24M, 25M or 27M, depended on platform. - "da_ref": the reference clock of analog phy, used if the clocks - of analog and digital phys are separated, otherwise uses - "ref" clock only if needed. - -- mediatek,eye-src : u32, the value of slew rate calibrate -- mediatek,eye-vrt : u32, the selection of VRT reference voltage -- mediatek,eye-term : u32, the selection of HS_TX TERM reference voltage -- mediatek,bc12 : bool, enable BC12 of u2phy if support it -- mediatek,discth : u32, the selection of disconnect threshold -- mediatek,intr : u32, the selection of internal R (resistance) - -Example: - -u3phy: usb-phy@11290000 { - compatible = "mediatek,mt8173-u3phy"; - reg = <0 0x11290000 0 0x800>; - #address-cells = <2>; - #size-cells = <2>; - ranges; - - u2port0: usb-phy@11290800 { - reg = <0 0x11290800 0 0x100>; - clocks = <&apmixedsys CLK_APMIXED_REF2USB_TX>; - clock-names = "ref"; - #phy-cells = <1>; - }; - - u3port0: usb-phy@11290900 { - reg = <0 0x11290800 0 0x700>; - clocks = <&clk26m>; - clock-names = "ref"; - #phy-cells = <1>; - }; - - u2port1: usb-phy@11291000 { - reg = <0 0x11291000 0 0x100>; - clocks = <&apmixedsys CLK_APMIXED_REF2USB_TX>; - clock-names = "ref"; - #phy-cells = <1>; - }; -}; - -Specifying phy control of devices ---------------------------------- - -Device nodes should specify the configuration required in their "phys" -property, containing a phandle to the phy port node and a device type; -phy-names for each port are optional. - -Example: - -#include - -usb30: usb@11270000 { - ... - phys = <&u2port0 PHY_TYPE_USB2>, <&u3port0 PHY_TYPE_USB3>; - phy-names = "usb2-0", "usb3-0"; - ... -}; - - -Layout differences of banks between mt8173/mt2701 and mt2712 -------------------------------------------------------------- -mt8173 and mt2701: -port offset bank -shared 0x0000 SPLLC - 0x0100 FMREG -u2 port0 0x0800 U2PHY_COM -u3 port0 0x0900 U3PHYD - 0x0a00 U3PHYD_BANK2 - 0x0b00 U3PHYA - 0x0c00 U3PHYA_DA -u2 port1 0x1000 U2PHY_COM -u3 port1 0x1100 U3PHYD - 0x1200 U3PHYD_BANK2 - 0x1300 U3PHYA - 0x1400 U3PHYA_DA -u2 port2 0x1800 U2PHY_COM - ... - -mt2712: -port offset bank -u2 port0 0x0000 MISC - 0x0100 FMREG - 0x0300 U2PHY_COM -u3 port0 0x0700 SPLLC - 0x0800 CHIP - 0x0900 U3PHYD - 0x0a00 U3PHYD_BANK2 - 0x0b00 U3PHYA - 0x0c00 U3PHYA_DA -u2 port1 0x1000 MISC - 0x1100 FMREG - 0x1300 U2PHY_COM -u3 port1 0x1700 SPLLC - 0x1800 CHIP - 0x1900 U3PHYD - 0x1a00 U3PHYD_BANK2 - 0x1b00 U3PHYA - 0x1c00 U3PHYA_DA -u2 port2 0x2000 MISC - ... - - SPLLC shared by u3 ports and FMREG shared by u2 ports on -mt8173/mt2701 are put back into each port; a new bank MISC for -u2 ports and CHIP for u3 ports are added on mt2712. diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt b/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt deleted file mode 100644 index 5789029a1d42..000000000000 --- a/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt +++ /dev/null @@ -1,38 +0,0 @@ -MediaTek Universal Flash Storage (UFS) M-PHY binding --------------------------------------------------------- - -UFS M-PHY nodes are defined to describe on-chip UFS M-PHY hardware macro. -Each UFS M-PHY node should have its own node. - -To bind UFS M-PHY with UFS host controller, the controller node should -contain a phandle reference to UFS M-PHY node. - -Required properties for UFS M-PHY nodes: -- compatible : Compatible list, contains the following controller: - "mediatek,mt8183-ufsphy" for ufs phy - persent on MT81xx chipsets. -- reg : Address and length of the UFS M-PHY register set. -- #phy-cells : This property shall be set to 0. -- clocks : List of phandle and clock specifier pairs. -- clock-names : List of clock input name strings sorted in the same - order as the clocks property. Following clocks are - mandatory. - "unipro": Unipro core control clock. - "mp": M-PHY core control clock. - -Example: - - ufsphy: phy@11fa0000 { - compatible = "mediatek,mt8183-ufsphy"; - reg = <0 0x11fa0000 0 0xc000>; - #phy-cells = <0>; - - clocks = <&infracfg_ao INFRACFG_AO_UNIPRO_SCK_CG>, - <&infracfg_ao INFRACFG_AO_UFS_MP_SAP_BCLK_CG>; - clock-names = "unipro", "mp"; - }; - - ufshci@11270000 { - ... - phys = <&ufsphy>; - }; diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt b/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt deleted file mode 100644 index e7caefa0b9c2..000000000000 --- a/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt +++ /dev/null @@ -1,109 +0,0 @@ -MediaTek XS-PHY binding --------------------------- - -The XS-PHY controller supports physical layer functionality for USB3.1 -GEN2 controller on MediaTek SoCs. - -Required properties (controller (parent) node): - - compatible : should be "mediatek,-xsphy", "mediatek,xsphy", - soc-model is the name of SoC, such as mt3611 etc; - when using "mediatek,xsphy" compatible string, you need SoC specific - ones in addition, one of: - - "mediatek,mt3611-xsphy" - - - #address-cells, #size-cells : should use the same values as the root node - - ranges: must be present - -Optional properties (controller (parent) node): - - reg : offset and length of register shared by multiple U3 ports, - exclude port's private register, if only U2 ports provided, - shouldn't use the property. - - mediatek,src-ref-clk-mhz : u32, frequency of reference clock for slew rate - calibrate - - mediatek,src-coef : u32, coefficient for slew rate calibrate, depends on - SoC process - -Required nodes : a sub-node is required for each port the controller - provides. Address range information including the usual - 'reg' property is used inside these nodes to describe - the controller's topology. - -Required properties (port (child) node): -- reg : address and length of the register set for the port. -- clocks : a list of phandle + clock-specifier pairs, one for each - entry in clock-names -- clock-names : must contain - "ref": 48M reference clock for HighSpeed analog phy; and 26M - reference clock for SuperSpeedPlus analog phy, sometimes is - 24M, 25M or 27M, depended on platform. -- #phy-cells : should be 1 - cell after port phandle is phy type from: - - PHY_TYPE_USB2 - - PHY_TYPE_USB3 - -The following optional properties are only for debug or HQA test -Optional properties (PHY_TYPE_USB2 port (child) node): -- mediatek,eye-src : u32, the value of slew rate calibrate -- mediatek,eye-vrt : u32, the selection of VRT reference voltage -- mediatek,eye-term : u32, the selection of HS_TX TERM reference voltage -- mediatek,efuse-intr : u32, the selection of Internal Resistor - -Optional properties (PHY_TYPE_USB3 port (child) node): -- mediatek,efuse-intr : u32, the selection of Internal Resistor -- mediatek,efuse-tx-imp : u32, the selection of TX Impedance -- mediatek,efuse-rx-imp : u32, the selection of RX Impedance - -Banks layout of xsphy -------------------------------------------------------------- -port offset bank -u2 port0 0x0000 MISC - 0x0100 FMREG - 0x0300 U2PHY_COM -u2 port1 0x1000 MISC - 0x1100 FMREG - 0x1300 U2PHY_COM -u2 port2 0x2000 MISC - ... -u31 common 0x3000 DIG_GLB - 0x3100 PHYA_GLB -u31 port0 0x3400 DIG_LN_TOP - 0x3500 DIG_LN_TX0 - 0x3600 DIG_LN_RX0 - 0x3700 DIG_LN_DAIF - 0x3800 PHYA_LN -u31 port1 0x3a00 DIG_LN_TOP - 0x3b00 DIG_LN_TX0 - 0x3c00 DIG_LN_RX0 - 0x3d00 DIG_LN_DAIF - 0x3e00 PHYA_LN - ... - -DIG_GLB & PHYA_GLB are shared by U31 ports. - -Example: - -u3phy: usb-phy@11c40000 { - compatible = "mediatek,mt3611-xsphy", "mediatek,xsphy"; - reg = <0 0x11c43000 0 0x0200>; - mediatek,src-ref-clk-mhz = <26>; - mediatek,src-coef = <17>; - #address-cells = <2>; - #size-cells = <2>; - ranges; - - u2port0: usb-phy@11c40000 { - reg = <0 0x11c40000 0 0x0400>; - clocks = <&clk48m>; - clock-names = "ref"; - mediatek,eye-src = <4>; - #phy-cells = <1>; - }; - - u3port0: usb-phy@11c43000 { - reg = <0 0x11c43400 0 0x0500>; - clocks = <&clk26m>; - clock-names = "ref"; - mediatek,efuse-intr = <28>; - #phy-cells = <1>; - }; -}; diff --git a/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml b/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml index 0ba61979b970..46df6786727a 100644 --- a/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml +++ b/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml @@ -45,6 +45,12 @@ properties: "#size-cells": const: 0 + vdda1v1-supply: + description: regulator providing 1V1 power supply to the PLL block + + vdda1v8-supply: + description: regulator providing 1V8 power supply to the PLL block + #Required child nodes: patternProperties: @@ -61,12 +67,6 @@ patternProperties: phy-supply: description: regulator providing 3V3 power supply to the PHY. - vdda1v1-supply: - description: regulator providing 1V1 power supply to the PLL block - - vdda1v8-supply: - description: regulator providing 1V8 power supply to the PLL block - "#phy-cells": enum: [ 0x0, 0x1 ] @@ -90,8 +90,6 @@ patternProperties: required: - reg - phy-supply - - vdda1v1-supply - - vdda1v8-supply - "#phy-cells" additionalProperties: false @@ -102,6 +100,8 @@ required: - clocks - "#address-cells" - "#size-cells" + - vdda1v1-supply + - vdda1v8-supply - usb-phy@0 - usb-phy@1 @@ -116,22 +116,20 @@ examples: reg = <0x5a006000 0x1000>; clocks = <&rcc USBPHY_K>; resets = <&rcc USBPHY_R>; + vdda1v1-supply = <®11>; + vdda1v8-supply = <®18>; #address-cells = <1>; #size-cells = <0>; usbphyc_port0: usb-phy@0 { reg = <0>; phy-supply = <&vdd_usb>; - vdda1v1-supply = <®11>; - vdda1v8-supply = <®18>; #phy-cells = <0>; }; usbphyc_port1: usb-phy@1 { reg = <1>; phy-supply = <&vdd_usb>; - vdda1v1-supply = <®11>; - vdda1v8-supply = <®18>; #phy-cells = <1>; }; }; diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml index ec05db374645..626447fee092 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml @@ -25,19 +25,32 @@ properties: - qcom,msm8998-qmp-pcie-phy - qcom,msm8998-qmp-ufs-phy - qcom,msm8998-qmp-usb3-phy + - qcom,sc8180x-qmp-ufs-phy + - qcom,sc8180x-qmp-usb3-phy - qcom,sdm845-qhp-pcie-phy - qcom,sdm845-qmp-pcie-phy - qcom,sdm845-qmp-ufs-phy - qcom,sdm845-qmp-usb3-uni-phy - qcom,sm8150-qmp-ufs-phy + - qcom,sm8150-qmp-usb3-phy + - qcom,sm8150-qmp-usb3-uni-phy - qcom,sm8250-qmp-ufs-phy - qcom,sm8250-qmp-gen3x1-pcie-phy - qcom,sm8250-qmp-gen3x2-pcie-phy - qcom,sm8250-qmp-modem-pcie-phy + - qcom,sm8250-qmp-usb3-phy + - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-ufs-phy + - qcom,sm8350-qmp-usb3-phy + - qcom,sm8350-qmp-usb3-uni-phy + - qcom,sdx55-qmp-usb3-uni-phy reg: + minItems: 1 + maxItems: 2 items: - description: Address and length of PHY's common serdes block. + - description: Address and length of PHY's DP_COM control block. "#clock-cells": enum: [ 1, 2 ] @@ -131,6 +144,32 @@ allOf: items: - const: phy - const: common + - if: + properties: + compatible: + contains: + enum: + - qcom,sdx55-qmp-usb3-uni-phy + then: + properties: + clocks: + items: + - description: Phy aux clock. + - description: Phy config clock. + - description: 19.2 MHz ref clk. + clock-names: + items: + - const: aux + - const: cfg_ahb + - const: ref + resets: + items: + - description: reset of phy block. + - description: phy common block reset. + reset-names: + items: + - const: phy + - const: common - if: properties: compatible: @@ -285,6 +324,64 @@ allOf: reset-names: items: - const: phy + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8150-qmp-usb3-phy + - qcom,sm8150-qmp-usb3-uni-phy + - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-usb3-uni-phy + then: + properties: + clocks: + items: + - description: Phy aux clock. + - description: 19.2 MHz ref clk source. + - description: 19.2 MHz ref clk. + - description: Phy common block aux clock. + clock-names: + items: + - const: aux + - const: ref_clk_src + - const: ref + - const: com_aux + resets: + items: + - description: reset of phy block. + - description: phy common block reset. + reset-names: + items: + - const: phy + - const: common + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8250-qmp-usb3-phy + - qcom,sm8350-qmp-usb3-phy + then: + properties: + clocks: + items: + - description: Phy aux clock. + - description: 19.2 MHz ref clk. + - description: Phy common block aux clock. + clock-names: + items: + - const: aux + - const: ref_clk_src + - const: com_aux + resets: + items: + - description: reset of phy block. + - description: phy common block reset. + reset-names: + items: + - const: phy + - const: common examples: - | diff --git a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml index d457fb6a4779..9f9cf07b7d45 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml @@ -21,6 +21,8 @@ properties: - qcom,ipq8074-qusb2-phy - qcom,msm8996-qusb2-phy - qcom,msm8998-qusb2-phy + - qcom,sdm660-qusb2-phy + - qcom,ipq6018-qusb2-phy - items: - enum: - qcom,sc7180-qusb2-phy diff --git a/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml b/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml index ca6a0836b53c..abcc4373f39e 100644 --- a/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml @@ -16,6 +16,7 @@ properties: compatible: enum: - qcom,usb-hs-28nm-femtophy + - qcom,usb-hs-28nm-mdm9607 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml b/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml index 4949a2851532..ee77c6458326 100644 --- a/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml @@ -17,6 +17,8 @@ properties: enum: - qcom,usb-snps-hs-7nm-phy - qcom,sm8150-usb-hs-phy + - qcom,sm8250-usb-hs-phy + - qcom,sm8350-usb-hs-phy - qcom,usb-snps-femto-v2-phy reg: diff --git a/Documentation/devicetree/bindings/phy/renesas,usb2-phy.yaml b/Documentation/devicetree/bindings/phy/renesas,usb2-phy.yaml index 829e8c7e467a..0f358d5b84ef 100644 --- a/Documentation/devicetree/bindings/phy/renesas,usb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/renesas,usb2-phy.yaml @@ -81,9 +81,8 @@ properties: if: properties: compatible: - items: - enum: - - renesas,usb2-phy-r7s9210 + contains: + const: renesas,usb2-phy-r7s9210 then: required: - clock-names diff --git a/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt b/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt index 00aa2d349e55..57d28c0d5696 100644 --- a/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt +++ b/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt @@ -16,11 +16,11 @@ Optional properties: - drive-impedance-ohm: Specifies the drive impedance in Ohm. Possible values are 33, 40, 50, 66 and 100. If not set, the default value of 50 will be applied. - - enable-strobe-pulldown: Enable internal pull-down for the strobe line. - If not set, pull-down is not used. - - output-tapdelay-select: Specifies the phyctrl_otapdlysec register. - If not set, the register defaults to 0x4. - Maximum value 0xf. + - rockchip,enable-strobe-pulldown: Enable internal pull-down for the strobe + line. If not set, pull-down is not used. + - rockchip,output-tapdelay-select: Specifies the phyctrl_otapdlysec register. + If not set, the register defaults to 0x4. + Maximum value 0xf. Example: diff --git a/Documentation/devicetree/bindings/phy/socionext,uniphier-ahci-phy.yaml b/Documentation/devicetree/bindings/phy/socionext,uniphier-ahci-phy.yaml index 34756347a14e..745c525ce6b9 100644 --- a/Documentation/devicetree/bindings/phy/socionext,uniphier-ahci-phy.yaml +++ b/Documentation/devicetree/bindings/phy/socionext,uniphier-ahci-phy.yaml @@ -20,7 +20,7 @@ properties: - socionext,uniphier-pxs3-ahci-phy reg: - description: PHY register region (offset and length) + maxItems: 1 "#phy-cells": const: 0 diff --git a/Documentation/devicetree/bindings/phy/socionext,uniphier-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/socionext,uniphier-pcie-phy.yaml index a06831fd64b9..3e0566899041 100644 --- a/Documentation/devicetree/bindings/phy/socionext,uniphier-pcie-phy.yaml +++ b/Documentation/devicetree/bindings/phy/socionext,uniphier-pcie-phy.yaml @@ -21,7 +21,7 @@ properties: - socionext,uniphier-pxs3-pcie-phy reg: - description: PHY register region (offset and length) + maxItems: 1 "#phy-cells": const: 0 diff --git a/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3hs-phy.yaml b/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3hs-phy.yaml index 6fa5caab1487..a681cbc3b4ef 100644 --- a/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3hs-phy.yaml +++ b/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3hs-phy.yaml @@ -24,7 +24,7 @@ properties: - socionext,uniphier-pxs3-usb3-hsphy reg: - description: PHY register region (offset and length) + maxItems: 1 "#phy-cells": const: 0 diff --git a/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3ss-phy.yaml b/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3ss-phy.yaml index 9d46715ed036..41c0dd68ee25 100644 --- a/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3ss-phy.yaml +++ b/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3ss-phy.yaml @@ -25,7 +25,7 @@ properties: - socionext,uniphier-pxs3-usb3-ssphy reg: - description: PHY register region (offset and length) + maxItems: 1 "#phy-cells": const: 0 diff --git a/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml b/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml index bcec422d7734..ff8a6d9eb153 100644 --- a/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml +++ b/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml @@ -55,7 +55,7 @@ properties: - ti,am654-phy-gmii-sel reg: - description: Address and length of the register set for the device + maxItems: 1 '#phy-cells': true diff --git a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml index c33e9bc79521..bbbd85501ada 100644 --- a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml +++ b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml @@ -151,7 +151,7 @@ patternProperties: WIZ node should have '1' subnode for the SERDES. It could be either Sierra SERDES or Torrent SERDES. Sierra SERDES should follow the bindings specified in - Documentation/devicetree/bindings/phy/phy-cadence-sierra.txt + Documentation/devicetree/bindings/phy/phy-cadence-sierra.yaml Torrent SERDES should follow the bindings specified in Documentation/devicetree/bindings/phy/phy-cadence-torrent.yaml diff --git a/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml index 5240487dfe50..cce63c3cc463 100644 --- a/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml @@ -53,6 +53,8 @@ properties: - allwinner,sun50i-h5-pinctrl - allwinner,sun50i-h6-pinctrl - allwinner,sun50i-h6-r-pinctrl + - allwinner,sun50i-h616-pinctrl + - allwinner,sun50i-h616-r-pinctrl - allwinner,suniv-f1c100s-pinctrl - nextthing,gr8-pinctrl @@ -61,7 +63,7 @@ properties: interrupts: minItems: 1 - maxItems: 7 + maxItems: 8 description: One interrupt per external interrupt bank supported on the controller, sorted by bank number ascending order. @@ -91,7 +93,7 @@ properties: bank found in the controller $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 1 - maxItems: 5 + maxItems: 8 patternProperties: # It's pretty scary, but the basic idea is that: @@ -145,6 +147,17 @@ allOf: # boards are defining it at the moment so it would generate a lot of # warnings. + - if: + properties: + compatible: + enum: + - allwinner,sun50i-h616-pinctrl + + then: + properties: + interrupts: + minItems: 8 + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml index 54631dc1adb0..100bb6dea3ec 100644 --- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml @@ -23,8 +23,7 @@ properties: compatible: const: aspeed,ast2400-pinctrl reg: - description: | - A hint for the memory regions associated with the pin-controller + maxItems: 2 patternProperties: '^.*$': @@ -63,7 +62,7 @@ examples: reg = <0x1e6e2000 0x1a8>; pinctrl: pinctrl { - compatible = "aspeed,g4-pinctrl"; + compatible = "aspeed,ast2400-pinctrl"; pinctrl_i2c3_default: i2c3_default { function = "I2C3"; diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml index a90c0fe0495f..904697bc9415 100644 --- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml @@ -24,8 +24,8 @@ properties: compatible: const: aspeed,ast2500-pinctrl reg: - description: | - A hint for the memory regions associated with the pin-controller + maxItems: 2 + aspeed,external-nodes: minItems: 2 maxItems: 2 @@ -81,7 +81,7 @@ examples: reg = <0x1e6e2000 0x1a8>; pinctrl: pinctrl { - compatible = "aspeed,g5-pinctrl"; + compatible = "aspeed,ast2500-pinctrl"; aspeed,external-nodes = <&gfx>, <&lhc>; pinctrl_i2c3_default: i2c3_default { diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml index c78ab7e2eee7..ad91c0bc54da 100644 --- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml @@ -95,7 +95,7 @@ examples: reg = <0x1e6e2000 0xf6c>; pinctrl: pinctrl { - compatible = "aspeed,g6-pinctrl"; + compatible = "aspeed,ast2600-pinctrl"; pinctrl_pwm10g1_default: pwm10g1_default { function = "PWM10"; diff --git a/Documentation/devicetree/bindings/pinctrl/atmel,at91-pio4-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/atmel,at91-pio4-pinctrl.txt index 265015bc0603..e2b861ce16d8 100644 --- a/Documentation/devicetree/bindings/pinctrl/atmel,at91-pio4-pinctrl.txt +++ b/Documentation/devicetree/bindings/pinctrl/atmel,at91-pio4-pinctrl.txt @@ -35,9 +35,11 @@ ioset settings. Use the macros from boot/dts/-pinfunc.h file to get the right representation of the pin. Optional properties: -- GENERIC_PINCONFIG: generic pinconfig options to use, bias-disable, -bias-pull-down, bias-pull-up, drive-open-drain, input-schmitt-enable, -input-debounce, output-low, output-high. +- GENERIC_PINCONFIG: generic pinconfig options to use: + - bias-disable, bias-pull-down, bias-pull-up, drive-open-drain, + input-schmitt-enable, input-debounce, output-low, output-high. + - for microchip,sama7g5-pinctrl only: + - slew-rate: 0 - disabled, 1 - enabled (default) - atmel,drive-strength: 0 or 1 for low drive, 2 for medium drive and 3 for high drive. The default value is low drive. diff --git a/Documentation/devicetree/bindings/pinctrl/brcm,ns2-pinmux.txt b/Documentation/devicetree/bindings/pinctrl/brcm,ns2-pinmux.txt index e295dda4bbba..40e0a9a19525 100644 --- a/Documentation/devicetree/bindings/pinctrl/brcm,ns2-pinmux.txt +++ b/Documentation/devicetree/bindings/pinctrl/brcm,ns2-pinmux.txt @@ -39,7 +39,7 @@ For example: <0x660009b0 0x40>; pinctrl-names = "default"; - pinctrl-0 = <&nand_sel &uart3_rx &sdio0_d4>; + pinctrl-0 = <&nand_sel>, <&uart3_rx>, <&sdio0_d4>; /* Select nand function */ nand_sel: nand_sel { diff --git a/Documentation/devicetree/bindings/pinctrl/brcm,nsp-pinmux.txt b/Documentation/devicetree/bindings/pinctrl/brcm,nsp-pinmux.txt index 603564e5fe6f..dede11e4ef78 100644 --- a/Documentation/devicetree/bindings/pinctrl/brcm,nsp-pinmux.txt +++ b/Documentation/devicetree/bindings/pinctrl/brcm,nsp-pinmux.txt @@ -30,7 +30,7 @@ For example: <0x1803f408 0x04>; pinctrl-names = "default"; - pinctrl-0 = <&pwm &gpio_b &nand_sel>; + pinctrl-0 = <&pwm>, <&gpio_b>, <&nand_sel>; pwm: pwm { function = "pwm"; diff --git a/Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml b/Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml new file mode 100644 index 000000000000..46fbc73ab26b --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/canaan,k210-fpioa.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Canaan Kendryte K210 FPIOA Device Tree Bindings + +maintainers: + - Damien Le Moal + +description: + The Canaan Kendryte K210 SoC Fully Programmable IO Array (FPIOA) + controller allows assiging any of 256 possible functions to any of + 48 IO pins of the SoC. Pin function configuration is performed on + a per-pin basis. + +properties: + compatible: + const: canaan,k210-fpioa + + reg: + maxItems: 1 + description: + Address and length of the register set for the FPIOA controller. + + clocks: + items: + - description: Controller reference clock source + - description: APB interface clock source + + clock-names: + items: + - const: ref + - const: pclk + + resets: + maxItems: 1 + + canaan,k210-sysctl-power: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: | + phandle of the K210 system controller node and offset of its + power domain control register. + +patternProperties: + '-pinmux$': + type: object + $ref: /schemas/pinctrl/pinmux-node.yaml + description: + FPIOA client devices use sub-nodes to define the desired pin + configuration. Client device sub-nodes use the pinux property + below. + + properties: + pinmux: + description: + List of IO pins alternate functions. The values for each IO + pin is a combination of an IO pin number (0 to 47) with the + desired function for the IO pin. Functions are defined as + macros in include/dt-bindings/pinctrl/k210-fpioa.h. + The K210_FPIOA(IO pin, function) macro is provided to + facilitate the combination of IO pin numbers and functions. + + required: + - pinmux + + additionalProperties: false + + '-pins$': + type: object + $ref: /schemas/pinctrl/pincfg-node.yaml + description: + FPIOA client devices use sub-nodes to define the desired + configuration of pins. Client device sub-nodes use the + properties below. + + properties: + pins: + description: + List of IO pins affected by the properties specified in this + subnode. IO pins are identified using the pin names "IO_xx". + Pin configuration nodes can also define the power domain to + be used for the SoC pin groups A0 (IO pins 0-5), + A1 (IO pins 6-11), A2 (IO pins 12-17), B0 (IO pins 18-23), + B1 (IO pins 24-29), B2 (IO pins 30-35), B3 (IO pins 30-35), + C0 (IO pins 36-41) and C1 (IO pins 42-47) using the + power-source property. + items: + anyOf: + - pattern: "^(IO_([0-9]*))|(A[0-2])|(B[3-5])|(C[6-7])$" + - enum: [ IO_0, IO_1, IO_2, IO_3, IO_4, IO_5, IO_6, IO_7, + IO_8, IO_9, IO_10, IO_11, IO_12, IO_13, IO_14, + IO_15, IO_16, IO_17, IO_18, IO_19, IO_20, IO_21, + IO_22, IO_23, IO_24, IO_25, IO_26, IO_27, IO_28, + IO_29, IO_30, IO_31, IO_32, IO_33, IO_34, IO_35, + IO_36, IO_37, IO_38, IO_39, IO_40, IO_41, IO_42, + IO_43, IO_44, IO_45, IO_46, IO_47, + A0, A1, A2, B3, B4, B5, C6, C7 ] + bias-disable: true + + bias-pull-down: true + + bias-pull-up: true + + drive-strength: true + + drive-strength-microamp: true + + input-enable: true + + input-disable: true + + input-schmitt-enable: true + + input-schmitt-disable: true + + input-polarity-invert: + description: + Enable or disable pin input polarity inversion. + + output-enable: true + + output-disable: true + + output-high: true + + output-low: true + + output-polarity-invert: + description: + Enable or disable pin output polarity inversion. + + slew-rate: true + + power-source: true + + additionalProperties: false + +required: + - compatible + - reg + - clocks + - canaan,k210-sysctl-power + +additionalProperties: false + +examples: + - | + #include + #include + #include + + fpioa: pinmux@502B0000 { + compatible = "canaan,k210-fpioa"; + reg = <0x502B0000 0x100>; + clocks = <&sysclk K210_CLK_FPIOA>, + <&sysclk K210_CLK_APB0>; + clock-names = "ref", "pclk"; + resets = <&sysrst K210_RST_FPIOA>; + canaan,k210-sysctl-power = <&sysctl 108>; + pinctrl-0 = <&jtag_pinctrl>; + pinctrl-names = "default"; + + jtag_pinctrl: jtag-pinmux { + pinmux = , + , + , + ; + }; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt index 8ac1d0851a0f..bfab5ca49fd1 100644 --- a/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt +++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt @@ -60,7 +60,7 @@ iomuxc-lpsr controller and SDA pad from iomuxc controller as: i2c1: i2c@30a20000 { pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_i2c1_1 &pinctrl_i2c1_2>; + pinctrl-0 = <&pinctrl_i2c1_1>, <&pinctrl_i2c1_2>; }; iomuxc-lpsr@302c0000 { diff --git a/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml b/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml index df0c83cb1c6e..4fe35e650909 100644 --- a/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml +++ b/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml @@ -99,8 +99,8 @@ patternProperties: '#interrupt-cells': description: - Specifies the pin (port and bit) and flags, as defined in - defined in include/dt-bindings/interrupt-controller/irq.h + Specifies the pin (port and bit) and flags, as defined in + defined in include/dt-bindings/interrupt-controller/irq.h const: 3 ngpios: diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-atlas7.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-atlas7.txt deleted file mode 100644 index fbdd1a716a1e..000000000000 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-atlas7.txt +++ /dev/null @@ -1,109 +0,0 @@ -CSR SiRFatlas7 pinmux controller - -Required properties: -- compatible : "sirf,atlas7-ioc" -- reg : Address range of the pinctrl registers - -For example, pinctrl might have properties like the following: - pinctrl: ioc@18880000 { - compatible = "sirf,atlas7-ioc"; - reg = <0x18880000 0x1000>; - - a_ac97_pmx: ac97@0 { - ac97 { - groups = "audio_ac97_grp"; - function = "audio_ac97"; - }; - }; - - ... - - sd2_pmx: sd2@0 { - sd2 { - groups = "sd2_grp0"; - function = "sd2"; - }; - }; - - ... - - - sample0_cfg: sample0@0 { - sample0 { - pins = "ldd_0", "ldd_1"; - bias-pull-up; - }; - }; - - sample1_cfg: sample1@0 { - sample1 { - pins = "ldd_2", "ldd_3"; - input-schmitt-enable; - }; - }; - - sample2_cfg: sample2@0 { - sample2 { - groups = "uart4_nopause_grp"; - bias-pull-down; - }; - }; - - sample3_cfg: sample3@0 { - sample3 { - pins = "ldd_4", "ldd_5"; - drive-strength = <2>; - }; - }; - }; - -Please refer to pinctrl-bindings.txt in this directory for details of the common -pinctrl bindings used by client devices. - -SiRFatlas7's pinmux nodes act as a container for an arbitrary number of subnodes. -Each of these subnodes represents some desired configuration for a group of pins. - -Required subnode-properties: -- groups : An array of strings. Each string contains the name of a group. -- function: A string containing the name of the function to mux to the - group. - - Valid values for group and function names can be found from looking at the - group and function arrays in driver files: - drivers/pinctrl/pinctrl-sirf.c - -For example, pinctrl might have subnodes like the following: - sd0_pmx: sd0@0 { - sd0 { - groups = "sd0_grp"; - function = "sd0"; - }; - }; - - sd1_pmx0: sd1@0 { - sd1 { - groups = "sd1_grp0"; - function = "sd1_m0"; - }; - }; - - sd1_pmx1: sd1@1 { - sd1 { - groups = "sd1_grp1"; - function = "sd1_m1"; - }; - }; - -For a specific board, if it wants to use sd1, -it can add the following to its board-specific .dts file. -sd1: sd@12340000 { - pinctrl-names = "default"; - pinctrl-0 = <&sd1_pmx0>; -} - -or - -sd1: sd@12340000 { - pinctrl-names = "default"; - pinctrl-0 = <&sd1_pmx1>; -} diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt index 4613bb17ace3..9dae60acf950 100644 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt +++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt @@ -77,13 +77,13 @@ For example: device { pinctrl-names = "active", "idle"; pinctrl-0 = <&state_0_node_a>; - pinctrl-1 = <&state_1_node_a &state_1_node_b>; + pinctrl-1 = <&state_1_node_a>, <&state_1_node_b>; }; /* For the same device if using state IDs */ device { pinctrl-0 = <&state_0_node_a>; - pinctrl-1 = <&state_1_node_a &state_1_node_b>; + pinctrl-1 = <&state_1_node_a>, <&state_1_node_b>; }; /* diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt index 8b94aa8f5971..6ec3c8d79f49 100644 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt +++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt @@ -134,7 +134,7 @@ gpio21: gpio@21 { #interrupt-cells = <0x2>; microchip,irq-mirror; pinctrl-names = "default"; - pinctrl-0 = <&i2cgpio0irq &gpio21pullups>; + pinctrl-0 = <&i2cgpio0irq>, <&gpio21pullups>; gpio21pullups: pinmux { pins = "gpio0", "gpio1", "gpio2", "gpio3", diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt index 931a18cd1e23..360e59c9301a 100644 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt +++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt @@ -91,7 +91,7 @@ Examples: pinctrl@1c20800 { compatible = "mediatek,mt8135-pinctrl"; reg = <0 0x1000B000 0 0x1000>; - mediatek,pctl-regmap = <&syscfg_pctl_a &syscfg_pctl_b>; + mediatek,pctl-regmap = <&syscfg_pctl_a>, <&syscfg_pctl_b>; pins-are-numbered; gpio-controller; #gpio-cells = <2>; diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt index f903eb4471f8..bfd222b05495 100644 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt +++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt @@ -8,7 +8,7 @@ Required properties: - reg : offset and length of the register set for the mux registers - #pinctrl-cells : number of cells in addition to the index, set to 1 - for pinctrl-single,pins and 2 for pinctrl-single,bits + or 2 for pinctrl-single,pins and set to 2 for pinctrl-single,bits - pinctrl-single,register-width : pinmux register access width in bits @@ -80,7 +80,7 @@ Optional properties: property. /* pin base, nr pins & gpio function */ - pinctrl-single,gpio-range = <&range 0 3 0 &range 3 9 1>; + pinctrl-single,gpio-range = <&range 0 3 0>, <&range 3 9 1>; - interrupt-controller : standard interrupt controller binding if using interrupts for wake-up events for example. In this case pinctrl-single @@ -185,10 +185,10 @@ pmx_gpio: pinmux@d401e000 { pinctrl-single,function-mask = <7>; /* sparse GPIO range could be supported */ - pinctrl-single,gpio-range = <&range 0 3 0 &range 3 9 1 - &range 12 1 0 &range 13 29 1 - &range 43 1 0 &range 44 49 1 - &range 94 1 1 &range 96 2 1>; + pinctrl-single,gpio-range = <&range 0 3 0>, <&range 3 9 1>, + <&range 12 1 0>, <&range 13 29 1>, + <&range 43 1 0>, <&range 44 49 1>, + <&range 94 1 1>, <&range 96 2 1>; range: gpio-range { #pinctrl-single,gpio-range-cells = <3>; diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-zx.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-zx.txt deleted file mode 100644 index 39170f372599..000000000000 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-zx.txt +++ /dev/null @@ -1,84 +0,0 @@ -* ZTE ZX Pin Controller - -The pin controller on ZTE ZX platforms is kinda of hybrid. It consists of -a main controller and an auxiliary one. For example, on ZX296718 SoC, the -main controller is TOP_PMM and the auxiliary one is AON_IOCFG. Both -controllers work together to control pin multiplexing and configuration in -the way illustrated as below. - - - GMII_RXD3 ---+ - | - DVI1_HS ---+----------------------------- GMII_RXD3 (TOP pin) - | - BGPIO16 ---+ ^ - | pinconf - ^ | - | pinmux | - | | - - TOP_PMM (main) AON_IOCFG (aux) - - | | | - | pinmux | | - | pinmux v | - v | pinconf - KEY_ROW2 ---+ v - PORT1_LCD_TE ---+ | - | AGPIO10 ---+------ KEY_ROW2 (AON pin) - I2S0_DOUT3 ---+ | - |-----------------------+ - PWM_OUT3 ---+ - | - VGA_VS1 ---+ - - -For most of pins like GMII_RXD3 in the figure, the pinmux function is -controlled by TOP_PMM block only, and this type of pins are meant by term -'TOP pins'. For pins like KEY_ROW2, the pinmux is controlled by both -TOP_PMM and AON_IOCFG blocks, as the available multiplexing functions for -the pin spread in both controllers. This type of pins are called 'AON pins'. -Though pinmux implementation is quite different, pinconf is same for both -types of pins. Both are controlled by auxiliary controller, i.e. AON_IOCFG -on ZX296718. - -Required properties: -- compatible: should be "zte,zx296718-pmm". -- reg: the register physical address and length. -- zte,auxiliary-controller: phandle to the auxiliary pin controller which - implements pinmux for AON pins and pinconf for all pins. - -The following pin configuration are supported. Please refer to -pinctrl-bindings.txt in this directory for more details of the common -pinctrl bindings used by client devices. - -- bias-pull-up -- bias-pull-down -- drive-strength -- input-enable -- slew-rate - -Examples: - -iocfg: pin-controller@119000 { - compatible = "zte,zx296718-iocfg"; - reg = <0x119000 0x1000>; -}; - -pmm: pin-controller@1462000 { - compatible = "zte,zx296718-pmm"; - reg = <0x1462000 0x1000>; - zte,auxiliary-controller = <&iocfg>; -}; - -&pmm { - vga_pins: vga { - pins = "KEY_COL1", "KEY_COL2", "KEY_ROW1", "KEY_ROW2"; - function = "VGA"; - }; -}; - -&vga { - pinctrl-names = "default"; - pinctrl-0 = <&vga_pins>; -}; diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt index 448d36a85730..0ba07bc96c55 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt @@ -8,6 +8,7 @@ of PMIC's from Qualcomm. Value type: Definition: Should contain one of: "qcom,pm8018-mpp", + "qcom,pm8019-mpp", "qcom,pm8038-mpp", "qcom,pm8058-mpp", "qcom,pm8821-mpp", diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sc8180x-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sc8180x-pinctrl.yaml new file mode 100644 index 000000000000..a82dab898395 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sc8180x-pinctrl.yaml @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,sc8180x-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. SC8180X TLMM block + +maintainers: + - Bjorn Andersson + +description: | + This binding describes the Top Level Mode Multiplexer block found in the + SC8180X platform. + +allOf: + - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml# + +properties: + compatible: + const: qcom,sc8180x-tlmm + + reg: + maxItems: 3 + + reg-names: + items: + - const: "west" + - const: "east" + - const: "south" + + interrupts: true + interrupt-controller: true + '#interrupt-cells': true + gpio-controller: true + gpio-reserved-ranges: true + '#gpio-cells': true + gpio-ranges: true + wakeup-parent: true + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +patternProperties: + '-state$': + oneOf: + - $ref: "#/$defs/qcom-sc8180x-tlmm-state" + - patternProperties: + ".*": + $ref: "#/$defs/qcom-sc8180x-tlmm-state" + +'$defs': + qcom-sc8180x-tlmm-state: + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + + properties: + pins: + description: + List of gpio pins affected by the properties specified in this + subnode. + items: + oneOf: + - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-8][0-9])$" + - enum: [ sdc2_clk, sdc2_cmd, sdc2_data, ufs_reset ] + minItems: 1 + maxItems: 16 + + function: + description: + Specify the alternative function to be configured for the specified + pins. + + enum: [ adsp_ext, agera_pll, aoss_cti, atest_char, atest_tsens, + atest_tsens2, atest_usb0, atest_usb1, atest_usb2, atest_usb3, + atest_usb4, audio_ref, btfm_slimbus, cam_mclk, cci_async, + cci_i2c, cci_timer0, cci_timer1, cci_timer2, cci_timer3, + cci_timer4, cci_timer5, cci_timer6, cci_timer7, cci_timer8, + cci_timer9, cri_trng, dbg_out, ddr_bist, ddr_pxi, debug_hot, + dp_hot, edp_hot, edp_lcd, emac_phy, emac_pps, gcc_gp1, gcc_gp2, + gcc_gp3, gcc_gp4, gcc_gp5, gpio, gps, grfc, hs1_mi2s, hs2_mi2s, + hs3_mi2s, jitter_bist, lpass_slimbus, m_voc, mdp_vsync, + mdp_vsync0, mdp_vsync1, mdp_vsync2, mdp_vsync3, mdp_vsync4, + mdp_vsync5, mss_lte, nav_pps, pa_indicator, pci_e0, pci_e1, + pci_e2, pci_e3, phase_flag, pll_bist, pll_bypassnl, pll_reset, + pri_mi2s, pri_mi2s_ws, prng_rosc, qdss_cti, qdss_gpio, qlink, + qspi0, qspi0_clk, qspi0_cs, qspi1, qspi1_clk, qspi1_cs, + qua_mi2s, qup0, qup1, qup2, qup3, qup4, qup5, qup6, qup7, qup8, + qup9, qup10, qup11, qup12, qup13, qup14, qup15, qup16, qup17, + qup18, qup19, qup_l4, qup_l5, qup_l6, rgmii, sd_write, sdc4, + sdc4_clk, sdc4_cmd, sec_mi2s, sp_cmu, spkr_i2s, ter_mi2s, tgu, + tsense_pwm1, tsense_pwm2, tsif1, tsif2, uim1, uim2, uim_batt, + usb0_phy, usb1_phy, usb2phy_ac, vfr_1, vsense_trigger, + wlan1_adc, wlan2_adc, wmss_reset ] + + bias-disable: true + bias-pull-down: true + bias-pull-up: true + drive-strength: true + input-enable: true + output-high: true + output-low: true + + required: + - pins + - function + + additionalProperties: false + +examples: + - | + #include + pinctrl@3100000 { + compatible = "qcom,sc8180x-tlmm"; + reg = <0x03100000 0x300000>, + <0x03500000 0x700000>, + <0x03d00000 0x300000>; + reg-names = "west", "east", "south"; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 190>; + + gpio-wo-subnode-state { + pins = "gpio1"; + function = "gpio"; + }; + + uart-w-subnodes-state { + rx { + pins = "gpio4"; + function = "qup6"; + bias-pull-up; + }; + + tx { + pins = "gpio5"; + function = "qup6"; + bias-disable; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm8350-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm8350-pinctrl.yaml new file mode 100644 index 000000000000..4f2667ea2805 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm8350-pinctrl.yaml @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,sm8350-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. SM8350 TLMM block + +maintainers: + - Vinod Koul + +description: | + This binding describes the Top Level Mode Multiplexer (TLMM) block found + in the SM8350 platform. + +allOf: + - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml# + +properties: + compatible: + const: qcom,sm8350-tlmm + + reg: + maxItems: 1 + + interrupts: true + interrupt-controller: true + '#interrupt-cells': true + gpio-controller: true + gpio-reserved-ranges: true + '#gpio-cells': true + gpio-ranges: true + wakeup-parent: true + +required: + - compatible + - reg + +additionalProperties: false + +patternProperties: + '-state$': + oneOf: + - $ref: "#/$defs/qcom-sm8350-tlmm-state" + - patternProperties: + ".*": + $ref: "#/$defs/qcom-sm8350-tlmm-state" + +$defs: + qcom-sm8350-tlmm-state: + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + + properties: + pins: + description: + List of gpio pins affected by the properties specified in this + subnode. + items: + oneOf: + - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|20[0-3])$" + - enum: [ sdc1_clk, sdc1_cmd, sdc1_data, sdc2_clk, sdc2_cmd, sdc2_data ] + minItems: 1 + maxItems: 36 + + function: + description: + Specify the alternative function to be configured for the specified + pins. + + enum: [ atest_char, atest_usb, audio_ref, cam_mclk, cci_async, + cci_i2c, cci_timer, cmu_rng, coex_uart1, coex_uart2, cri_trng, + cri_trng0, cri_trng1, dbg_out, ddr_bist, ddr_pxi0, ddr_pxi1, + ddr_pxi2, ddr_pxi3, dp_hot, dp_lcd, gcc_gp1, gcc_gp2, gcc_gp3, + gpio, ibi_i3c, jitter_bist, lpass_slimbus, mdp_vsync, mdp_vsync0, + mdp_vsync1, mdp_vsync2, mdp_vsync3, mi2s0_data0, mi2s0_data1, + mi2s0_sck, mi2s0_ws, mi2s1_data0, mi2s1_data1, mi2s1_sck, + mi2s1_ws, mi2s2_data0, mi2s2_data1, mi2s2_sck, mi2s2_ws, + mss_grfc0, mss_grfc1, mss_grfc10, mss_grfc11, mss_grfc12, + mss_grfc2, mss_grfc3, mss_grfc4, mss_grfc5, mss_grfc6, + mss_grfc7, mss_grfc8, mss_grfc9, nav_gpio, pa_indicator, + pcie0_clkreqn, pcie1_clkreqn, phase_flag, pll_bist, pll_clk, + pri_mi2s, prng_rosc, qdss_cti, qdss_gpio, qlink0_enable, + qlink0_request, qlink0_wmss, qlink1_enable, qlink1_request, + qlink1_wmss, qlink2_enable, qlink2_request, qlink2_wmss, qspi0, + qspi1, qspi2, qspi3, qspi_clk, qspi_cs, qup0, qup1, qup10, + qup11, qup12, qup13, qup14, qup15, qup16, qup17, qup18, qup19, + qup2, qup3, qup4, qup5, qup6, qup7, qup8, qup9, qup_l4, qup_l5, + qup_l6, sd_write, sdc40, sdc41, sdc42, sdc43, sdc4_clk, + sdc4_cmd, sec_mi2s, tb_trig, tgu_ch0, tgu_ch1, tgu_ch2, + tgu_ch3, tsense_pwm1, tsense_pwm2, uim0_clk, uim0_data, + uim0_present, uim0_reset, uim1_clk, uim1_data, uim1_present, + uim1_reset, usb2phy_ac, usb_phy, vfr_0, vfr_1, vsense_trigger ] + + + bias-disable: true + bias-pull-down: true + bias-pull-up: true + drive-strength: true + input-enable: true + output-high: true + output-low: true + + required: + - pins + - function + + additionalProperties: false + +examples: + - | + #include + pinctrl@f100000 { + compatible = "qcom,sm8350-tlmm"; + reg = <0x0f100000 0x300000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 203>; + + gpio-wo-subnode-state { + pins = "gpio1"; + function = "gpio"; + }; + + uart-w-subnodes-state { + rx { + pins = "gpio18"; + function = "qup3"; + bias-pull-up; + }; + + tx { + pins = "gpio19"; + function = "qup3"; + bias-disable; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,tlmm-common.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,tlmm-common.yaml new file mode 100644 index 000000000000..3b37cf102d41 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,tlmm-common.yaml @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,tlmm-common.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. Top Level Mode Multiplexer (TLMM) definitions + +maintainers: + - Bjorn Andersson + +description: + This defines the common properties used to describe all Qualcomm Top Level + Mode Multiplexer bindings and pinconf/pinmux states for these. + +properties: + interrupts: + description: + Specifies the TLMM summary IRQ + maxItems: 1 + + interrupt-controller: true + + '#interrupt-cells': + description: + Specifies the PIN numbers and Flags, as defined in defined in + include/dt-bindings/interrupt-controller/irq.h + const: 2 + + gpio-controller: true + + '#gpio-cells': + description: + Specifying the pin number and flags, as defined in + include/dt-bindings/gpio/gpio.h + const: 2 + + gpio-ranges: + maxItems: 1 + + wakeup-parent: + description: + Specifying the interrupt-controller used to wake up the system when the + TLMM block has been powered down. + maxItems: 1 + + gpio-reserved-ranges: + description: + Pins can be reserved for trusted applications and thereby unaccessible + from the OS. This property can be used to mark the pins which resources + should not be accessed by the OS. Please see the ../gpio/gpio.txt for more + information. + +required: + - interrupts + - interrupt-controller + - '#interrupt-cells' + - gpio-controller + - '#gpio-cells' + - gpio-ranges + +additionalProperties: true + +$defs: + qcom-tlmm-state: + allOf: + - $ref: pincfg-node.yaml# + - $ref: pinmux-node.yaml# + + properties: + drive-strength: + enum: [2, 4, 6, 8, 10, 12, 14, 16] + default: 2 + description: + Selects the drive strength for the specified pins, in mA. + + bias-pull-down: true + bias-pull-up: true + bias-disable: true + input-enable: true + output-high: true + output-low: true + + additionalProperties: true +... diff --git a/Documentation/devicetree/bindings/pinctrl/ralink,rt2880-pinmux.yaml b/Documentation/devicetree/bindings/pinctrl/ralink,rt2880-pinmux.yaml index 7dea3e26d99e..b32f2676cab5 100644 --- a/Documentation/devicetree/bindings/pinctrl/ralink,rt2880-pinmux.yaml +++ b/Documentation/devicetree/bindings/pinctrl/ralink,rt2880-pinmux.yaml @@ -15,39 +15,38 @@ description: properties: compatible: - enum: - - ralink,rt2880-pinmux + const: ralink,rt2880-pinmux - pinctrl-0: - description: - A phandle to the node containing the subnodes containing default - configurations. This is for pinctrl hogs. +patternProperties: + '-pins$': + type: object + patternProperties: + '^(.*-)?pinmux$': + type: object + description: node for pinctrl. + $ref: pinmux-node.yaml# - pinctrl-names: - description: - A pinctrl state named "default" can be defined. - const: default + properties: + groups: + description: Name of the pin group to use for the functions. + enum: [i2c, spi, uart1, uart2, uart3, rgmii1, rgmii2, mdio, + pcie, sdhci] + function: + description: The mux function to select + enum: [gpio, i2c, spi, uart1, uart2, uart3, rgmii1, rgmii2, + mdio, nand1, nand2, sdhci] + + required: + - groups + - function + + additionalProperties: false + + additionalProperties: false required: - compatible -patternProperties: - '[a-z0-9_-]+': - if: - type: object - description: node for pinctrl. - $ref: "pinmux-node.yaml" - then: - properties: - groups: - description: Name of the pin group to use for the functions. - enum: [i2c, spi, uart1, uart2, uart3, rgmii1, rgmii2, mdio, - pcie, sdhci] - function: - description: The mux function to select - enum: [gpio, i2c, spi, uart1, uart2, uart3, rgmii1, rgmii2, - mdio, nand1, nand2, sdhci] - additionalProperties: false examples: @@ -55,14 +54,9 @@ examples: - | pinctrl { compatible = "ralink,rt2880-pinmux"; - pinctrl-names = "default"; - pinctrl-0 = <&state_default>; - state_default: pinctrl0 { - }; - - i2c_pins: i2c0 { - i2c0 { + i2c_pins: i2c0-pins { + pinmux { groups = "i2c"; function = "i2c"; }; diff --git a/Documentation/devicetree/bindings/pinctrl/renesas,pfc.yaml b/Documentation/devicetree/bindings/pinctrl/renesas,pfc.yaml index 5b5b1b9d2ec7..ac4e068aa03f 100644 --- a/Documentation/devicetree/bindings/pinctrl/renesas,pfc.yaml +++ b/Documentation/devicetree/bindings/pinctrl/renesas,pfc.yaml @@ -43,11 +43,12 @@ properties: - renesas,pfc-r8a77980 # R-Car V3H - renesas,pfc-r8a77990 # R-Car E3 - renesas,pfc-r8a77995 # R-Car D3 + - renesas,pfc-r8a779a0 # R-Car V3U - renesas,pfc-sh73a0 # SH-Mobile AG5 reg: minItems: 1 - maxItems: 2 + maxItems: 10 gpio-controller: true @@ -76,11 +77,10 @@ required: if: properties: compatible: - items: - enum: - - renesas,pfc-r8a73a4 - - renesas,pfc-r8a7740 - - renesas,pfc-sh73a0 + enum: + - renesas,pfc-r8a73a4 + - renesas,pfc-r8a7740 + - renesas,pfc-sh73a0 then: required: - interrupts-extended diff --git a/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt index 7734ab6fec44..38a1416fd2cd 100644 --- a/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt +++ b/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt @@ -336,7 +336,7 @@ Example 3: A uart client node that supports 'default' and 'flow-control' states. interrupts = <0 52 0>; pinctrl-names = "default", "flow-control; pinctrl-0 = <&uart0_data>; - pinctrl-1 = <&uart0_data &uart0_fctl>; + pinctrl-1 = <&uart0_data>, <&uart0_fctl>; }; Example 4: Set up the default pin state for uart controller. diff --git a/Documentation/devicetree/bindings/power/renesas,apmu.yaml b/Documentation/devicetree/bindings/power/renesas,apmu.yaml index 60a23b3beb40..391897d897f2 100644 --- a/Documentation/devicetree/bindings/power/renesas,apmu.yaml +++ b/Documentation/devicetree/bindings/power/renesas,apmu.yaml @@ -52,5 +52,5 @@ examples: apmu@e6152000 { compatible = "renesas,r8a7791-apmu", "renesas,apmu"; reg = <0xe6152000 0x188>; - cpus = <&cpu0 &cpu1>; + cpus = <&cpu0>, <&cpu1>; }; diff --git a/Documentation/devicetree/bindings/power/supply/bq256xx.yaml b/Documentation/devicetree/bindings/power/supply/bq256xx.yaml new file mode 100644 index 000000000000..18b54783e11a --- /dev/null +++ b/Documentation/devicetree/bindings/power/supply/bq256xx.yaml @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: (GPL-2.0-only or BSD-2-Clause) +# Copyright (C) 2020 Texas Instruments Incorporated +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/power/supply/bq256xx.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: TI bq256xx Switch Mode Buck Charger + +maintainers: + - Ricardo Rivera-Matos + +description: | + The bq256xx devices are a family of highly-integrated battery charge + management and system power management ICs for single cell Li-ion and Li- + polymer batteries. + + Datasheets: + - https://www.ti.com/lit/ds/symlink/bq25600.pdf + - https://www.ti.com/lit/ds/symlink/bq25601.pdf + - https://www.ti.com/lit/ds/symlink/bq25600d.pdf + - https://www.ti.com/lit/ds/symlink/bq25601d.pdf + - https://www.ti.com/lit/ds/symlink/bq25611d.pdf + - https://www.ti.com/lit/ds/symlink/bq25618.pdf + - https://www.ti.com/lit/ds/symlink/bq25619.pdf + +properties: + compatible: + enum: + - ti,bq25600 + - ti,bq25601 + - ti,bq25600d + - ti,bq25601d + - ti,bq25611d + - ti,bq25618 + - ti,bq25619 + + reg: + maxItems: 1 + + ti,watchdog-timeout-ms: + $ref: /schemas/types.yaml#/definitions/uint32 + default: 0 + description: | + Watchdog timer in ms. 0 (default) disables the watchdog + minimum: 0 + maximum: 160000 + enum: [ 0, 40000, 80000, 160000] + + input-voltage-limit-microvolt: + description: | + Minimum input voltage limit in µV with a 100000 µV step + minimum: 3900000 + maximum: 5400000 + + input-current-limit-microamp: + description: | + Maximum input current limit in µA with a 100000 µA step + minimum: 100000 + maximum: 3200000 + + monitored-battery: + $ref: /schemas/types.yaml#/definitions/phandle + description: phandle to the battery node being monitored + + interrupts: + maxItems: 1 + description: | + Interrupt sends an active low, 256 μs pulse to host to report the charger + device status and faults. + +required: + - compatible + - reg + - monitored-battery + +additionalProperties: false + +examples: + - | + bat: battery { + compatible = "simple-battery"; + constant-charge-current-max-microamp = <2040000>; + constant-charge-voltage-max-microvolt = <4352000>; + precharge-current-microamp = <180000>; + charge-term-current-microamp = <180000>; + }; + #include + #include + i2c { + + clock-frequency = <400000>; + + #address-cells = <1>; + #size-cells = <0>; + + charger@6b { + compatible = "ti,bq25601"; + reg = <0x6b>; + monitored-battery = <&bat>; + + interrupt-parent = <&gpio1>; + interrupts = <16 IRQ_TYPE_EDGE_FALLING>; + ti,watchdog-timeout-ms = <40000>; + + input-voltage-limit-microvolt = <4500000>; + input-current-limit-microamp = <2400000>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/power/supply/bq25980.yaml b/Documentation/devicetree/bindings/power/supply/bq25980.yaml index f6b3dd4093ca..06eca6667f67 100644 --- a/Documentation/devicetree/bindings/power/supply/bq25980.yaml +++ b/Documentation/devicetree/bindings/power/supply/bq25980.yaml @@ -70,6 +70,7 @@ properties: description: Enables bypass mode at boot time interrupts: + maxItems: 1 description: | Indicates that the device state has changed. diff --git a/Documentation/devicetree/bindings/power/supply/ltc4162-l.yaml b/Documentation/devicetree/bindings/power/supply/ltc4162-l.yaml new file mode 100644 index 000000000000..1f88c9e013f4 --- /dev/null +++ b/Documentation/devicetree/bindings/power/supply/ltc4162-l.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +# Copyright (C) 2020 Topic Embedded Products +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/power/supply/ltc4162-l.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Linear Technology (Analog Devices) LTC4162-L Charger + +maintainers: + - Mike Looijmans + +description: | + The LTC ® 4162-L is an advanced monolithic synchronous step-down switching + battery charger and PowerPath (TM) manager that seamlessly manages power + distribution between input sources such as wall adapters, backplanes, solar + panels, etc., and a rechargeable Lithium-Ion/Polymer battery. + + Specifications about the charger can be found at: + https://www.analog.com/en/products/ltc4162-s.html + +properties: + compatible: + enum: + - lltc,ltc4162-l + + reg: + maxItems: 1 + description: I2C address of the charger. + + lltc,rsnsb-micro-ohms: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Battery sense resistor in microohm. + minimum: 1000 + + lltc,rsnsi-micro-ohms: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Input current sense resistor in microohm. + minimum: 1000 + + lltc,cell-count: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + Number of battery cells. If not provided, will be obtained from the chip + once the external power is applied. Omit this when the number of cells + is somewhat dynamic. Without it, several measurements will return 0 until + the charger is connected to an external supply. + +required: + - compatible + - reg + - lltc,rsnsb-micro-ohms + - lltc,rsnsi-micro-ohms + +additionalProperties: false + +examples: + - | + i2c0 { + #address-cells = <1>; + #size-cells = <0>; + charger: battery-charger@68 { + compatible = "lltc,ltc4162-l"; + reg = <0x68>; + lltc,rsnsb-micro-ohms = <10000>; + lltc,rsnsi-micro-ohms = <16000>; + lltc,cell-count = <2>; + }; + }; diff --git a/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml b/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml index 239b49fad805..658cec67743e 100644 --- a/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml +++ b/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml @@ -59,9 +59,7 @@ additionalProperties: false examples: - | - i2c@1 { - compatible = "abc,acme-1234"; - reg = <0x01 0x400>; + i2c { #address-cells = <1>; #size-cells = <0>; phc@5b { diff --git a/Documentation/devicetree/bindings/pwm/pwm-sifive.yaml b/Documentation/devicetree/bindings/pwm/pwm-sifive.yaml index 5ac25275d8bf..84e66913d042 100644 --- a/Documentation/devicetree/bindings/pwm/pwm-sifive.yaml +++ b/Documentation/devicetree/bindings/pwm/pwm-sifive.yaml @@ -25,12 +25,15 @@ description: properties: compatible: items: - - const: sifive,fu540-c000-pwm + - enum: + - sifive,fu540-c000-pwm + - sifive,fu740-c000-pwm - const: sifive,pwm0 description: Should be "sifive,-pwm" and "sifive,pwm". Supported - compatible strings are "sifive,fu540-c000-pwm" for the SiFive PWM v0 - as integrated onto the SiFive FU540 chip, and "sifive,pwm0" for the + compatible strings are "sifive,fu540-c000-pwm" and + "sifive,fu740-c000-pwm" for the SiFive PWM v0 as integrated onto the + SiFive FU540 and FU740 chip respectively, and "sifive,pwm0" for the SiFive PWM v0 IP block with no chip integration tweaks. Please refer to sifive-blocks-ip-versioning.txt for details. diff --git a/Documentation/devicetree/bindings/pwm/pwm-zx.txt b/Documentation/devicetree/bindings/pwm/pwm-zx.txt deleted file mode 100644 index 3c8fe7aa8269..000000000000 --- a/Documentation/devicetree/bindings/pwm/pwm-zx.txt +++ /dev/null @@ -1,22 +0,0 @@ -ZTE ZX PWM controller - -Required properties: - - compatible: Should be "zte,zx296718-pwm". - - reg: Physical base address and length of the controller's registers. - - clocks : The phandle and specifier referencing the controller's clocks. - - clock-names: "pclk" for PCLK, "wclk" for WCLK to the PWM controller. The - PCLK is for register access, while WCLK is the reference clock for - calculating period and duty cycles. - - #pwm-cells: Should be 3. See pwm.yaml in this directory for a description of - the cells format. - -Example: - - pwm: pwm@1439000 { - compatible = "zte,zx296718-pwm"; - reg = <0x1439000 0x1000>; - clocks = <&lsp1crm LSP1_PWM_PCLK>, - <&lsp1crm LSP1_PWM_WCLK>; - clock-names = "pclk", "wclk"; - #pwm-cells = <3>; - }; diff --git a/Documentation/devicetree/bindings/regulator/max8997-regulator.txt b/Documentation/devicetree/bindings/regulator/max8997-regulator.txt index 6fe825b8ac1b..b53c5e2b335f 100644 --- a/Documentation/devicetree/bindings/regulator/max8997-regulator.txt +++ b/Documentation/devicetree/bindings/regulator/max8997-regulator.txt @@ -35,6 +35,7 @@ Optional properties: - interrupts: Interrupt specifiers for two interrupt sources. - First interrupt specifier is for 'irq1' interrupt. - Second interrupt specifier is for 'alert' interrupt. +- charger-supply: regulator node for charging current. - max8997,pmic-buck1-uses-gpio-dvs: 'buck1' can be controlled by gpio dvs. - max8997,pmic-buck2-uses-gpio-dvs: 'buck2' can be controlled by gpio dvs. - max8997,pmic-buck5-uses-gpio-dvs: 'buck5' can be controlled by gpio dvs. diff --git a/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt b/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt index d86584ed4d93..451cc4e86b01 100644 --- a/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt +++ b/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt @@ -4,7 +4,8 @@ Required properties: - compatible: "microchip,mcp16502" - reg: I2C slave address - lpm-gpios: GPIO for LPM pin. Note that this GPIO *must* remain high during - suspend-to-ram, keeping the PMIC into HIBERNATE mode. + suspend-to-ram, keeping the PMIC into HIBERNATE mode; this + property is optional; - regulators: A node that houses a sub-node for each regulator within the device. Each sub-node is identified using the node's name. The content of each sub-node is defined by the diff --git a/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml b/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml new file mode 100644 index 000000000000..61dd5af80db6 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/mt6315-regulator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Mediatek MT6315 Regulator + +maintainers: + - Hsin-Hsiung Wang + +description: | + The MT6315 is a power management IC (PMIC) configurable with SPMI. + that contains 4 BUCKs output which can combine with each other + by different efuse settings. + +properties: + compatible: + const: mediatek,mt6315-regulator + + reg: + maxItems: 1 + + regulators: + type: object + description: List of regulators and its properties + + patternProperties: + "^vbuck[1-4]$": + type: object + $ref: "regulator.yaml#" + + properties: + regulator-name: + pattern: "^vbuck[1-4]$" + + additionalProperties: false + +required: + - compatible + - reg + - regulators + +additionalProperties: false + +examples: + - | + pmic@6 { + compatible = "mediatek,mt6315-regulator"; + reg = <0x6 0>; + + regulators { + vbuck1 { + regulator-compatible = "vbuck1"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1193750>; + regulator-enable-ramp-delay = <256>; + regulator-allowed-modes = <0 1 2 4>; + }; + + vbuck3 { + regulator-compatible = "vbuck3"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1193750>; + regulator-enable-ramp-delay = <256>; + regulator-allowed-modes = <0 1 2 4>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/regulator/nxp,pca9450-regulator.yaml b/Documentation/devicetree/bindings/regulator/nxp,pca9450-regulator.yaml index c2b0a8b6da1e..f70f2e758a00 100644 --- a/Documentation/devicetree/bindings/regulator/nxp,pca9450-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/nxp,pca9450-regulator.yaml @@ -87,6 +87,11 @@ properties: additionalProperties: false + sd-vsel-gpios: + description: GPIO that is used to switch LDO5 between being configured by + LDO5CTRL_L or LDO5CTRL_H register. Use this if the SD_VSEL signal is + connected to a host GPIO. + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/regulator/nxp,pf8x00-regulator.yaml b/Documentation/devicetree/bindings/regulator/nxp,pf8x00-regulator.yaml index 956156fe52a3..8761437ed8ad 100644 --- a/Documentation/devicetree/bindings/regulator/nxp,pf8x00-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/nxp,pf8x00-regulator.yaml @@ -62,8 +62,11 @@ properties: $ref: "/schemas/types.yaml#/definitions/uint32" minimum: 2100 maximum: 4500 + deprecated: true description: BUCK regulators current limit in mA. + This property is deprecated, please use + "regulator-max-microamp" instead. Listed current limits in mA are, 2100 (default) @@ -73,21 +76,11 @@ properties: nxp,phase-shift: $ref: "/schemas/types.yaml#/definitions/uint32" - minimum: 45 - maximum: 0 + default: 0 + enum: [ 0, 45, 90, 135, 180, 225, 270, 315 ] description: BUCK regulators phase shift control in degrees. - Listed phase shift control values in degrees are, - 45 - 90 - 135 - 180 - 225 - 270 - 315 - 0 (default) - unevaluatedProperties: false "^vsnvs$": diff --git a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.txt b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.txt index 7d462b899473..ce1e04354006 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.txt +++ b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.txt @@ -50,6 +50,8 @@ First Level Nodes - PMIC "qcom,pm8350-rpmh-regulators" "qcom,pm8350c-rpmh-regulators" "qcom,pm8998-rpmh-regulators" + "qcom,pmc8180-rpmh-regulators" + "qcom,pmc8180c-rpmh-regulators" "qcom,pmi8998-rpmh-regulators" "qcom,pm6150-rpmh-regulators" "qcom,pm6150l-rpmh-regulators" diff --git a/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml index 53853ec20fe2..cf784bd1f5e5 100644 --- a/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml @@ -22,11 +22,17 @@ properties: type: object properties: + qcom,soft-start-us: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Regulator soft start time in microseconds. + enum: [200, 400, 600, 800] + default: 200 interrupts: - maxItems: 1 + minItems: 1 + maxItems: 2 description: - Short-circuit interrupt for lab. + Short-circuit and over-current interrupts for lab. required: - interrupts @@ -35,11 +41,17 @@ properties: type: object properties: + qcom,discharge-resistor-kohms: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Discharge resistor value in KiloOhms. + enum: [300, 64, 32, 16] + default: 300 interrupts: - maxItems: 1 + minItems: 1 + maxItems: 2 description: - Short-circuit interrupt for lab. + Short-circuit and over-current interrupts for ibb. required: - interrupts @@ -57,13 +69,15 @@ examples: compatible = "qcom,pmi8998-lab-ibb"; lab { - interrupts = <0x3 0x0 IRQ_TYPE_EDGE_RISING>; - interrupt-names = "sc-err"; + interrupts = <0x3 0xde 0x1 IRQ_TYPE_EDGE_RISING>, + <0x3 0xde 0x0 IRQ_TYPE_LEVEL_LOW>; + interrupt-names = "sc-err", "ocp"; }; ibb { - interrupts = <0x3 0x2 IRQ_TYPE_EDGE_RISING>; - interrupt-names = "sc-err"; + interrupts = <0x3 0xdc 0x2 IRQ_TYPE_EDGE_RISING>, + <0x3 0xdc 0x0 IRQ_TYPE_LEVEL_LOW>; + interrupt-names = "sc-err", "ocp"; }; }; diff --git a/Documentation/devicetree/bindings/regulator/richtek,rt4831-regulator.yaml b/Documentation/devicetree/bindings/regulator/richtek,rt4831-regulator.yaml new file mode 100644 index 000000000000..d9c23333e157 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/richtek,rt4831-regulator.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/richtek,rt4831-regulator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Richtek RT4831 Display Bias Voltage Regulator + +maintainers: + - ChiYuan Huang + +description: | + RT4831 is a multifunctional device that can provide power to the LCD display + and LCD backlight. + + For Display Bias Voltage DSVP and DSVN, the output range is about 4V to 6.5V. + It is sufficient to meet the current LCD power requirement. + + DSVLCM is a boost regulator in IC internal as DSVP and DSVN input power. + Its voltage should be configured above 0.15V to 0.2V gap larger than the + voltage needed for DSVP and DSVN. Too much voltage gap could improve the + voltage drop from the heavy loading scenario. But it also make the power + efficiency worse. It's a trade-off. + + Datasheet is available at + https://www.richtek.com/assets/product_file/RT4831A/DS4831A-05.pdf + +patternProperties: + "^DSV(LCM|P|N)$": + type: object + $ref: regulator.yaml# + description: + Properties for single Display Bias Voltage regulator. + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/remoteproc/ingenic,vpu.yaml b/Documentation/devicetree/bindings/remoteproc/ingenic,vpu.yaml index c019f9fbe916..d0aa91bbf5e5 100644 --- a/Documentation/devicetree/bindings/remoteproc/ingenic,vpu.yaml +++ b/Documentation/devicetree/bindings/remoteproc/ingenic,vpu.yaml @@ -44,7 +44,7 @@ properties: - const: vpu interrupts: - description: VPU hardware interrupt + maxItems: 1 required: - compatible diff --git a/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt b/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt index 3ba668bab14b..3f5f78764b60 100644 --- a/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt +++ b/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt @@ -6,10 +6,10 @@ Mediatek SoCs. Required properties: - compatible Should be "mediatek,mt8183-scp" -- reg Should contain the address ranges for the two memory - regions, SRAM and CFG. -- reg-names Contains the corresponding names for the two memory - regions. These should be named "sram" & "cfg". +- reg Should contain the address ranges for memory regions: + SRAM, CFG, and L1TCM. +- reg-names Contains the corresponding names for the memory regions: + "sram", "cfg", and "l1tcm". - clocks Clock for co-processor (See: ../clock/clock-bindings.txt) - clock-names Contains the corresponding name for the clock. This should be named "main". diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt index 54737024da20..1c330a8941f9 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt @@ -25,6 +25,10 @@ on the Qualcomm ADSP Hexagon core. "qcom,sm8250-adsp-pas" "qcom,sm8250-cdsp-pas" "qcom,sm8250-slpi-pas" + "qcom,sm8350-adsp-pas" + "qcom,sm8350-cdsp-pas" + "qcom,sm8350-slpi-pas" + "qcom,sm8350-mpss-pas" - interrupts-extended: Usage: required @@ -51,10 +55,14 @@ on the Qualcomm ADSP Hexagon core. qcom,sm8250-adsp-pas: qcom,sm8250-cdsp-pas: qcom,sm8250-slpi-pas: + qcom,sm8350-adsp-pas: + qcom,sm8350-cdsp-pas: + qcom,sm8350-slpi-pas: must be "wdog", "fatal", "ready", "handover", "stop-ack" qcom,qcs404-wcss-pas: qcom,sc7180-mpss-pas: qcom,sm8150-mpss-pas: + qcom,sm8350-mpss-pas: must be "wdog", "fatal", "ready", "handover", "stop-ack", "shutdown-ack" @@ -114,13 +122,17 @@ on the Qualcomm ADSP Hexagon core. qcom,sm8150-adsp-pas: qcom,sm8150-cdsp-pas: qcom,sm8250-cdsp-pas: + qcom,sm8350-cdsp-pas: must be "cx", "load_state" qcom,sc7180-mpss-pas: qcom,sm8150-mpss-pas: + qcom,sm8350-mpss-pas: must be "cx", "load_state", "mss" qcom,sm8250-adsp-pas: + qcom,sm8350-adsp-pas: qcom,sm8150-slpi-pas: qcom,sm8250-slpi-pas: + qcom,sm8350-slpi-pas: must be "lcx", "lmx", "load_state" - memory-region: diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt b/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt index cc0b7fc1c29b..da09c0d79ac0 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt @@ -80,6 +80,7 @@ and its resource dependencies. It is described by the following properties: Definition: must be one of: "qcom,wcn3620", "qcom,wcn3660", + "qcom,wcn3660b", "qcom,wcn3680" - clocks: diff --git a/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml b/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml index 084960a8f17a..1a1159097a2a 100644 --- a/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml +++ b/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml @@ -70,10 +70,13 @@ properties: the firmware image. clocks: + maxItems: 1 description: | Main functional clock for the remote processor resets: + minItems: 1 + maxItems: 2 description: | Reset handles for the remote processor diff --git a/Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml b/Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml new file mode 100644 index 000000000000..53e4ede9c0bd --- /dev/null +++ b/Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/reset/canaan,k210-rst.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Canaan Kendryte K210 Reset Controller Device Tree Bindings + +maintainers: + - Damien Le Moal + +description: | + Canaan Kendryte K210 reset controller driver which supports the SoC + system controller supplied reset registers for the various peripherals + of the SoC. The K210 reset controller node must be defined as a child + node of the K210 system controller node. + + See also: + - dt-bindings/reset/k210-rst.h + +properties: + compatible: + const: canaan,k210-rst + + '#reset-cells': + const: 1 + +required: + - '#reset-cells' + - compatible + +additionalProperties: false + +examples: + - | + #include + sysrst: reset-controller { + compatible = "canaan,k210-rst"; + #reset-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/riscv/canaan.yaml b/Documentation/devicetree/bindings/riscv/canaan.yaml new file mode 100644 index 000000000000..f8f3f286bd55 --- /dev/null +++ b/Documentation/devicetree/bindings/riscv/canaan.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/riscv/canaan.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Canaan SoC-based boards + +maintainers: + - Damien Le Moal + +description: + Canaan Kendryte K210 SoC-based boards + +properties: + $nodename: + const: '/' + compatible: + oneOf: + - items: + - const: sipeed,maix-bit + - const: sipeed,maix-bitm + - const: canaan,kendryte-k210 + + - items: + - const: sipeed,maix-go + - const: canaan,kendryte-k210 + + - items: + - const: sipeed,maix-dock-m1 + - const: sipeed,maix-dock-m1w + - const: canaan,kendryte-k210 + + - items: + - const: sipeed,maixduino + - const: canaan,kendryte-k210 + + - items: + - const: canaan,kendryte-kd233 + - const: canaan,kendryte-k210 + + - items: + - const: canaan,kendryte-k210 + +additionalProperties: true + +... diff --git a/Documentation/devicetree/bindings/riscv/cpus.yaml b/Documentation/devicetree/bindings/riscv/cpus.yaml index c6925e0b16e4..e534f6a7cfa1 100644 --- a/Documentation/devicetree/bindings/riscv/cpus.yaml +++ b/Documentation/devicetree/bindings/riscv/cpus.yaml @@ -28,11 +28,18 @@ properties: - items: - enum: - sifive,rocket0 + - sifive,bullet0 - sifive,e5 + - sifive,e7 - sifive,e51 + - sifive,e71 - sifive,u54-mc + - sifive,u74-mc - sifive,u54 + - sifive,u74 - sifive,u5 + - sifive,u7 + - canaan,k210 - const: riscv - const: riscv # Simulator only description: @@ -50,6 +57,7 @@ properties: - riscv,sv32 - riscv,sv39 - riscv,sv48 + - riscv,none riscv,isa: description: diff --git a/Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml b/Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml index efc0198eeb74..23b227614366 100644 --- a/Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml +++ b/Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml @@ -27,6 +27,7 @@ select: items: - enum: - sifive,fu540-c000-ccache + - sifive,fu740-c000-ccache required: - compatible @@ -34,7 +35,9 @@ select: properties: compatible: items: - - const: sifive,fu540-c000-ccache + - enum: + - sifive,fu540-c000-ccache + - sifive,fu740-c000-ccache - const: cache cache-block-size: @@ -52,10 +55,13 @@ properties: cache-unified: true interrupts: - description: | - Must contain entries for DirError, DataError and DataFail signals. minItems: 3 - maxItems: 3 + maxItems: 4 + items: + - description: DirError interrupt + - description: DataError interrupt + - description: DataFail interrupt + - description: DirFail interrupt reg: maxItems: 1 @@ -63,10 +69,31 @@ properties: next-level-cache: true memory-region: + maxItems: 1 description: | The reference to the reserved-memory for the L2 Loosely Integrated Memory region. The reserved memory node should be defined as per the bindings in reserved-memory.txt. +if: + properties: + compatible: + contains: + const: sifive,fu540-c000-ccache + +then: + properties: + interrupts: + description: | + Must contain entries for DirError, DataError and DataFail signals. + maxItems: 3 + +else: + properties: + interrupts: + description: | + Must contain entries for DirError, DataError, DataFail, DirFail signals. + minItems: 4 + additionalProperties: false required: diff --git a/Documentation/devicetree/bindings/riscv/sifive.yaml b/Documentation/devicetree/bindings/riscv/sifive.yaml index 3a8647d1da4c..ee0a239af4c2 100644 --- a/Documentation/devicetree/bindings/riscv/sifive.yaml +++ b/Documentation/devicetree/bindings/riscv/sifive.yaml @@ -17,11 +17,18 @@ properties: $nodename: const: '/' compatible: - items: - - enum: - - sifive,hifive-unleashed-a00 - - const: sifive,fu540-c000 - - const: sifive,fu540 + oneOf: + - items: + - enum: + - sifive,hifive-unleashed-a00 + - const: sifive,fu540-c000 + - const: sifive,fu540 + + - items: + - enum: + - sifive,hifive-unmatched-a00 + - const: sifive,fu740-c000 + - const: sifive,fu740 additionalProperties: true diff --git a/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml b/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml index 02bbfe726c62..994de43d17fa 100644 --- a/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml @@ -20,6 +20,7 @@ properties: - atmel,sama5d4-rtc - atmel,sama5d2-rtc - microchip,sam9x60-rtc + - microchip,sama7g5-rtc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/rtc/nxp,pcf2127.yaml b/Documentation/devicetree/bindings/rtc/nxp,pcf2127.yaml new file mode 100644 index 000000000000..cde7b1675ead --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/nxp,pcf2127.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/nxp,pcf2127.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: NXP PCF2127 Real Time Clock + +allOf: + - $ref: "rtc.yaml#" + +maintainers: + - Alexandre Belloni + +properties: + compatible: + const: nxp,pcf2127 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + start-year: true + + reset-source: true + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + rtc@51 { + compatible = "nxp,pcf2127"; + reg = <0x51>; + pinctrl-0 = <&rtc_nint_pins>; + interrupts-extended = <&gpio1 16 IRQ_TYPE_LEVEL_HIGH>; + reset-source; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/rtc/pcf8563.txt b/Documentation/devicetree/bindings/rtc/pcf8563.txt index 6076fe76dbfa..0a900f7c8977 100644 --- a/Documentation/devicetree/bindings/rtc/pcf8563.txt +++ b/Documentation/devicetree/bindings/rtc/pcf8563.txt @@ -5,7 +5,8 @@ Philips PCF8563/Epson RTC8564 Real Time Clock Required properties: - compatible: Should contain "nxp,pcf8563", "epson,rtc8564" or - "microcrystal,rv8564" + "microcrystal,rv8564" or + "nxp,pca8565" - reg: I2C address for chip. Optional property: diff --git a/Documentation/devicetree/bindings/rtc/sirf,prima2-sysrtc.txt b/Documentation/devicetree/bindings/rtc/sirf,prima2-sysrtc.txt deleted file mode 100644 index 58885b55da21..000000000000 --- a/Documentation/devicetree/bindings/rtc/sirf,prima2-sysrtc.txt +++ /dev/null @@ -1,13 +0,0 @@ -SiRFSoC Real Time Clock - -Required properties: -- compatible: must be "sirf,prima2-sysrtc" -- reg: address range of rtc register set. -- interrupts: rtc alarm interrupts. - -Example: - rtc@2000 { - compatible = "sirf,prima2-sysrtc"; - reg = <0x2000 0x1000>; - interrupts = <52 53 54>; - }; diff --git a/Documentation/devicetree/bindings/rtc/stericsson,coh901331.txt b/Documentation/devicetree/bindings/rtc/stericsson,coh901331.txt deleted file mode 100644 index e615a897b20e..000000000000 --- a/Documentation/devicetree/bindings/rtc/stericsson,coh901331.txt +++ /dev/null @@ -1,16 +0,0 @@ -ST-Ericsson COH 901 331 Real Time Clock - -Required properties: -- compatible: must be "stericsson,coh901331" -- reg: address range of rtc register set. -- interrupts: rtc alarm interrupt. -- clocks: phandle to the rtc clock source - -Example: - rtc: rtc@c0017000 { - compatible = "stericsson,coh901331"; - reg = <0xc0017000 0x1000>; - interrupt-parent = <&vicb>; - interrupts = <10>; - clocks = <&rtc_clk>; - }; diff --git a/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml b/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml index c7d14de214c4..7548d8714871 100644 --- a/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml @@ -48,12 +48,8 @@ properties: - microcrystal,rv3029 # Real Time Clock - microcrystal,rv8523 - # Real-time clock - - nxp,pcf2127 - # Real-time clock - - nxp,pcf2129 - # Real-time clock - nxp,pca2129 + - nxp,pcf2129 # Real-time Clock Module - pericom,pt7c4338 # I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC diff --git a/Documentation/devicetree/bindings/serial/renesas,hscif.yaml b/Documentation/devicetree/bindings/serial/renesas,hscif.yaml index 512a84942f78..ee9804cd49bb 100644 --- a/Documentation/devicetree/bindings/serial/renesas,hscif.yaml +++ b/Documentation/devicetree/bindings/serial/renesas,hscif.yaml @@ -82,6 +82,8 @@ properties: maxItems: 1 dmas: + minItems: 2 + maxItems: 4 description: Must contain a list of pairs of references to DMA specifiers, one for transmission, and one for reception. diff --git a/Documentation/devicetree/bindings/serial/renesas,scif.yaml b/Documentation/devicetree/bindings/serial/renesas,scif.yaml index 672158906c33..22d76829f7ae 100644 --- a/Documentation/devicetree/bindings/serial/renesas,scif.yaml +++ b/Documentation/devicetree/bindings/serial/renesas,scif.yaml @@ -120,6 +120,8 @@ properties: maxItems: 1 dmas: + minItems: 2 + maxItems: 4 description: Must contain a list of pairs of references to DMA specifiers, one for transmission, and one for reception. diff --git a/Documentation/devicetree/bindings/serial/renesas,scifa.yaml b/Documentation/devicetree/bindings/serial/renesas,scifa.yaml index dbffb9534835..3c67d3202e1b 100644 --- a/Documentation/devicetree/bindings/serial/renesas,scifa.yaml +++ b/Documentation/devicetree/bindings/serial/renesas,scifa.yaml @@ -55,6 +55,8 @@ properties: maxItems: 1 dmas: + minItems: 2 + maxItems: 4 description: Must contain a list of pairs of references to DMA specifiers, one for transmission, and one for reception. diff --git a/Documentation/devicetree/bindings/serial/renesas,scifb.yaml b/Documentation/devicetree/bindings/serial/renesas,scifb.yaml index 147f8a37e02a..d5571c7a4424 100644 --- a/Documentation/devicetree/bindings/serial/renesas,scifb.yaml +++ b/Documentation/devicetree/bindings/serial/renesas,scifb.yaml @@ -55,6 +55,8 @@ properties: maxItems: 1 dmas: + minItems: 2 + maxItems: 4 description: Must contain a list of pairs of references to DMA specifiers, one for transmission, and one for reception. diff --git a/Documentation/devicetree/bindings/serial/sifive-serial.yaml b/Documentation/devicetree/bindings/serial/sifive-serial.yaml index 3ac5c7ff2758..5fa94dacbba9 100644 --- a/Documentation/devicetree/bindings/serial/sifive-serial.yaml +++ b/Documentation/devicetree/bindings/serial/sifive-serial.yaml @@ -20,6 +20,7 @@ properties: - enum: - sifive,fu540-c000-uart - sifive,fu740-c000-uart + - canaan,k210-uarths - const: sifive,uart0 description: diff --git a/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml b/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml index dd47fef9854d..559aff13ae23 100644 --- a/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml +++ b/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml @@ -88,6 +88,7 @@ properties: description: Phandle to the codec analog controls in the PRCM allwinner,pa-gpios: + maxItems: 1 description: GPIO to enable the external amplifier required: diff --git a/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml b/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml index 5095b780e2c7..837e3faa63a9 100644 --- a/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml +++ b/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml @@ -55,6 +55,7 @@ patternProperties: maxItems: 1 reg: + maxItems: 1 description: dai link address. cpu: diff --git a/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml b/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml index 1c6947294825..5fff586dc802 100644 --- a/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml +++ b/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml @@ -62,12 +62,15 @@ properties: description: Supply for the micbias on the headset mic earpath-sel-gpios: + maxItems: 1 description: GPIO for switching between tv-out and mic paths headset-detect-gpios: + maxItems: 1 description: GPIO for detection of headset insertion headset-key-gpios: + maxItems: 1 description: GPIO for detection of headset key press io-channels: diff --git a/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml b/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml index 578928e67e5c..095775c598fa 100644 --- a/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml +++ b/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml @@ -53,9 +53,11 @@ properties: description: Supply for the micbias on the Sub microphone fm-sel-gpios: + maxItems: 1 description: GPIO pin for FM selection lineout-sel-gpios: + maxItems: 1 description: GPIO pin for line out selection required: diff --git a/Documentation/devicetree/bindings/sound/tas2562.yaml b/Documentation/devicetree/bindings/sound/tas2562.yaml index 27f7132ba2ef..acd4bbe69731 100644 --- a/Documentation/devicetree/bindings/sound/tas2562.yaml +++ b/Documentation/devicetree/bindings/sound/tas2562.yaml @@ -36,10 +36,12 @@ properties: I2C address of the device can be one of these 0x4c, 0x4d, 0x4e or 0x4f shut-down-gpios: + maxItems: 1 description: GPIO used to control the state of the device. deprecated: true shutdown-gpios: + maxItems: 1 description: GPIO used to control the state of the device. interrupts: diff --git a/Documentation/devicetree/bindings/sound/tas2770.yaml b/Documentation/devicetree/bindings/sound/tas2770.yaml index 07e7f9951d2e..027bebf4e8cf 100644 --- a/Documentation/devicetree/bindings/sound/tas2770.yaml +++ b/Documentation/devicetree/bindings/sound/tas2770.yaml @@ -27,9 +27,11 @@ properties: I2C address of the device can be between 0x41 to 0x48. reset-gpio: + maxItems: 1 description: GPIO used to reset the device. shutdown-gpios: + maxItems: 1 description: GPIO used to control the state of the device. interrupts: diff --git a/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml b/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml index df18be9d7b15..54d64785aad2 100644 --- a/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml +++ b/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml @@ -35,6 +35,7 @@ properties: I2C addresss of the device can be one of these 0x4c, 0x4d, 0x4e or 0x4f reset-gpios: + maxItems: 1 description: | GPIO used for hardware reset. diff --git a/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml b/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml index 7866a655d81c..908248260afa 100644 --- a/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml +++ b/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml @@ -25,6 +25,7 @@ properties: - enum: - allwinner,sun8i-r40-spi - allwinner,sun50i-h6-spi + - allwinner,sun50i-h616-spi - const: allwinner,sun8i-h3-spi reg: diff --git a/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt b/Documentation/devicetree/bindings/spi/cadence-quadspi.txt similarity index 97% rename from Documentation/devicetree/bindings/mtd/cadence-quadspi.txt rename to Documentation/devicetree/bindings/spi/cadence-quadspi.txt index 945be7d5b236..8ace832a2d80 100644 --- a/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt +++ b/Documentation/devicetree/bindings/spi/cadence-quadspi.txt @@ -5,6 +5,7 @@ Required properties: Generic default - "cdns,qspi-nor". For TI 66AK2G SoC - "ti,k2g-qspi", "cdns,qspi-nor". For TI AM654 SoC - "ti,am654-ospi", "cdns,qspi-nor". + For Intel LGM SoC - "intel,lgm-qspi", "cdns,qspi-nor". - reg : Contains two entries, each of which is a tuple consisting of a physical address and length. The first entry is the address and length of the controller register set. The second entry is the diff --git a/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad.yaml b/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad.yaml new file mode 100644 index 000000000000..35a8045b2c70 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad.yaml @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spi/nvidia,tegra210-quad.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Tegra Quad SPI Controller + +maintainers: + - Thierry Reding + - Jonathan Hunter + +allOf: + - $ref: "spi-controller.yaml#" + +properties: + compatible: + enum: + - nvidia,tegra210-qspi + - nvidia,tegra186-qspi + - nvidia,tegra194-qspi + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clock-names: + items: + - const: qspi + - const: qspi_out + + clocks: + maxItems: 2 + + resets: + maxItems: 1 + + dmas: + maxItems: 2 + + dma-names: + items: + - const: rx + - const: tx + +patternProperties: + "@[0-9a-f]+": + type: object + + properties: + spi-rx-bus-width: + enum: [1, 2, 4] + + spi-tx-bus-width: + enum: [1, 2, 4] + + nvidia,tx-clk-tap-delay: + description: + Delays the clock going out to device with this tap value. + Tap value varies based on platform design trace lengths from Tegra + QSPI to corresponding slave device. + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 31 + + nvidia,rx-clk-tap-delay: + description: + Delays the clock coming in from the device with this tap value. + Tap value varies based on platform design trace lengths from Tegra + QSPI to corresponding slave device. + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 255 + + required: + - reg + +required: + - compatible + - reg + - interrupts + - clock-names + - clocks + - resets + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + spi@70410000 { + compatible = "nvidia,tegra210-qspi"; + reg = <0x70410000 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&tegra_car TEGRA210_CLK_QSPI>, + <&tegra_car TEGRA210_CLK_QSPI_PM>; + clock-names = "qspi", "qspi_out"; + resets = <&tegra_car 211>; + dmas = <&apbdma 5>, <&apbdma 5>; + dma-names = "rx", "tx"; + + flash@0 { + compatible = "spi-nor"; + reg = <0>; + spi-max-frequency = <104000000>; + spi-tx-bus-width = <2>; + spi-rx-bus-width = <2>; + nvidia,tx-clk-tap-delay = <0>; + nvidia,rx-clk-tap-delay = <0>; + }; + }; diff --git a/Documentation/devicetree/bindings/spi/realtek,rtl-spi.yaml b/Documentation/devicetree/bindings/spi/realtek,rtl-spi.yaml new file mode 100644 index 000000000000..30a62a211984 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/realtek,rtl-spi.yaml @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spi/realtek,rtl-spi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Realtek RTL838x/RTL839x SPI controller + +maintainers: + - Bert Vermeulen + - Birger Koblitz + +allOf: + - $ref: "spi-controller.yaml#" + +properties: + compatible: + oneOf: + - const: realtek,rtl8380-spi + - const: realtek,rtl8382-spi + - const: realtek,rtl8391-spi + - const: realtek,rtl8392-spi + - const: realtek,rtl8393-spi + + reg: + maxItems: 1 + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + spi: spi@1200 { + compatible = "realtek,rtl8382-spi"; + reg = <0x1200 0x100>; + #address-cells = <1>; + #size-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/spi/renesas,rspi.yaml b/Documentation/devicetree/bindings/spi/renesas,rspi.yaml index 10e83cb17e8d..8397f60d80a2 100644 --- a/Documentation/devicetree/bindings/spi/renesas,rspi.yaml +++ b/Documentation/devicetree/bindings/spi/renesas,rspi.yaml @@ -68,6 +68,8 @@ properties: maxItems: 1 dmas: + minItems: 2 + maxItems: 4 description: Must contain a list of pairs of references to DMA specifiers, one for transmission, and one for reception. diff --git a/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml b/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml index 44c7ddb4b109..b104899205f6 100644 --- a/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml +++ b/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml @@ -47,6 +47,7 @@ properties: - renesas,msiof-r8a77980 # R-Car V3H - renesas,msiof-r8a77990 # R-Car E3 - renesas,msiof-r8a77995 # R-Car D3 + - renesas,msiof-r8a779a0 # R-Car V3U - const: renesas,rcar-gen3-msiof # generic R-Car Gen3 and RZ/G2 # compatible device - items: diff --git a/Documentation/devicetree/bindings/spi/spi-controller.yaml b/Documentation/devicetree/bindings/spi/spi-controller.yaml index 5f505810104d..06786f1b43d2 100644 --- a/Documentation/devicetree/bindings/spi/spi-controller.yaml +++ b/Documentation/devicetree/bindings/spi/spi-controller.yaml @@ -152,8 +152,9 @@ patternProperties: spi-rx-bus-width: description: Bus width to the SPI bus used for read transfers. + If 0 is provided, then no RX will be possible on this device. $ref: /schemas/types.yaml#/definitions/uint32 - enum: [1, 2, 4, 8] + enum: [0, 1, 2, 4, 8] default: 1 spi-rx-delay-us: @@ -163,8 +164,9 @@ patternProperties: spi-tx-bus-width: description: Bus width to the SPI bus used for write transfers. + If 0 is provided, then no TX will be possible on this device. $ref: /schemas/types.yaml#/definitions/uint32 - enum: [1, 2, 4, 8] + enum: [0, 1, 2, 4, 8] default: 1 spi-tx-delay-us: diff --git a/Documentation/devicetree/bindings/spi/spi-sirf.txt b/Documentation/devicetree/bindings/spi/spi-sirf.txt deleted file mode 100644 index ddd78ff68fae..000000000000 --- a/Documentation/devicetree/bindings/spi/spi-sirf.txt +++ /dev/null @@ -1,42 +0,0 @@ -* CSR SiRFprimaII Serial Peripheral Interface - -Required properties: -- compatible : Should be "sirf,prima2-spi", "sirf,prima2-usp" - or "sirf,atlas7-usp" -- reg : Offset and length of the register set for the device -- interrupts : Should contain SPI interrupt -- resets: phandle to the reset controller asserting this device in - reset - See ../reset/reset.txt for details. -- dmas : Must contain an entry for each entry in clock-names. - See ../dma/dma.txt for details. -- dma-names : Must include the following entries: - - rx - - tx -- clocks : Must contain an entry for each entry in clock-names. - See ../clocks/clock-bindings.txt for details. - -- #address-cells: Number of cells required to define a chip select - address on the SPI bus. Should be set to 1. -- #size-cells: Should be zero. - -Optional properties: -- spi-max-frequency: Specifies maximum SPI clock frequency, - Units - Hz. Definition as per - Documentation/devicetree/bindings/spi/spi-bus.txt -- cs-gpios: should specify GPIOs used for chipselects. - -Example: - -spi0: spi@b00d0000 { - compatible = "sirf,prima2-spi"; - reg = <0xb00d0000 0x10000>; - interrupts = <15>; - dmas = <&dmac1 9>, - <&dmac1 4>; - dma-names = "rx", "tx"; - #address-cells = <1>; - #size-cells = <0>; - clocks = <&clks 19>; - resets = <&rstc 26>; -}; diff --git a/Documentation/devicetree/bindings/sram/sram.yaml b/Documentation/devicetree/bindings/sram/sram.yaml index 6d65771e979c..c1a5afa73cfe 100644 --- a/Documentation/devicetree/bindings/sram/sram.yaml +++ b/Documentation/devicetree/bindings/sram/sram.yaml @@ -35,6 +35,7 @@ properties: maxItems: 1 clocks: + maxItems: 1 description: A list of phandle and clock specifier pair that controls the single SRAM clock. @@ -46,6 +47,7 @@ properties: const: 1 ranges: + maxItems: 1 description: Should translate from local addresses within the sram to bus addresses. diff --git a/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml b/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml index 31edd051295a..bf97d1fb33e7 100644 --- a/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml +++ b/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml @@ -103,12 +103,12 @@ allOf: compatible: contains: enum: - - const: allwinner,sun8i-h3-ths - - const: allwinner,sun8i-r40-ths - - const: allwinner,sun50i-a64-ths - - const: allwinner,sun50i-a100-ths - - const: allwinner,sun50i-h5-ths - - const: allwinner,sun50i-h6-ths + - allwinner,sun8i-h3-ths + - allwinner,sun8i-r40-ths + - allwinner,sun50i-a64-ths + - allwinner,sun50i-a100-ths + - allwinner,sun50i-h5-ths + - allwinner,sun50i-h6-ths then: required: diff --git a/Documentation/devicetree/bindings/thermal/qcom-spmi-adc-tm5.yaml b/Documentation/devicetree/bindings/thermal/qcom-spmi-adc-tm5.yaml new file mode 100644 index 000000000000..7cd364430573 --- /dev/null +++ b/Documentation/devicetree/bindings/thermal/qcom-spmi-adc-tm5.yaml @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thermal/qcom-spmi-adc-tm5.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm's SPMI PMIC ADC Thermal Monitoring +maintainers: + - Dmitry Baryshkov + +properties: + compatible: + const: qcom,spmi-adc-tm5 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + "#thermal-sensor-cells": + const: 1 + description: + Number of cells required to uniquely identify the thermal sensors. Since + we have multiple sensors this is set to 1 + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + + qcom,avg-samples: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Number of samples to be used for measurement. + enum: + - 1 + - 2 + - 4 + - 8 + - 16 + default: 1 + + qcom,decimation: + $ref: /schemas/types.yaml#/definitions/uint32 + description: This parameter is used to decrease ADC sampling rate. + Quicker measurements can be made by reducing decimation ratio. + enum: + - 250 + - 420 + - 840 + default: 840 + +patternProperties: + "^([-a-z0-9]*)@[0-7]$": + type: object + description: + Represent one thermal sensor. + + properties: + reg: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Specify the sensor channel. There are 8 channels in PMIC5's ADC TM + minimum: 0 + maximum: 7 + + io-channels: + description: + From common IIO binding. Used to pipe PMIC ADC channel to thermal monitor + + qcom,ratiometric: + $ref: /schemas/types.yaml#/definitions/flag + description: + Channel calibration type. + If this property is specified VADC will use the VDD reference + (1.875V) and GND for channel calibration. If property is not found, + channel will be calibrated with 0V and 1.25V reference channels, + also known as absolute calibration. + + qcom,hw-settle-time-us: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Time between AMUX getting configured and the ADC starting conversion. + enum: [15, 100, 200, 300, 400, 500, 600, 700, 1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000] + + qcom,pre-scaling: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Used for scaling the channel input signal before the + signal is fed to VADC. The configuration for this node is to know the + pre-determined ratio and use it for post scaling. It is a pair of + integers, denoting the numerator and denominator of the fraction by + which input signal is multiplied. For example, <1 3> indicates the + signal is scaled down to 1/3 of its value before ADC measurement. If + property is not found default value depending on chip will be used. + items: + - const: 1 + - enum: [ 1, 3, 4, 6, 20, 8, 10 ] + + required: + - reg + - io-channels + + additionalProperties: + false + +required: + - compatible + - reg + - interrupts + - "#address-cells" + - "#size-cells" + - "#thermal-sensor-cells" + +additionalProperties: false + +examples: + - | + #include + #include + spmi_bus { + #address-cells = <1>; + #size-cells = <0>; + pm8150b_adc: adc@3100 { + reg = <0x3100>; + compatible = "qcom,spmi-adc5"; + #address-cells = <1>; + #size-cells = <0>; + #io-channel-cells = <1>; + + /* Other propreties are omitted */ + conn-therm@4f { + reg = ; + qcom,ratiometric; + qcom,hw-settle-time = <200>; + }; + }; + + pm8150b_adc_tm: adc-tm@3500 { + compatible = "qcom,spmi-adc-tm5"; + reg = <0x3500>; + interrupts = <0x2 0x35 0x0 IRQ_TYPE_EDGE_RISING>; + #thermal-sensor-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + + conn-therm@0 { + reg = <0>; + io-channels = <&pm8150b_adc ADC5_AMUX_THM3_100K_PU>; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/thermal/tango-thermal.txt b/Documentation/devicetree/bindings/thermal/tango-thermal.txt deleted file mode 100644 index 2c918d742867..000000000000 --- a/Documentation/devicetree/bindings/thermal/tango-thermal.txt +++ /dev/null @@ -1,17 +0,0 @@ -* Tango Thermal - -The SMP8758 SoC includes 3 instances of this temperature sensor -(in the CPU, video decoder, and PCIe controller). - -Required properties: -- #thermal-sensor-cells: Should be 0 (see Documentation/devicetree/bindings/thermal/thermal-sensor.yaml) -- compatible: "sigma,smp8758-thermal" -- reg: Address range of the thermal registers - -Example: - - cpu_temp: thermal@920100 { - #thermal-sensor-cells = <0>; - compatible = "sigma,smp8758-thermal"; - reg = <0x920100 12>; - }; diff --git a/Documentation/devicetree/bindings/thermal/zx2967-thermal.txt b/Documentation/devicetree/bindings/thermal/zx2967-thermal.txt deleted file mode 100644 index 3dc1c6bf0478..000000000000 --- a/Documentation/devicetree/bindings/thermal/zx2967-thermal.txt +++ /dev/null @@ -1,116 +0,0 @@ -* ZTE zx2967 family Thermal - -Required Properties: -- compatible: should be one of the following. - * zte,zx296718-thermal -- reg: physical base address of the controller and length of memory mapped - region. -- clocks : Pairs of phandle and specifier referencing the controller's clocks. -- clock-names: "topcrm" for the topcrm clock. - "apb" for the apb clock. -- #thermal-sensor-cells: must be 0. - -Please note: slope coefficient defined in thermal-zones section need to be -multiplied by 1000. - -Example for tempsensor: - - tempsensor: tempsensor@148a000 { - compatible = "zte,zx296718-thermal"; - reg = <0x0148a000 0x20>; - clocks = <&topcrm TEMPSENSOR_GATE>, <&audiocrm AUDIO_TS_PCLK>; - clock-names = "topcrm", "apb"; - #thermal-sensor-cells = <0>; - }; - -Example for cooling device: - - cooling_dev: cooling_dev { - cluster0_cooling_dev: cluster0-cooling-dev { - #cooling-cells = <2>; - cpumask = <0xf>; - capacitance = <1500>; - }; - - cluster1_cooling_dev: cluster1-cooling-dev { - #cooling-cells = <2>; - cpumask = <0x30>; - capacitance = <2000>; - }; - }; - -Example for thermal zones: - - thermal-zones { - zx296718_thermal: zx296718_thermal { - polling-delay-passive = <500>; - polling-delay = <1000>; - sustainable-power = <6500>; - - thermal-sensors = <&tempsensor 0>; - /* - * slope need to be multiplied by 1000. - */ - coefficients = <1951 (-922)>; - - trips { - trip0: switch_on_temperature { - temperature = <90000>; - hysteresis = <2000>; - type = "passive"; - }; - - trip1: desired_temperature { - temperature = <100000>; - hysteresis = <2000>; - type = "passive"; - }; - - crit: critical_temperature { - temperature = <110000>; - hysteresis = <2000>; - type = "critical"; - }; - }; - - cooling-maps { - map0 { - trip = <&trip0>; - cooling-device = <&gpu 2 5>; - }; - - map1 { - trip = <&trip0>; - cooling-device = <&cluster0_cooling_dev 1 2>; - }; - - map2 { - trip = <&trip1>; - cooling-device = <&cluster0_cooling_dev 1 2>; - }; - - map3 { - trip = <&crit>; - cooling-device = <&cluster0_cooling_dev 1 2>; - }; - - map4 { - trip = <&trip0>; - cooling-device = <&cluster1_cooling_dev 1 2>; - contribution = <9000>; - }; - - map5 { - trip = <&trip1>; - cooling-device = <&cluster1_cooling_dev 1 2>; - contribution = <4096>; - }; - - map6 { - trip = <&crit>; - cooling-device = <&cluster1_cooling_dev 1 2>; - contribution = <4096>; - }; - }; - }; - }; diff --git a/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml b/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml index d918cee100ac..1c7cf32e7ac2 100644 --- a/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml +++ b/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml @@ -22,6 +22,8 @@ properties: maxItems: 1 interrupts: + minItems: 2 + maxItems: 6 description: List of timers interrupts diff --git a/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml b/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml index 40fc4bcb3145..b6a6d03a08b2 100644 --- a/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml +++ b/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml @@ -46,8 +46,7 @@ required: if: properties: compatible: - items: - const: allwinner,sun5i-a13-hstimer + const: allwinner,sun5i-a13-hstimer then: properties: diff --git a/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml b/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml index 1a721d8af67a..a8de99b0c0f9 100644 --- a/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml +++ b/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml @@ -18,7 +18,7 @@ properties: - const: intel,ixp4xx-timer reg: - description: Should contain registers location and length + maxItems: 1 interrupts: minItems: 1 diff --git a/Documentation/devicetree/bindings/timer/sifive,clint.yaml b/Documentation/devicetree/bindings/timer/sifive,clint.yaml index 2a0e9cd9fbcf..a35952f48742 100644 --- a/Documentation/devicetree/bindings/timer/sifive,clint.yaml +++ b/Documentation/devicetree/bindings/timer/sifive,clint.yaml @@ -23,15 +23,19 @@ description: properties: compatible: items: - - const: sifive,fu540-c000-clint + - enum: + - sifive,fu540-c000-clint + - canaan,k210-clint - const: sifive,clint0 description: - Should be "sifive,-clint" and "sifive,clint". + Should be ",-clint" and "sifive,clint". Supported compatible strings are - "sifive,fu540-c000-clint" for the SiFive CLINT v0 as integrated - onto the SiFive FU540 chip, and "sifive,clint0" for the SiFive - CLINT v0 IP block with no chip integration tweaks. + onto the SiFive FU540 chip, "canaan,k210-clint" for the SiFive + CLINT v0 as integrated onto the Canaan Kendryte K210 chip, and + "sifive,clint0" for the SiFive CLINT v0 IP block with no chip + integration tweaks. Please refer to sifive-blocks-ip-versioning.txt for details reg: diff --git a/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml b/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml index d65faf289a83..d33c9205a909 100644 --- a/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml +++ b/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml @@ -24,6 +24,9 @@ properties: interrupts: maxItems: 1 + resets: + maxItems: 1 + clocks: minItems: 1 items: diff --git a/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml b/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml index d9207bf9d894..0f520f17735e 100644 --- a/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml +++ b/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml @@ -39,7 +39,7 @@ properties: maxItems: 1 phys: - description: PHY specifier for the OTG PHY + maxItems: 1 phy-names: const: usb diff --git a/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml b/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml index ffa148b9eaa8..d4618d15ecc1 100644 --- a/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml +++ b/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml @@ -22,6 +22,8 @@ properties: description: Interrupt for signals mirrored to out-gpios. in-gpios: + minItems: 1 + maxItems: 2 description: Array of one or two GPIO pins used for input signals. brcm,in-functions: @@ -33,6 +35,7 @@ properties: description: Array of enable and mask pairs, one per gpio in-gpios. out-gpios: + maxItems: 1 description: Array of one GPIO pin used for output signals. brcm,out-functions: diff --git a/Documentation/devicetree/bindings/usb/generic-ehci.yaml b/Documentation/devicetree/bindings/usb/generic-ehci.yaml index 6816de7dfc00..cf83f2d9afac 100644 --- a/Documentation/devicetree/bindings/usb/generic-ehci.yaml +++ b/Documentation/devicetree/bindings/usb/generic-ehci.yaml @@ -128,7 +128,7 @@ properties: Phandle of a companion. phys: - description: PHY specifier for the USB PHY + maxItems: 1 phy-names: const: usb diff --git a/Documentation/devicetree/bindings/usb/generic-ohci.yaml b/Documentation/devicetree/bindings/usb/generic-ohci.yaml index 53df281f618c..0f5f6ea702d0 100644 --- a/Documentation/devicetree/bindings/usb/generic-ohci.yaml +++ b/Documentation/devicetree/bindings/usb/generic-ohci.yaml @@ -101,7 +101,7 @@ properties: Overrides the detected port count phys: - description: PHY specifier for the USB PHY + maxItems: 1 phy-names: const: usb diff --git a/Documentation/devicetree/bindings/usb/ingenic,musb.yaml b/Documentation/devicetree/bindings/usb/ingenic,musb.yaml index 678396eeeb78..f506225a4d57 100644 --- a/Documentation/devicetree/bindings/usb/ingenic,musb.yaml +++ b/Documentation/devicetree/bindings/usb/ingenic,musb.yaml @@ -40,7 +40,7 @@ properties: - const: mc phys: - description: PHY specifier for the USB PHY + maxItems: 1 usb-role-switch: type: boolean diff --git a/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml b/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml index 929a3f413b44..9fcf54b10b07 100644 --- a/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml +++ b/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml @@ -54,18 +54,19 @@ properties: description: phandle of a companion. ports: + $ref: /schemas/graph.yaml#/properties/ports description: | any connector to the data bus of this controller should be modelled using the OF graph bindings specified, if the "usb-role-switch" property is used. - type: object + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: High Speed (HS) data bus. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Super Speed (SS) data bus. required: diff --git a/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml b/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml index 54c361d4a7af..e67223d90bb7 100644 --- a/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml +++ b/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml @@ -68,6 +68,7 @@ properties: Integer to use BUSWAIT register. renesas,enable-gpio: + maxItems: 1 description: | gpio specifier to check GPIO determining if USB function should be enabled. diff --git a/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml b/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml index 52ceb07294a3..b86bf6bc9cd6 100644 --- a/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml +++ b/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml @@ -26,17 +26,17 @@ properties: maxItems: 1 ports: + $ref: /schemas/graph.yaml#/properties/ports description: OF graph bindings (specified in bindings/graph.txt) that model SS data bus to the SS capable connector. - type: object + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Super Speed (SS) MUX inputs connected to SS capable connector. - $ref: /connector/usb-connector.yaml#/properties/ports/properties/port@1 port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Output of 2:1 MUX connected to Super Speed (SS) data bus. required: diff --git a/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml b/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml index c80a83571919..7ec87a783c5c 100644 --- a/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml +++ b/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml @@ -19,7 +19,7 @@ properties: - const: ti,am64-usb reg: - description: module registers + maxItems: 1 ranges: true @@ -28,6 +28,7 @@ properties: PM domain provider node and an args specifier containing the USB device id value. See, Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt + maxItems: 1 clocks: description: Clock phandles to usb2_refclk and lpm_clk diff --git a/Documentation/devicetree/bindings/usb/ti,keystone-dwc3.yaml b/Documentation/devicetree/bindings/usb/ti,keystone-dwc3.yaml index ca7fbe3ed22e..9a068d3bc73b 100644 --- a/Documentation/devicetree/bindings/usb/ti,keystone-dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/ti,keystone-dwc3.yaml @@ -43,12 +43,14 @@ properties: maxItems: 2 power-domains: + maxItems: 1 description: Should contain a phandle to a PM domain provider node and an args specifier containing the USB device id value. This property is as per the binding, Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt phys: + maxItems: 1 description: PHY specifier for the USB3.0 PHY. Some SoCs need the USB3.0 PHY to be turned on before the controller. diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index b4e026f2d6c5..f6064d84a424 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -1085,6 +1085,8 @@ patternProperties: description: Shenzhen Sunchip Technology Co., Ltd "^SUNW,.*": description: Sun Microsystems, Inc + "^silvaco,.*": + description: Silvaco, Inc. "^swir,.*": description: Sierra Wireless "^syna,.*": @@ -1272,6 +1274,8 @@ patternProperties: description: YSH & ATIL "^yones-toptech,.*": description: Yones Toptech Co., Ltd. + "^ys,.*": + description: Shenzhen Yashi Changhua Intelligent Technology Co., Ltd. "^ysoft,.*": description: Y Soft Corporation a.s. "^zealz,.*": diff --git a/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml b/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml index 5ac607de8be4..9aa3c313c49f 100644 --- a/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml @@ -19,13 +19,11 @@ properties: - const: allwinner,sun4i-a10-wdt - const: allwinner,sun6i-a31-wdt - items: - - const: allwinner,sun50i-a64-wdt - - const: allwinner,sun6i-a31-wdt - - items: - - const: allwinner,sun50i-a100-wdt - - const: allwinner,sun6i-a31-wdt - - items: - - const: allwinner,sun50i-h6-wdt + - enum: + - allwinner,sun50i-a64-wdt + - allwinner,sun50i-a100-wdt + - allwinner,sun50i-h6-wdt + - allwinner,sun50i-h616-wdt - const: allwinner,sun6i-a31-wdt - items: - const: allwinner,suniv-f1c100s-wdt diff --git a/Documentation/devicetree/bindings/watchdog/intel,keembay-wdt.yaml b/Documentation/devicetree/bindings/watchdog/intel,keembay-wdt.yaml new file mode 100644 index 000000000000..1437ff8a122f --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/intel,keembay-wdt.yaml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/intel,keembay-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel Keem Bay SoC non-secure Watchdog Timer + +maintainers: + - Wan Ahmad Zainie + +properties: + compatible: + enum: + - intel,keembay-wdt + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interrupts: + items: + - description: interrupt specifier for threshold interrupt line + - description: interrupt specifier for timeout interrupt line + + interrupt-names: + items: + - const: threshold + - const: timeout + +required: + - compatible + - reg + - interrupts + - interrupt-names + - clocks + +additionalProperties: false + +examples: + - | + #include + #include + #define KEEM_BAY_A53_TIM + + watchdog: watchdog@2033009c { + compatible = "intel,keembay-wdt"; + reg = <0x2033009c 0x10>; + interrupts = , + ; + interrupt-names = "threshold", "timeout"; + clocks = <&scmi_clk KEEM_BAY_A53_TIM>; + }; + +... diff --git a/Documentation/devicetree/bindings/watchdog/mtk-wdt.txt b/Documentation/devicetree/bindings/watchdog/mtk-wdt.txt index 4dd36bd3f1ad..e36ba60de829 100644 --- a/Documentation/devicetree/bindings/watchdog/mtk-wdt.txt +++ b/Documentation/devicetree/bindings/watchdog/mtk-wdt.txt @@ -4,14 +4,15 @@ Required properties: - compatible should contain: "mediatek,mt2701-wdt", "mediatek,mt6589-wdt": for MT2701 - "mediatek,mt2712-wdt", "mediatek,mt6589-wdt": for MT2712 + "mediatek,mt2712-wdt": for MT2712 "mediatek,mt6589-wdt": for MT6589 "mediatek,mt6797-wdt", "mediatek,mt6589-wdt": for MT6797 "mediatek,mt7622-wdt", "mediatek,mt6589-wdt": for MT7622 "mediatek,mt7623-wdt", "mediatek,mt6589-wdt": for MT7623 "mediatek,mt7629-wdt", "mediatek,mt6589-wdt": for MT7629 - "mediatek,mt8183-wdt", "mediatek,mt6589-wdt": for MT8183 + "mediatek,mt8183-wdt": for MT8183 "mediatek,mt8516-wdt", "mediatek,mt6589-wdt": for MT8516 + "mediatek,mt8192-wdt": for MT8192 - reg : Specifies base physical address and size of the registers. diff --git a/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml index 6933005b52bd..ab66d3f0c476 100644 --- a/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml @@ -50,6 +50,7 @@ properties: - renesas,r8a77980-wdt # R-Car V3H - renesas,r8a77990-wdt # R-Car E3 - renesas,r8a77995-wdt # R-Car D3 + - renesas,r8a779a0-wdt # R-Car V3U - const: renesas,rcar-gen3-wdt # R-Car Gen3 and RZ/G2 reg: diff --git a/Documentation/devicetree/bindings/watchdog/sigma,smp8642-wdt.txt b/Documentation/devicetree/bindings/watchdog/sigma,smp8642-wdt.txt deleted file mode 100644 index 5b7ec2c707d8..000000000000 --- a/Documentation/devicetree/bindings/watchdog/sigma,smp8642-wdt.txt +++ /dev/null @@ -1,18 +0,0 @@ -Sigma Designs SMP86xx/SMP87xx watchdog - -Required properties: -- compatible: Should be "sigma,smp8642-wdt" -- reg: Specifies the physical address region -- clocks: Should be a phandle to the clock - -Optional properties: -- timeout-sec: watchdog timeout in seconds - -Example: - -watchdog@1fd00 { - compatible = "sigma,smp8642-wdt"; - reg = <0x1fd00 8>; - clocks = <&xtal_in_clk>; - timeout-sec = <30>; -}; diff --git a/Documentation/devicetree/bindings/watchdog/sirfsoc_wdt.txt b/Documentation/devicetree/bindings/watchdog/sirfsoc_wdt.txt deleted file mode 100644 index 0dce5e3100b4..000000000000 --- a/Documentation/devicetree/bindings/watchdog/sirfsoc_wdt.txt +++ /dev/null @@ -1,18 +0,0 @@ -SiRFSoC Timer and Watchdog Timer(WDT) Controller - -Required properties: -- compatible: "sirf,prima2-tick" -- reg: Address range of tick timer/WDT register set -- interrupts: interrupt number to the cpu - -Optional properties: -- timeout-sec : Contains the watchdog timeout in seconds - -Example: - -timer@b0020000 { - compatible = "sirf,prima2-tick"; - reg = <0xb0020000 0x1000>; - interrupts = <0>; - timeout-sec = <30>; -}; diff --git a/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml b/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml index f7ee9229c29f..b58596b1831d 100644 --- a/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml @@ -18,10 +18,16 @@ properties: - const: snps,dw-wdt - items: - enum: + - rockchip,px30-wdt - rockchip,rk3066-wdt - rockchip,rk3188-wdt + - rockchip,rk3228-wdt - rockchip,rk3288-wdt + - rockchip,rk3308-wdt + - rockchip,rk3328-wdt - rockchip,rk3368-wdt + - rockchip,rk3399-wdt + - rockchip,rv1108-wdt - const: snps,dw-wdt reg: diff --git a/Documentation/devicetree/bindings/watchdog/stericsson-coh901327.txt b/Documentation/devicetree/bindings/watchdog/stericsson-coh901327.txt deleted file mode 100644 index 8ffb88e39e76..000000000000 --- a/Documentation/devicetree/bindings/watchdog/stericsson-coh901327.txt +++ /dev/null @@ -1,19 +0,0 @@ -ST-Ericsson COH 901 327 Watchdog timer - -Required properties: -- compatible: must be "stericsson,coh901327". -- reg: physical base address of the controller and length of memory mapped - region. -- interrupts: the interrupt used for the watchdog timeout warning. - -Optional properties: -- timeout-sec: contains the watchdog timeout in seconds. - -Example: - -watchdog: watchdog@c0012000 { - compatible = "stericsson,coh901327"; - reg = <0xc0012000 0x1000>; - interrupts = <3>; - timeout-sec = <60>; -}; diff --git a/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml b/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml index c1348db59374..054584d7543a 100644 --- a/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml @@ -57,8 +57,8 @@ examples: */ #include - watchdog0: rti@2200000 { - compatible = "ti,rti-wdt"; + watchdog@2200000 { + compatible = "ti,j7-rti-wdt"; reg = <0x2200000 0x100>; clocks = <&k3_clks 252 1>; power-domains = <&k3_pds 252 TI_SCI_PD_EXCLUSIVE>; diff --git a/Documentation/devicetree/bindings/watchdog/zte,zx2967-wdt.txt b/Documentation/devicetree/bindings/watchdog/zte,zx2967-wdt.txt deleted file mode 100644 index 06ce67766756..000000000000 --- a/Documentation/devicetree/bindings/watchdog/zte,zx2967-wdt.txt +++ /dev/null @@ -1,32 +0,0 @@ -ZTE zx2967 Watchdog timer - -Required properties: - -- compatible : should be one of the following. - * zte,zx296718-wdt -- reg : Specifies base physical address and size of the registers. -- clocks : Pairs of phandle and specifier referencing the controller's clocks. -- resets : Reference to the reset controller controlling the watchdog - controller. - -Optional properties: - -- timeout-sec : Contains the watchdog timeout in seconds. -- zte,wdt-reset-sysctrl : Directs how to reset system by the watchdog. - if we don't want to restart system when watchdog been triggered, - it's not required, vice versa. - It should include following fields. - * phandle of aon-sysctrl. - * offset of register that be written, should be 0xb0. - * configure value that be written to aon-sysctrl. - * bit mask, corresponding bits will be affected. - -Example: - -wdt: watchdog@1465000 { - compatible = "zte,zx296718-wdt"; - reg = <0x1465000 0x1000>; - clocks = <&topcrm WDT_WCLK>; - resets = <&toprst 35>; - zte,wdt-reset-sysctrl = <&aon_sysctrl 0xb0 1 0x115>; -}; diff --git a/Documentation/devicetree/usage-model.rst b/Documentation/devicetree/usage-model.rst index e1b42dc63f01..1eb83496ca1e 100644 --- a/Documentation/devicetree/usage-model.rst +++ b/Documentation/devicetree/usage-model.rst @@ -12,7 +12,7 @@ This article describes how Linux uses the device tree. An overview of the device tree data format can be found on the device tree usage page at devicetree.org\ [1]_. -.. [1] https://elinux.org/Device_Tree_Usage +.. [1] https://www.devicetree.org/specifications/ The "Open Firmware Device Tree", or simply Device Tree (DT), is a data structure and language for describing hardware. More specifically, it diff --git a/Documentation/doc-guide/sphinx.rst b/Documentation/doc-guide/sphinx.rst index 36ac2166ad67..ec3e71f56009 100644 --- a/Documentation/doc-guide/sphinx.rst +++ b/Documentation/doc-guide/sphinx.rst @@ -340,16 +340,26 @@ Rendered as: Cross-referencing ----------------- -Cross-referencing from one documentation page to another can be done by passing -the path to the file starting from the Documentation folder. -For example, to cross-reference to this page (the .rst extension is optional):: +Cross-referencing from one documentation page to another can be done simply by +writing the path to the document file, no special syntax required. The path can +be either absolute or relative. For absolute paths, start it with +"Documentation/". For example, to cross-reference to this page, all the +following are valid options, depending on the current document's directory (note +that the ``.rst`` extension is required):: - See Documentation/doc-guide/sphinx.rst. + See Documentation/doc-guide/sphinx.rst. This always works. + Take a look at sphinx.rst, which is at this same directory. + Read ../sphinx.rst, which is one directory above. -If you want to use a relative path, you need to use Sphinx's ``doc`` directive. -For example, referencing this page from the same directory would be done as:: +If you want the link to have a different rendered text other than the document's +title, you need to use Sphinx's ``doc`` role. For example:: - See :doc:`sphinx`. + See :doc:`my custom link text for document sphinx `. + +For most use cases, the former is preferred, as it is cleaner and more suited +for people reading the source files. If you come across a ``:doc:`` usage that +isn't adding any value, please feel free to convert it to just the document +path. For information on cross-referencing to kernel-doc functions or types, see Documentation/doc-guide/kernel-doc.rst. diff --git a/Documentation/driver-api/cxl/index.rst b/Documentation/driver-api/cxl/index.rst new file mode 100644 index 000000000000..036e49553542 --- /dev/null +++ b/Documentation/driver-api/cxl/index.rst @@ -0,0 +1,12 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==================== +Compute Express Link +==================== + +.. toctree:: + :maxdepth: 1 + + memory-devices + +.. only:: subproject and html diff --git a/Documentation/driver-api/cxl/memory-devices.rst b/Documentation/driver-api/cxl/memory-devices.rst new file mode 100644 index 000000000000..1bad466f9167 --- /dev/null +++ b/Documentation/driver-api/cxl/memory-devices.rst @@ -0,0 +1,46 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: + +=================================== +Compute Express Link Memory Devices +=================================== + +A Compute Express Link Memory Device is a CXL component that implements the +CXL.mem protocol. It contains some amount of volatile memory, persistent memory, +or both. It is enumerated as a PCI device for configuration and passing +messages over an MMIO mailbox. Its contribution to the System Physical +Address space is handled via HDM (Host Managed Device Memory) decoders +that optionally define a device's contribution to an interleaved address +range across multiple devices underneath a host-bridge or interleaved +across host-bridges. + +Driver Infrastructure +===================== + +This section covers the driver infrastructure for a CXL memory device. + +CXL Memory Device +----------------- + +.. kernel-doc:: drivers/cxl/mem.c + :doc: cxl mem + +.. kernel-doc:: drivers/cxl/mem.c + :internal: + +CXL Bus +------- +.. kernel-doc:: drivers/cxl/bus.c + :doc: cxl bus + +External Interfaces +=================== + +CXL IOCTL Interface +------------------- + +.. kernel-doc:: include/uapi/linux/cxl_mem.h + :doc: UAPI + +.. kernel-doc:: include/uapi/linux/cxl_mem.h + :internal: diff --git a/Documentation/driver-api/gpio/consumer.rst b/Documentation/driver-api/gpio/consumer.rst index 173e4c7b037d..22271c342d92 100644 --- a/Documentation/driver-api/gpio/consumer.rst +++ b/Documentation/driver-api/gpio/consumer.rst @@ -361,12 +361,13 @@ corresponding chip driver. In that case a significantly improved performance can be expected. If simultaneous access is not possible the GPIOs will be accessed sequentially. -The functions take three arguments: +The functions take four arguments: + * array_size - the number of array elements * desc_array - an array of GPIO descriptors * array_info - optional information obtained from gpiod_get_array() * value_bitmap - a bitmap to store the GPIOs' values (get) or - a bitmap of values to assign to the GPIOs (set) + a bitmap of values to assign to the GPIOs (set) The descriptor array can be obtained using the gpiod_get_array() function or one of its variants. If the group of descriptors returned by that function diff --git a/Documentation/driver-api/gpio/driver.rst b/Documentation/driver-api/gpio/driver.rst index 0fb57e298b41..d6b0d779859b 100644 --- a/Documentation/driver-api/gpio/driver.rst +++ b/Documentation/driver-api/gpio/driver.rst @@ -640,8 +640,8 @@ compliance: level and edge IRQs * [1] http://www.spinics.net/lists/linux-omap/msg120425.html -* [2] https://lkml.org/lkml/2015/9/25/494 -* [3] https://lkml.org/lkml/2015/9/25/495 +* [2] https://lore.kernel.org/r/1443209283-20781-2-git-send-email-grygorii.strashko@ti.com +* [3] https://lore.kernel.org/r/1443209283-20781-3-git-send-email-grygorii.strashko@ti.com Requesting self-owned GPIO pins diff --git a/Documentation/driver-api/gpio/intro.rst b/Documentation/driver-api/gpio/intro.rst index 74591489d0b5..94dd7185e76e 100644 --- a/Documentation/driver-api/gpio/intro.rst +++ b/Documentation/driver-api/gpio/intro.rst @@ -106,11 +106,11 @@ don't. When you need open drain signaling but your hardware doesn't directly support it, there's a common idiom you can use to emulate it with any GPIO pin that can be used as either an input or an output: - LOW: gpiod_direction_output(gpio, 0) ... this drives the signal and overrides - the pullup. + **LOW**: ``gpiod_direction_output(gpio, 0)`` ... this drives the signal and + overrides the pullup. - HIGH: gpiod_direction_input(gpio) ... this turns off the output, so the pullup - (or some other device) controls the signal. + **HIGH**: ``gpiod_direction_input(gpio)`` ... this turns off the output, so + the pullup (or some other device) controls the signal. The same logic can be applied to emulate open source signaling, by driving the high signal and configuring the GPIO as input for low. This open drain/open diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index 2456d0a97ed8..b0ab367896ab 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -35,6 +35,7 @@ available subsections can be seen below. usb/index firewire pci/index + cxl/index spi i2c ipmb @@ -93,12 +94,12 @@ available subsections can be seen below. pps ptp phy/index - pti_intel_mid pwm pldmfw/index rfkill serial/index sm501 + surface_aggregator/index switchtec sync_file vfio-mediated-device diff --git a/Documentation/driver-api/men-chameleon-bus.rst b/Documentation/driver-api/men-chameleon-bus.rst index 1b1f048aa748..6f0b9ee47595 100644 --- a/Documentation/driver-api/men-chameleon-bus.rst +++ b/Documentation/driver-api/men-chameleon-bus.rst @@ -18,6 +18,7 @@ MEN Chameleon Bus 4.1 The driver structure 4.2 Probing and attaching 4.3 Initializing the driver + 4.4 Using DMA Introduction @@ -173,3 +174,14 @@ module at the MCB core:: The module_mcb_driver() macro can be used to reduce the above code:: module_mcb_driver(foo_driver); + +Using DMA +--------- + +To make use of the kernel's DMA-API's function, you will need to use the +carrier device's 'struct device'. Fortunately 'struct mcb_device' embeds a +pointer (->dma_dev) to the carrier's device for DMA purposes:: + + ret = dma_set_mask_and_coherent(&mdev->dma_dev, DMA_BIT_MASK(dma_bits)); + if (rc) + /* Handle errors */ diff --git a/Documentation/driver-api/pti_intel_mid.rst b/Documentation/driver-api/pti_intel_mid.rst deleted file mode 100644 index bacc2a4ee89f..000000000000 --- a/Documentation/driver-api/pti_intel_mid.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -============= -Intel MID PTI -============= - -The Intel MID PTI project is HW implemented in Intel Atom -system-on-a-chip designs based on the Parallel Trace -Interface for MIPI P1149.7 cJTAG standard. The kernel solution -for this platform involves the following files:: - - ./include/linux/pti.h - ./drivers/.../n_tracesink.h - ./drivers/.../n_tracerouter.c - ./drivers/.../n_tracesink.c - ./drivers/.../pti.c - -pti.c is the driver that enables various debugging features -popular on platforms from certain mobile manufacturers. -n_tracerouter.c and n_tracesink.c allow extra system information to -be collected and routed to the pti driver, such as trace -debugging data from a modem. Although n_tracerouter -and n_tracesink are a part of the complete PTI solution, -these two line disciplines can work separately from -pti.c and route any data stream from one /dev/tty node -to another /dev/tty node via kernel-space. This provides -a stable, reliable connection that will not break unless -the user-space application shuts down (plus avoids -kernel->user->kernel context switch overheads of routing -data). - -An example debugging usage for this driver system: - - * Hook /dev/ttyPTI0 to syslogd. Opening this port will also start - a console device to further capture debugging messages to PTI. - * Hook /dev/ttyPTI1 to modem debugging data to write to PTI HW. - This is where n_tracerouter and n_tracesink are used. - * Hook /dev/pti to a user-level debugging application for writing - to PTI HW. - * `Use mipi_` Kernel Driver API in other device drivers for - debugging to PTI by first requesting a PTI write address via - mipi_request_masterchannel(1). - -Below is example pseudo-code on how a 'privileged' application -can hook up n_tracerouter and n_tracesink to any tty on -a system. 'Privileged' means the application has enough -privileges to successfully manipulate the ldisc drivers -but is not just blindly executing as 'root'. Keep in mind -the use of ioctl(,TIOCSETD,) is not specific to the n_tracerouter -and n_tracesink line discpline drivers but is a generic -operation for a program to use a line discpline driver -on a tty port other than the default n_tty: - -.. code-block:: c - - /////////// To hook up n_tracerouter and n_tracesink ///////// - - // Note that n_tracerouter depends on n_tracesink. - #include - #define ONE_TTY "/dev/ttyOne" - #define TWO_TTY "/dev/ttyTwo" - - // needed global to hand onto ldisc connection - static int g_fd_source = -1; - static int g_fd_sink = -1; - - // these two vars used to grab LDISC values from loaded ldisc drivers - // in OS. Look at /proc/tty/ldiscs to get the right numbers from - // the ldiscs loaded in the system. - int source_ldisc_num, sink_ldisc_num = -1; - int retval; - - g_fd_source = open(ONE_TTY, O_RDWR); // must be R/W - g_fd_sink = open(TWO_TTY, O_RDWR); // must be R/W - - if (g_fd_source <= 0) || (g_fd_sink <= 0) { - // doubt you'll want to use these exact error lines of code - printf("Error on open(). errno: %d\n",errno); - return errno; - } - - retval = ioctl(g_fd_sink, TIOCSETD, &sink_ldisc_num); - if (retval < 0) { - printf("Error on ioctl(). errno: %d\n", errno); - return errno; - } - - retval = ioctl(g_fd_source, TIOCSETD, &source_ldisc_num); - if (retval < 0) { - printf("Error on ioctl(). errno: %d\n", errno); - return errno; - } - - /////////// To disconnect n_tracerouter and n_tracesink //////// - - // First make sure data through the ldiscs has stopped. - - // Second, disconnect ldiscs. This provides a - // little cleaner shutdown on tty stack. - sink_ldisc_num = 0; - source_ldisc_num = 0; - ioctl(g_fd_uart, TIOCSETD, &sink_ldisc_num); - ioctl(g_fd_gadget, TIOCSETD, &source_ldisc_num); - - // Three, program closes connection, and cleanup: - close(g_fd_uart); - close(g_fd_gadget); - g_fd_uart = g_fd_gadget = NULL; diff --git a/Documentation/driver-api/surface_aggregator/client-api.rst b/Documentation/driver-api/surface_aggregator/client-api.rst new file mode 100644 index 000000000000..8e0b000d0e64 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/client-api.rst @@ -0,0 +1,38 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +=============================== +Client Driver API Documentation +=============================== + +.. contents:: + :depth: 2 + + +Serial Hub Communication +======================== + +.. kernel-doc:: include/linux/surface_aggregator/serial_hub.h + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_packet_layer.c + :export: + + +Controller and Core Interface +============================= + +.. kernel-doc:: include/linux/surface_aggregator/controller.h + +.. kernel-doc:: drivers/platform/surface/aggregator/controller.c + :export: + +.. kernel-doc:: drivers/platform/surface/aggregator/core.c + :export: + + +Client Bus and Client Device API +================================ + +.. kernel-doc:: include/linux/surface_aggregator/device.h + +.. kernel-doc:: drivers/platform/surface/aggregator/bus.c + :export: diff --git a/Documentation/driver-api/surface_aggregator/client.rst b/Documentation/driver-api/surface_aggregator/client.rst new file mode 100644 index 000000000000..26d13085a117 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/client.rst @@ -0,0 +1,393 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. |ssam_controller| replace:: :c:type:`struct ssam_controller ` +.. |ssam_device| replace:: :c:type:`struct ssam_device ` +.. |ssam_device_driver| replace:: :c:type:`struct ssam_device_driver ` +.. |ssam_client_bind| replace:: :c:func:`ssam_client_bind` +.. |ssam_client_link| replace:: :c:func:`ssam_client_link` +.. |ssam_get_controller| replace:: :c:func:`ssam_get_controller` +.. |ssam_controller_get| replace:: :c:func:`ssam_controller_get` +.. |ssam_controller_put| replace:: :c:func:`ssam_controller_put` +.. |ssam_device_alloc| replace:: :c:func:`ssam_device_alloc` +.. |ssam_device_add| replace:: :c:func:`ssam_device_add` +.. |ssam_device_remove| replace:: :c:func:`ssam_device_remove` +.. |ssam_device_driver_register| replace:: :c:func:`ssam_device_driver_register` +.. |ssam_device_driver_unregister| replace:: :c:func:`ssam_device_driver_unregister` +.. |module_ssam_device_driver| replace:: :c:func:`module_ssam_device_driver` +.. |SSAM_DEVICE| replace:: :c:func:`SSAM_DEVICE` +.. |ssam_notifier_register| replace:: :c:func:`ssam_notifier_register` +.. |ssam_notifier_unregister| replace:: :c:func:`ssam_notifier_unregister` +.. |ssam_request_sync| replace:: :c:func:`ssam_request_sync` +.. |ssam_event_mask| replace:: :c:type:`enum ssam_event_mask ` + + +====================== +Writing Client Drivers +====================== + +For the API documentation, refer to: + +.. toctree:: + :maxdepth: 2 + + client-api + + +Overview +======== + +Client drivers can be set up in two main ways, depending on how the +corresponding device is made available to the system. We specifically +differentiate between devices that are presented to the system via one of +the conventional ways, e.g. as platform devices via ACPI, and devices that +are non-discoverable and instead need to be explicitly provided by some +other mechanism, as discussed further below. + + +Non-SSAM Client Drivers +======================= + +All communication with the SAM EC is handled via the |ssam_controller| +representing that EC to the kernel. Drivers targeting a non-SSAM device (and +thus not being a |ssam_device_driver|) need to explicitly establish a +connection/relation to that controller. This can be done via the +|ssam_client_bind| function. Said function returns a reference to the SSAM +controller, but, more importantly, also establishes a device link between +client device and controller (this can also be done separate via +|ssam_client_link|). It is important to do this, as it, first, guarantees +that the returned controller is valid for use in the client driver for as +long as this driver is bound to its device, i.e. that the driver gets +unbound before the controller ever becomes invalid, and, second, as it +ensures correct suspend/resume ordering. This setup should be done in the +driver's probe function, and may be used to defer probing in case the SSAM +subsystem is not ready yet, for example: + +.. code-block:: c + + static int client_driver_probe(struct platform_device *pdev) + { + struct ssam_controller *ctrl; + + ctrl = ssam_client_bind(&pdev->dev); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl) == -ENODEV ? -EPROBE_DEFER : PTR_ERR(ctrl); + + // ... + + return 0; + } + +The controller may be separately obtained via |ssam_get_controller| and its +lifetime be guaranteed via |ssam_controller_get| and |ssam_controller_put|. +Note that none of these functions, however, guarantee that the controller +will not be shut down or suspended. These functions essentially only operate +on the reference, i.e. only guarantee a bare minimum of accessibility +without any guarantees at all on practical operability. + + +Adding SSAM Devices +=================== + +If a device does not already exist/is not already provided via conventional +means, it should be provided as |ssam_device| via the SSAM client device +hub. New devices can be added to this hub by entering their UID into the +corresponding registry. SSAM devices can also be manually allocated via +|ssam_device_alloc|, subsequently to which they have to be added via +|ssam_device_add| and eventually removed via |ssam_device_remove|. By +default, the parent of the device is set to the controller device provided +for allocation, however this may be changed before the device is added. Note +that, when changing the parent device, care must be taken to ensure that the +controller lifetime and suspend/resume ordering guarantees, in the default +setup provided through the parent-child relation, are preserved. If +necessary, by use of |ssam_client_link| as is done for non-SSAM client +drivers and described in more detail above. + +A client device must always be removed by the party which added the +respective device before the controller shuts down. Such removal can be +guaranteed by linking the driver providing the SSAM device to the controller +via |ssam_client_link|, causing it to unbind before the controller driver +unbinds. Client devices registered with the controller as parent are +automatically removed when the controller shuts down, but this should not be +relied upon, especially as this does not extend to client devices with a +different parent. + + +SSAM Client Drivers +=================== + +SSAM client device drivers are, in essence, no different than other device +driver types. They are represented via |ssam_device_driver| and bind to a +|ssam_device| via its UID (:c:type:`struct ssam_device.uid `) +member and the match table +(:c:type:`struct ssam_device_driver.match_table `), +which should be set when declaring the driver struct instance. Refer to the +|SSAM_DEVICE| macro documentation for more details on how to define members +of the driver's match table. + +The UID for SSAM client devices consists of a ``domain``, a ``category``, +a ``target``, an ``instance``, and a ``function``. The ``domain`` is used +differentiate between physical SAM devices +(:c:type:`SSAM_DOMAIN_SERIALHUB `), i.e. devices that can +be accessed via the Surface Serial Hub, and virtual ones +(:c:type:`SSAM_DOMAIN_VIRTUAL `), such as client-device +hubs, that have no real representation on the SAM EC and are solely used on +the kernel/driver-side. For physical devices, ``category`` represents the +target category, ``target`` the target ID, and ``instance`` the instance ID +used to access the physical SAM device. In addition, ``function`` references +a specific device functionality, but has no meaning to the SAM EC. The +(default) name of a client device is generated based on its UID. + +A driver instance can be registered via |ssam_device_driver_register| and +unregistered via |ssam_device_driver_unregister|. For convenience, the +|module_ssam_device_driver| macro may be used to define module init- and +exit-functions registering the driver. + +The controller associated with a SSAM client device can be found in its +:c:type:`struct ssam_device.ctrl ` member. This reference is +guaranteed to be valid for at least as long as the client driver is bound, +but should also be valid for as long as the client device exists. Note, +however, that access outside of the bound client driver must ensure that the +controller device is not suspended while making any requests or +(un-)registering event notifiers (and thus should generally be avoided). This +is guaranteed when the controller is accessed from inside the bound client +driver. + + +Making Synchronous Requests +=========================== + +Synchronous requests are (currently) the main form of host-initiated +communication with the EC. There are a couple of ways to define and execute +such requests, however, most of them boil down to something similar as shown +in the example below. This example defines a write-read request, meaning +that the caller provides an argument to the SAM EC and receives a response. +The caller needs to know the (maximum) length of the response payload and +provide a buffer for it. + +Care must be taken to ensure that any command payload data passed to the SAM +EC is provided in little-endian format and, similarly, any response payload +data received from it is converted from little-endian to host endianness. + +.. code-block:: c + + int perform_request(struct ssam_controller *ctrl, u32 arg, u32 *ret) + { + struct ssam_request rqst; + struct ssam_response resp; + int status; + + /* Convert request argument to little-endian. */ + __le32 arg_le = cpu_to_le32(arg); + __le32 ret_le = cpu_to_le32(0); + + /* + * Initialize request specification. Replace this with your values. + * The rqst.payload field may be NULL if rqst.length is zero, + * indicating that the request does not have any argument. + * + * Note: The request parameters used here are not valid, i.e. + * they do not correspond to an actual SAM/EC request. + */ + rqst.target_category = SSAM_SSH_TC_SAM; + rqst.target_id = 0x01; + rqst.command_id = 0x02; + rqst.instance_id = 0x03; + rqst.flags = SSAM_REQUEST_HAS_RESPONSE; + rqst.length = sizeof(arg_le); + rqst.payload = (u8 *)&arg_le; + + /* Initialize request response. */ + resp.capacity = sizeof(ret_le); + resp.length = 0; + resp.pointer = (u8 *)&ret_le; + + /* + * Perform actual request. The response pointer may be null in case + * the request does not have any response. This must be consistent + * with the SSAM_REQUEST_HAS_RESPONSE flag set in the specification + * above. + */ + status = ssam_request_sync(ctrl, &rqst, &resp); + + /* + * Alternatively use + * + * ssam_request_sync_onstack(ctrl, &rqst, &resp, sizeof(arg_le)); + * + * to perform the request, allocating the message buffer directly + * on the stack as opposed to allocation via kzalloc(). + */ + + /* + * Convert request response back to native format. Note that in the + * error case, this value is not touched by the SSAM core, i.e. + * 'ret_le' will be zero as specified in its initialization. + */ + *ret = le32_to_cpu(ret_le); + + return status; + } + +Note that |ssam_request_sync| in its essence is a wrapper over lower-level +request primitives, which may also be used to perform requests. Refer to its +implementation and documentation for more details. + +An arguably more user-friendly way of defining such functions is by using +one of the generator macros, for example via: + +.. code-block:: c + + SSAM_DEFINE_SYNC_REQUEST_W(__ssam_tmp_perf_mode_set, __le32, { + .target_category = SSAM_SSH_TC_TMP, + .target_id = 0x01, + .command_id = 0x03, + .instance_id = 0x00, + }); + +This example defines a function + +.. code-block:: c + + int __ssam_tmp_perf_mode_set(struct ssam_controller *ctrl, const __le32 *arg); + +executing the specified request, with the controller passed in when calling +said function. In this example, the argument is provided via the ``arg`` +pointer. Note that the generated function allocates the message buffer on +the stack. Thus, if the argument provided via the request is large, these +kinds of macros should be avoided. Also note that, in contrast to the +previous non-macro example, this function does not do any endianness +conversion, which has to be handled by the caller. Apart from those +differences the function generated by the macro is similar to the one +provided in the non-macro example above. + +The full list of such function-generating macros is + +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_N` for requests without return value and + without argument. +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_R` for requests with return value but no + argument. +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_W` for requests without return value but + with argument. + +Refer to their respective documentation for more details. For each one of +these macros, a special variant is provided, which targets request types +applicable to multiple instances of the same device type: + +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_MD_N` +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_MD_R` +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_MD_W` + +The difference of those macros to the previously mentioned versions is, that +the device target and instance IDs are not fixed for the generated function, +but instead have to be provided by the caller of said function. + +Additionally, variants for direct use with client devices, i.e. +|ssam_device|, are also provided. These can, for example, be used as +follows: + +.. code-block:: c + + SSAM_DEFINE_SYNC_REQUEST_CL_R(ssam_bat_get_sta, __le32, { + .target_category = SSAM_SSH_TC_BAT, + .command_id = 0x01, + }); + +This invocation of the macro defines a function + +.. code-block:: c + + int ssam_bat_get_sta(struct ssam_device *sdev, __le32 *ret); + +executing the specified request, using the device IDs and controller given +in the client device. The full list of such macros for client devices is: + +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_CL_N` +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_CL_R` +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_CL_W` + + +Handling Events +=============== + +To receive events from the SAM EC, an event notifier must be registered for +the desired event via |ssam_notifier_register|. The notifier must be +unregistered via |ssam_notifier_unregister| once it is not required any +more. + +Event notifiers are registered by providing (at minimum) a callback to call +in case an event has been received, the registry specifying how the event +should be enabled, an event ID specifying for which target category and, +optionally and depending on the registry used, for which instance ID events +should be enabled, and finally, flags describing how the EC will send these +events. If the specific registry does not enable events by instance ID, the +instance ID must be set to zero. Additionally, a priority for the respective +notifier may be specified, which determines its order in relation to any +other notifier registered for the same target category. + +By default, event notifiers will receive all events for the specific target +category, regardless of the instance ID specified when registering the +notifier. The core may be instructed to only call a notifier if the target +ID or instance ID (or both) of the event match the ones implied by the +notifier IDs (in case of target ID, the target ID of the registry), by +providing an event mask (see |ssam_event_mask|). + +In general, the target ID of the registry is also the target ID of the +enabled event (with the notable exception being keyboard input events on the +Surface Laptop 1 and 2, which are enabled via a registry with target ID 1, +but provide events with target ID 2). + +A full example for registering an event notifier and handling received +events is provided below: + +.. code-block:: c + + u32 notifier_callback(struct ssam_event_notifier *nf, + const struct ssam_event *event) + { + int status = ... + + /* Handle the event here ... */ + + /* Convert return value and indicate that we handled the event. */ + return ssam_notifier_from_errno(status) | SSAM_NOTIF_HANDLED; + } + + int setup_notifier(struct ssam_device *sdev, + struct ssam_event_notifier *nf) + { + /* Set priority wrt. other handlers of same target category. */ + nf->base.priority = 1; + + /* Set event/notifier callback. */ + nf->base.fn = notifier_callback; + + /* Specify event registry, i.e. how events get enabled/disabled. */ + nf->event.reg = SSAM_EVENT_REGISTRY_KIP; + + /* Specify which event to enable/disable */ + nf->event.id.target_category = sdev->uid.category; + nf->event.id.instance = sdev->uid.instance; + + /* + * Specify for which events the notifier callback gets executed. + * This essentially tells the core if it can skip notifiers that + * don't have target or instance IDs matching those of the event. + */ + nf->event.mask = SSAM_EVENT_MASK_STRICT; + + /* Specify event flags. */ + nf->event.flags = SSAM_EVENT_SEQUENCED; + + return ssam_notifier_register(sdev->ctrl, nf); + } + +Multiple event notifiers can be registered for the same event. The event +handler core takes care of enabling and disabling events when notifiers are +registered and unregistered, by keeping track of how many notifiers for a +specific event (combination of registry, event target category, and event +instance ID) are currently registered. This means that a specific event will +be enabled when the first notifier for it is being registered and disabled +when the last notifier for it is being unregistered. Note that the event +flags are therefore only used on the first registered notifier, however, one +should take care that notifiers for a specific event are always registered +with the same flag and it is considered a bug to do otherwise. diff --git a/Documentation/driver-api/surface_aggregator/clients/cdev.rst b/Documentation/driver-api/surface_aggregator/clients/cdev.rst new file mode 100644 index 000000000000..248c1372d879 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/clients/cdev.rst @@ -0,0 +1,87 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. |u8| replace:: :c:type:`u8 ` +.. |u16| replace:: :c:type:`u16 ` +.. |ssam_cdev_request| replace:: :c:type:`struct ssam_cdev_request ` +.. |ssam_cdev_request_flags| replace:: :c:type:`enum ssam_cdev_request_flags ` + +============================== +User-Space EC Interface (cdev) +============================== + +The ``surface_aggregator_cdev`` module provides a misc-device for the SSAM +controller to allow for a (more or less) direct connection from user-space to +the SAM EC. It is intended to be used for development and debugging, and +therefore should not be used or relied upon in any other way. Note that this +module is not loaded automatically, but instead must be loaded manually. + +The provided interface is accessible through the ``/dev/surface/aggregator`` +device-file. All functionality of this interface is provided via IOCTLs. +These IOCTLs and their respective input/output parameter structs are defined in +``include/uapi/linux/surface_aggregator/cdev.h``. + +A small python library and scripts for accessing this interface can be found +at https://github.com/linux-surface/surface-aggregator-module/tree/master/scripts/ssam. + + +Controller IOCTLs +================= + +The following IOCTLs are provided: + +.. flat-table:: Controller IOCTLs + :widths: 1 1 1 1 4 + :header-rows: 1 + + * - Type + - Number + - Direction + - Name + - Description + + * - ``0xA5`` + - ``1`` + - ``WR`` + - ``REQUEST`` + - Perform synchronous SAM request. + + +``REQUEST`` +----------- + +Defined as ``_IOWR(0xA5, 1, struct ssam_cdev_request)``. + +Executes a synchronous SAM request. The request specification is passed in +as argument of type |ssam_cdev_request|, which is then written to/modified +by the IOCTL to return status and result of the request. + +Request payload data must be allocated separately and is passed in via the +``payload.data`` and ``payload.length`` members. If a response is required, +the response buffer must be allocated by the caller and passed in via the +``response.data`` member. The ``response.length`` member must be set to the +capacity of this buffer, or if no response is required, zero. Upon +completion of the request, the call will write the response to the response +buffer (if its capacity allows it) and overwrite the length field with the +actual size of the response, in bytes. + +Additionally, if the request has a response, this must be indicated via the +request flags, as is done with in-kernel requests. Request flags can be set +via the ``flags`` member and the values correspond to the values found in +|ssam_cdev_request_flags|. + +Finally, the status of the request itself is returned in the ``status`` +member (a negative errno value indicating failure). Note that failure +indication of the IOCTL is separated from failure indication of the request: +The IOCTL returns a negative status code if anything failed during setup of +the request (``-EFAULT``) or if the provided argument or any of its fields +are invalid (``-EINVAL``). In this case, the status value of the request +argument may be set, providing more detail on what went wrong (e.g. +``-ENOMEM`` for out-of-memory), but this value may also be zero. The IOCTL +will return with a zero status code in case the request has been set up, +submitted, and completed (i.e. handed back to user-space) successfully from +inside the IOCTL, but the request ``status`` member may still be negative in +case the actual execution of the request failed after it has been submitted. + +A full definition of the argument struct is provided below: + +.. kernel-doc:: include/uapi/linux/surface_aggregator/cdev.h diff --git a/Documentation/driver-api/surface_aggregator/clients/index.rst b/Documentation/driver-api/surface_aggregator/clients/index.rst new file mode 100644 index 000000000000..3ccabce23271 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/clients/index.rst @@ -0,0 +1,21 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +=========================== +Client Driver Documentation +=========================== + +This is the documentation for client drivers themselves. Refer to +:doc:`../client` for documentation on how to write client drivers. + +.. toctree:: + :maxdepth: 1 + + cdev + san + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/driver-api/surface_aggregator/clients/san.rst b/Documentation/driver-api/surface_aggregator/clients/san.rst new file mode 100644 index 000000000000..38c2580e7758 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/clients/san.rst @@ -0,0 +1,44 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. |san_client_link| replace:: :c:func:`san_client_link` +.. |san_dgpu_notifier_register| replace:: :c:func:`san_dgpu_notifier_register` +.. |san_dgpu_notifier_unregister| replace:: :c:func:`san_dgpu_notifier_unregister` + +=================== +Surface ACPI Notify +=================== + +The Surface ACPI Notify (SAN) device provides the bridge between ACPI and +SAM controller. Specifically, ACPI code can execute requests and handle +battery and thermal events via this interface. In addition to this, events +relating to the discrete GPU (dGPU) of the Surface Book 2 can be sent from +ACPI code (note: the Surface Book 3 uses a different method for this). The +only currently known event sent via this interface is a dGPU power-on +notification. While this driver handles the former part internally, it only +relays the dGPU events to any other driver interested via its public API and +does not handle them. + +The public interface of this driver is split into two parts: Client +registration and notifier-block registration. + +A client to the SAN interface can be linked as consumer to the SAN device +via |san_client_link|. This can be used to ensure that the a client +receiving dGPU events does not miss any events due to the SAN interface not +being set up as this forces the client driver to unbind once the SAN driver +is unbound. + +Notifier-blocks can be registered by any device for as long as the module is +loaded, regardless of being linked as client or not. Registration is done +with |san_dgpu_notifier_register|. If the notifier is not needed any more, it +should be unregistered via |san_dgpu_notifier_unregister|. + +Consult the API documentation below for more details. + + +API Documentation +================= + +.. kernel-doc:: include/linux/surface_acpi_notify.h + +.. kernel-doc:: drivers/platform/surface/surface_acpi_notify.c + :export: diff --git a/Documentation/driver-api/surface_aggregator/index.rst b/Documentation/driver-api/surface_aggregator/index.rst new file mode 100644 index 000000000000..6f3e1094904d --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/index.rst @@ -0,0 +1,21 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +======================================= +Surface System Aggregator Module (SSAM) +======================================= + +.. toctree:: + :maxdepth: 2 + + overview + client + clients/index + ssh + internal + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/driver-api/surface_aggregator/internal-api.rst b/Documentation/driver-api/surface_aggregator/internal-api.rst new file mode 100644 index 000000000000..639a67b5a392 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/internal-api.rst @@ -0,0 +1,67 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +========================== +Internal API Documentation +========================== + +.. contents:: + :depth: 2 + + +Packet Transport Layer +====================== + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_parser.h + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_parser.c + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_msgb.h + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_packet_layer.h + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_packet_layer.c + :internal: + + +Request Transport Layer +======================= + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_request_layer.h + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_request_layer.c + :internal: + + +Controller +========== + +.. kernel-doc:: drivers/platform/surface/aggregator/controller.h + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/controller.c + :internal: + + +Client Device Bus +================= + +.. kernel-doc:: drivers/platform/surface/aggregator/bus.c + :internal: + + +Core +==== + +.. kernel-doc:: drivers/platform/surface/aggregator/core.c + :internal: + + +Trace Helpers +============= + +.. kernel-doc:: drivers/platform/surface/aggregator/trace.h diff --git a/Documentation/driver-api/surface_aggregator/internal.rst b/Documentation/driver-api/surface_aggregator/internal.rst new file mode 100644 index 000000000000..72704734982a --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/internal.rst @@ -0,0 +1,577 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. |ssh_ptl| replace:: :c:type:`struct ssh_ptl ` +.. |ssh_ptl_submit| replace:: :c:func:`ssh_ptl_submit` +.. |ssh_ptl_cancel| replace:: :c:func:`ssh_ptl_cancel` +.. |ssh_ptl_shutdown| replace:: :c:func:`ssh_ptl_shutdown` +.. |ssh_ptl_rx_rcvbuf| replace:: :c:func:`ssh_ptl_rx_rcvbuf` +.. |ssh_rtl| replace:: :c:type:`struct ssh_rtl ` +.. |ssh_rtl_submit| replace:: :c:func:`ssh_rtl_submit` +.. |ssh_rtl_cancel| replace:: :c:func:`ssh_rtl_cancel` +.. |ssh_rtl_shutdown| replace:: :c:func:`ssh_rtl_shutdown` +.. |ssh_packet| replace:: :c:type:`struct ssh_packet ` +.. |ssh_packet_get| replace:: :c:func:`ssh_packet_get` +.. |ssh_packet_put| replace:: :c:func:`ssh_packet_put` +.. |ssh_packet_ops| replace:: :c:type:`struct ssh_packet_ops ` +.. |ssh_packet_base_priority| replace:: :c:type:`enum ssh_packet_base_priority ` +.. |ssh_packet_flags| replace:: :c:type:`enum ssh_packet_flags ` +.. |SSH_PACKET_PRIORITY| replace:: :c:func:`SSH_PACKET_PRIORITY` +.. |ssh_frame| replace:: :c:type:`struct ssh_frame ` +.. |ssh_command| replace:: :c:type:`struct ssh_command ` +.. |ssh_request| replace:: :c:type:`struct ssh_request ` +.. |ssh_request_get| replace:: :c:func:`ssh_request_get` +.. |ssh_request_put| replace:: :c:func:`ssh_request_put` +.. |ssh_request_ops| replace:: :c:type:`struct ssh_request_ops ` +.. |ssh_request_init| replace:: :c:func:`ssh_request_init` +.. |ssh_request_flags| replace:: :c:type:`enum ssh_request_flags ` +.. |ssam_controller| replace:: :c:type:`struct ssam_controller ` +.. |ssam_device| replace:: :c:type:`struct ssam_device ` +.. |ssam_device_driver| replace:: :c:type:`struct ssam_device_driver ` +.. |ssam_client_bind| replace:: :c:func:`ssam_client_bind` +.. |ssam_client_link| replace:: :c:func:`ssam_client_link` +.. |ssam_request_sync| replace:: :c:type:`struct ssam_request_sync ` +.. |ssam_event_registry| replace:: :c:type:`struct ssam_event_registry ` +.. |ssam_event_id| replace:: :c:type:`struct ssam_event_id ` +.. |ssam_nf| replace:: :c:type:`struct ssam_nf ` +.. |ssam_nf_refcount_inc| replace:: :c:func:`ssam_nf_refcount_inc` +.. |ssam_nf_refcount_dec| replace:: :c:func:`ssam_nf_refcount_dec` +.. |ssam_notifier_register| replace:: :c:func:`ssam_notifier_register` +.. |ssam_notifier_unregister| replace:: :c:func:`ssam_notifier_unregister` +.. |ssam_cplt| replace:: :c:type:`struct ssam_cplt ` +.. |ssam_event_queue| replace:: :c:type:`struct ssam_event_queue ` +.. |ssam_request_sync_submit| replace:: :c:func:`ssam_request_sync_submit` + +===================== +Core Driver Internals +===================== + +Architectural overview of the Surface System Aggregator Module (SSAM) core +and Surface Serial Hub (SSH) driver. For the API documentation, refer to: + +.. toctree:: + :maxdepth: 2 + + internal-api + + +Overview +======== + +The SSAM core implementation is structured in layers, somewhat following the +SSH protocol structure: + +Lower-level packet transport is implemented in the *packet transport layer +(PTL)*, directly building on top of the serial device (serdev) +infrastructure of the kernel. As the name indicates, this layer deals with +the packet transport logic and handles things like packet validation, packet +acknowledgment (ACKing), packet (retransmission) timeouts, and relaying +packet payloads to higher-level layers. + +Above this sits the *request transport layer (RTL)*. This layer is centered +around command-type packet payloads, i.e. requests (sent from host to EC), +responses of the EC to those requests, and events (sent from EC to host). +It, specifically, distinguishes events from request responses, matches +responses to their corresponding requests, and implements request timeouts. + +The *controller* layer is building on top of this and essentially decides +how request responses and, especially, events are dealt with. It provides an +event notifier system, handles event activation/deactivation, provides a +workqueue for event and asynchronous request completion, and also manages +the message counters required for building command messages (``SEQ``, +``RQID``). This layer basically provides a fundamental interface to the SAM +EC for use in other kernel drivers. + +While the controller layer already provides an interface for other kernel +drivers, the client *bus* extends this interface to provide support for +native SSAM devices, i.e. devices that are not defined in ACPI and not +implemented as platform devices, via |ssam_device| and |ssam_device_driver| +simplify management of client devices and client drivers. + +Refer to :doc:`client` for documentation regarding the client device/driver +API and interface options for other kernel drivers. It is recommended to +familiarize oneself with that chapter and the :doc:`ssh` before continuing +with the architectural overview below. + + +Packet Transport Layer +====================== + +The packet transport layer is represented via |ssh_ptl| and is structured +around the following key concepts: + +Packets +------- + +Packets are the fundamental transmission unit of the SSH protocol. They are +managed by the packet transport layer, which is essentially the lowest layer +of the driver and is built upon by other components of the SSAM core. +Packets to be transmitted by the SSAM core are represented via |ssh_packet| +(in contrast, packets received by the core do not have any specific +structure and are managed entirely via the raw |ssh_frame|). + +This structure contains the required fields to manage the packet inside the +transport layer, as well as a reference to the buffer containing the data to +be transmitted (i.e. the message wrapped in |ssh_frame|). Most notably, it +contains an internal reference count, which is used for managing its +lifetime (accessible via |ssh_packet_get| and |ssh_packet_put|). When this +counter reaches zero, the ``release()`` callback provided to the packet via +its |ssh_packet_ops| reference is executed, which may then deallocate the +packet or its enclosing structure (e.g. |ssh_request|). + +In addition to the ``release`` callback, the |ssh_packet_ops| reference also +provides a ``complete()`` callback, which is run once the packet has been +completed and provides the status of this completion, i.e. zero on success +or a negative errno value in case of an error. Once the packet has been +submitted to the packet transport layer, the ``complete()`` callback is +always guaranteed to be executed before the ``release()`` callback, i.e. the +packet will always be completed, either successfully, with an error, or due +to cancellation, before it will be released. + +The state of a packet is managed via its ``state`` flags +(|ssh_packet_flags|), which also contains the packet type. In particular, +the following bits are noteworthy: + +* ``SSH_PACKET_SF_LOCKED_BIT``: This bit is set when completion, either + through error or success, is imminent. It indicates that no further + references of the packet should be taken and any existing references + should be dropped as soon as possible. The process setting this bit is + responsible for removing any references to this packet from the packet + queue and pending set. + +* ``SSH_PACKET_SF_COMPLETED_BIT``: This bit is set by the process running the + ``complete()`` callback and is used to ensure that this callback only runs + once. + +* ``SSH_PACKET_SF_QUEUED_BIT``: This bit is set when the packet is queued on + the packet queue and cleared when it is dequeued. + +* ``SSH_PACKET_SF_PENDING_BIT``: This bit is set when the packet is added to + the pending set and cleared when it is removed from it. + +Packet Queue +------------ + +The packet queue is the first of the two fundamental collections in the +packet transport layer. It is a priority queue, with priority of the +respective packets based on the packet type (major) and number of tries +(minor). See |SSH_PACKET_PRIORITY| for more details on the priority value. + +All packets to be transmitted by the transport layer must be submitted to +this queue via |ssh_ptl_submit|. Note that this includes control packets +sent by the transport layer itself. Internally, data packets can be +re-submitted to this queue due to timeouts or NAK packets sent by the EC. + +Pending Set +----------- + +The pending set is the second of the two fundamental collections in the +packet transport layer. It stores references to packets that have already +been transmitted, but wait for acknowledgment (e.g. the corresponding ACK +packet) by the EC. + +Note that a packet may both be pending and queued if it has been +re-submitted due to a packet acknowledgment timeout or NAK. On such a +re-submission, packets are not removed from the pending set. + +Transmitter Thread +------------------ + +The transmitter thread is responsible for most of the actual work regarding +packet transmission. In each iteration, it (waits for and) checks if the +next packet on the queue (if any) can be transmitted and, if so, removes it +from the queue and increments its counter for the number of transmission +attempts, i.e. tries. If the packet is sequenced, i.e. requires an ACK by +the EC, the packet is added to the pending set. Next, the packet's data is +submitted to the serdev subsystem. In case of an error or timeout during +this submission, the packet is completed by the transmitter thread with the +status value of the callback set accordingly. In case the packet is +unsequenced, i.e. does not require an ACK by the EC, the packet is completed +with success on the transmitter thread. + +Transmission of sequenced packets is limited by the number of concurrently +pending packets, i.e. a limit on how many packets may be waiting for an ACK +from the EC in parallel. This limit is currently set to one (see :doc:`ssh` +for the reasoning behind this). Control packets (i.e. ACK and NAK) can +always be transmitted. + +Receiver Thread +--------------- + +Any data received from the EC is put into a FIFO buffer for further +processing. This processing happens on the receiver thread. The receiver +thread parses and validates the received message into its |ssh_frame| and +corresponding payload. It prepares and submits the necessary ACK (and on +validation error or invalid data NAK) packets for the received messages. + +This thread also handles further processing, such as matching ACK messages +to the corresponding pending packet (via sequence ID) and completing it, as +well as initiating re-submission of all currently pending packets on +receival of a NAK message (re-submission in case of a NAK is similar to +re-submission due to timeout, see below for more details on that). Note that +the successful completion of a sequenced packet will always run on the +receiver thread (whereas any failure-indicating completion will run on the +process where the failure occurred). + +Any payload data is forwarded via a callback to the next upper layer, i.e. +the request transport layer. + +Timeout Reaper +-------------- + +The packet acknowledgment timeout is a per-packet timeout for sequenced +packets, started when the respective packet begins (re-)transmission (i.e. +this timeout is armed once per transmission attempt on the transmitter +thread). It is used to trigger re-submission or, when the number of tries +has been exceeded, cancellation of the packet in question. + +This timeout is handled via a dedicated reaper task, which is essentially a +work item (re-)scheduled to run when the next packet is set to time out. The +work item then checks the set of pending packets for any packets that have +exceeded the timeout and, if there are any remaining packets, re-schedules +itself to the next appropriate point in time. + +If a timeout has been detected by the reaper, the packet will either be +re-submitted if it still has some remaining tries left, or completed with +``-ETIMEDOUT`` as status if not. Note that re-submission, in this case and +triggered by receival of a NAK, means that the packet is added to the queue +with a now incremented number of tries, yielding a higher priority. The +timeout for the packet will be disabled until the next transmission attempt +and the packet remains on the pending set. + +Note that due to transmission and packet acknowledgment timeouts, the packet +transport layer is always guaranteed to make progress, if only through +timing out packets, and will never fully block. + +Concurrency and Locking +----------------------- + +There are two main locks in the packet transport layer: One guarding access +to the packet queue and one guarding access to the pending set. These +collections may only be accessed and modified under the respective lock. If +access to both collections is needed, the pending lock must be acquired +before the queue lock to avoid deadlocks. + +In addition to guarding the collections, after initial packet submission +certain packet fields may only be accessed under one of the locks. +Specifically, the packet priority must only be accessed while holding the +queue lock and the packet timestamp must only be accessed while holding the +pending lock. + +Other parts of the packet transport layer are guarded independently. State +flags are managed by atomic bit operations and, if necessary, memory +barriers. Modifications to the timeout reaper work item and expiration date +are guarded by their own lock. + +The reference of the packet to the packet transport layer (``ptl``) is +somewhat special. It is either set when the upper layer request is submitted +or, if there is none, when the packet is first submitted. After it is set, +it will not change its value. Functions that may run concurrently with +submission, i.e. cancellation, can not rely on the ``ptl`` reference to be +set. Access to it in these functions is guarded by ``READ_ONCE()``, whereas +setting ``ptl`` is equally guarded with ``WRITE_ONCE()`` for symmetry. + +Some packet fields may be read outside of the respective locks guarding +them, specifically priority and state for tracing. In those cases, proper +access is ensured by employing ``WRITE_ONCE()`` and ``READ_ONCE()``. Such +read-only access is only allowed when stale values are not critical. + +With respect to the interface for higher layers, packet submission +(|ssh_ptl_submit|), packet cancellation (|ssh_ptl_cancel|), data receival +(|ssh_ptl_rx_rcvbuf|), and layer shutdown (|ssh_ptl_shutdown|) may always be +executed concurrently with respect to each other. Note that packet +submission may not run concurrently with itself for the same packet. +Equally, shutdown and data receival may also not run concurrently with +themselves (but may run concurrently with each other). + + +Request Transport Layer +======================= + +The request transport layer is represented via |ssh_rtl| and builds on top +of the packet transport layer. It deals with requests, i.e. SSH packets sent +by the host containing a |ssh_command| as frame payload. This layer +separates responses to requests from events, which are also sent by the EC +via a |ssh_command| payload. While responses are handled in this layer, +events are relayed to the next upper layer, i.e. the controller layer, via +the corresponding callback. The request transport layer is structured around +the following key concepts: + +Request +------- + +Requests are packets with a command-type payload, sent from host to EC to +query data from or trigger an action on it (or both simultaneously). They +are represented by |ssh_request|, wrapping the underlying |ssh_packet| +storing its message data (i.e. SSH frame with command payload). Note that +all top-level representations, e.g. |ssam_request_sync| are built upon this +struct. + +As |ssh_request| extends |ssh_packet|, its lifetime is also managed by the +reference counter inside the packet struct (which can be accessed via +|ssh_request_get| and |ssh_request_put|). Once the counter reaches zero, the +``release()`` callback of the |ssh_request_ops| reference of the request is +called. + +Requests can have an optional response that is equally sent via a SSH +message with command-type payload (from EC to host). The party constructing +the request must know if a response is expected and mark this in the request +flags provided to |ssh_request_init|, so that the request transport layer +can wait for this response. + +Similar to |ssh_packet|, |ssh_request| also has a ``complete()`` callback +provided via its request ops reference and is guaranteed to be completed +before it is released once it has been submitted to the request transport +layer via |ssh_rtl_submit|. For a request without a response, successful +completion will occur once the underlying packet has been successfully +transmitted by the packet transport layer (i.e. from within the packet +completion callback). For a request with response, successful completion +will occur once the response has been received and matched to the request +via its request ID (which happens on the packet layer's data-received +callback running on the receiver thread). If the request is completed with +an error, the status value will be set to the corresponding (negative) errno +value. + +The state of a request is again managed via its ``state`` flags +(|ssh_request_flags|), which also encode the request type. In particular, +the following bits are noteworthy: + +* ``SSH_REQUEST_SF_LOCKED_BIT``: This bit is set when completion, either + through error or success, is imminent. It indicates that no further + references of the request should be taken and any existing references + should be dropped as soon as possible. The process setting this bit is + responsible for removing any references to this request from the request + queue and pending set. + +* ``SSH_REQUEST_SF_COMPLETED_BIT``: This bit is set by the process running the + ``complete()`` callback and is used to ensure that this callback only runs + once. + +* ``SSH_REQUEST_SF_QUEUED_BIT``: This bit is set when the request is queued on + the request queue and cleared when it is dequeued. + +* ``SSH_REQUEST_SF_PENDING_BIT``: This bit is set when the request is added to + the pending set and cleared when it is removed from it. + +Request Queue +------------- + +The request queue is the first of the two fundamental collections in the +request transport layer. In contrast to the packet queue of the packet +transport layer, it is not a priority queue and the simple first come first +serve principle applies. + +All requests to be transmitted by the request transport layer must be +submitted to this queue via |ssh_rtl_submit|. Once submitted, requests may +not be re-submitted, and will not be re-submitted automatically on timeout. +Instead, the request is completed with a timeout error. If desired, the +caller can create and submit a new request for another try, but it must not +submit the same request again. + +Pending Set +----------- + +The pending set is the second of the two fundamental collections in the +request transport layer. This collection stores references to all pending +requests, i.e. requests awaiting a response from the EC (similar to what the +pending set of the packet transport layer does for packets). + +Transmitter Task +---------------- + +The transmitter task is scheduled when a new request is available for +transmission. It checks if the next request on the request queue can be +transmitted and, if so, submits its underlying packet to the packet +transport layer. This check ensures that only a limited number of +requests can be pending, i.e. waiting for a response, at the same time. If +the request requires a response, the request is added to the pending set +before its packet is submitted. + +Packet Completion Callback +-------------------------- + +The packet completion callback is executed once the underlying packet of a +request has been completed. In case of an error completion, the +corresponding request is completed with the error value provided in this +callback. + +On successful packet completion, further processing depends on the request. +If the request expects a response, it is marked as transmitted and the +request timeout is started. If the request does not expect a response, it is +completed with success. + +Data-Received Callback +---------------------- + +The data received callback notifies the request transport layer of data +being received by the underlying packet transport layer via a data-type +frame. In general, this is expected to be a command-type payload. + +If the request ID of the command is one of the request IDs reserved for +events (one to ``SSH_NUM_EVENTS``, inclusively), it is forwarded to the +event callback registered in the request transport layer. If the request ID +indicates a response to a request, the respective request is looked up in +the pending set and, if found and marked as transmitted, completed with +success. + +Timeout Reaper +-------------- + +The request-response-timeout is a per-request timeout for requests expecting +a response. It is used to ensure that a request does not wait indefinitely +on a response from the EC and is started after the underlying packet has +been successfully completed. + +This timeout is, similar to the packet acknowledgment timeout on the packet +transport layer, handled via a dedicated reaper task. This task is +essentially a work-item (re-)scheduled to run when the next request is set +to time out. The work item then scans the set of pending requests for any +requests that have timed out and completes them with ``-ETIMEDOUT`` as +status. Requests will not be re-submitted automatically. Instead, the issuer +of the request must construct and submit a new request, if so desired. + +Note that this timeout, in combination with packet transmission and +acknowledgment timeouts, guarantees that the request layer will always make +progress, even if only through timing out packets, and never fully block. + +Concurrency and Locking +----------------------- + +Similar to the packet transport layer, there are two main locks in the +request transport layer: One guarding access to the request queue and one +guarding access to the pending set. These collections may only be accessed +and modified under the respective lock. + +Other parts of the request transport layer are guarded independently. State +flags are (again) managed by atomic bit operations and, if necessary, memory +barriers. Modifications to the timeout reaper work item and expiration date +are guarded by their own lock. + +Some request fields may be read outside of the respective locks guarding +them, specifically the state for tracing. In those cases, proper access is +ensured by employing ``WRITE_ONCE()`` and ``READ_ONCE()``. Such read-only +access is only allowed when stale values are not critical. + +With respect to the interface for higher layers, request submission +(|ssh_rtl_submit|), request cancellation (|ssh_rtl_cancel|), and layer +shutdown (|ssh_rtl_shutdown|) may always be executed concurrently with +respect to each other. Note that request submission may not run concurrently +with itself for the same request (and also may only be called once per +request). Equally, shutdown may also not run concurrently with itself. + + +Controller Layer +================ + +The controller layer extends on the request transport layer to provide an +easy-to-use interface for client drivers. It is represented by +|ssam_controller| and the SSH driver. While the lower level transport layers +take care of transmitting and handling packets and requests, the controller +layer takes on more of a management role. Specifically, it handles device +initialization, power management, and event handling, including event +delivery and registration via the (event) completion system (|ssam_cplt|). + +Event Registration +------------------ + +In general, an event (or rather a class of events) has to be explicitly +requested by the host before the EC will send it (HID input events seem to +be the exception). This is done via an event-enable request (similarly, +events should be disabled via an event-disable request once no longer +desired). + +The specific request used to enable (or disable) an event is given via an +event registry, i.e. the governing authority of this event (so to speak), +represented by |ssam_event_registry|. As parameters to this request, the +target category and, depending on the event registry, instance ID of the +event to be enabled must be provided. This (optional) instance ID must be +zero if the registry does not use it. Together, target category and instance +ID form the event ID, represented by |ssam_event_id|. In short, both, event +registry and event ID, are required to uniquely identify a respective class +of events. + +Note that a further *request ID* parameter must be provided for the +enable-event request. This parameter does not influence the class of events +being enabled, but instead is set as the request ID (RQID) on each event of +this class sent by the EC. It is used to identify events (as a limited +number of request IDs is reserved for use in events only, specifically one +to ``SSH_NUM_EVENTS`` inclusively) and also map events to their specific +class. Currently, the controller always sets this parameter to the target +category specified in |ssam_event_id|. + +As multiple client drivers may rely on the same (or overlapping) classes of +events and enable/disable calls are strictly binary (i.e. on/off), the +controller has to manage access to these events. It does so via reference +counting, storing the counter inside an RB-tree based mapping with event +registry and ID as key (there is no known list of valid event registry and +event ID combinations). See |ssam_nf|, |ssam_nf_refcount_inc|, and +|ssam_nf_refcount_dec| for details. + +This management is done together with notifier registration (described in +the next section) via the top-level |ssam_notifier_register| and +|ssam_notifier_unregister| functions. + +Event Delivery +-------------- + +To receive events, a client driver has to register an event notifier via +|ssam_notifier_register|. This increments the reference counter for that +specific class of events (as detailed in the previous section), enables the +class on the EC (if it has not been enabled already), and installs the +provided notifier callback. + +Notifier callbacks are stored in lists, with one (RCU) list per target +category (provided via the event ID; NB: there is a fixed known number of +target categories). There is no known association from the combination of +event registry and event ID to the command data (target ID, target category, +command ID, and instance ID) that can be provided by an event class, apart +from target category and instance ID given via the event ID. + +Note that due to the way notifiers are (or rather have to be) stored, client +drivers may receive events that they have not requested and need to account +for them. Specifically, they will, by default, receive all events from the +same target category. To simplify dealing with this, filtering of events by +target ID (provided via the event registry) and instance ID (provided via +the event ID) can be requested when registering a notifier. This filtering +is applied when iterating over the notifiers at the time they are executed. + +All notifier callbacks are executed on a dedicated workqueue, the so-called +completion workqueue. After an event has been received via the callback +installed in the request layer (running on the receiver thread of the packet +transport layer), it will be put on its respective event queue +(|ssam_event_queue|). From this event queue the completion work item of that +queue (running on the completion workqueue) will pick up the event and +execute the notifier callback. This is done to avoid blocking on the +receiver thread. + +There is one event queue per combination of target ID and target category. +This is done to ensure that notifier callbacks are executed in sequence for +events of the same target ID and target category. Callbacks can be executed +in parallel for events with a different combination of target ID and target +category. + +Concurrency and Locking +----------------------- + +Most of the concurrency related safety guarantees of the controller are +provided by the lower-level request transport layer. In addition to this, +event (un-)registration is guarded by its own lock. + +Access to the controller state is guarded by the state lock. This lock is a +read/write semaphore. The reader part can be used to ensure that the state +does not change while functions depending on the state to stay the same +(e.g. |ssam_notifier_register|, |ssam_notifier_unregister|, +|ssam_request_sync_submit|, and derivatives) are executed and this guarantee +is not already provided otherwise (e.g. through |ssam_client_bind| or +|ssam_client_link|). The writer part guards any transitions that will change +the state, i.e. initialization, destruction, suspension, and resumption. + +The controller state may be accessed (read-only) outside the state lock for +smoke-testing against invalid API usage (e.g. in |ssam_request_sync_submit|). +Note that such checks are not supposed to (and will not) protect against all +invalid usages, but rather aim to help catch them. In those cases, proper +variable access is ensured by employing ``WRITE_ONCE()`` and ``READ_ONCE()``. + +Assuming any preconditions on the state not changing have been satisfied, +all non-initialization and non-shutdown functions may run concurrently with +each other. This includes |ssam_notifier_register|, |ssam_notifier_unregister|, +|ssam_request_sync_submit|, as well as all functions building on top of those. diff --git a/Documentation/driver-api/surface_aggregator/overview.rst b/Documentation/driver-api/surface_aggregator/overview.rst new file mode 100644 index 000000000000..1e9d57e50063 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/overview.rst @@ -0,0 +1,77 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +======== +Overview +======== + +The Surface/System Aggregator Module (SAM, SSAM) is an (arguably *the*) +embedded controller (EC) on Microsoft Surface devices. It has been originally +introduced on 4th generation devices (Surface Pro 4, Surface Book 1), but +its responsibilities and feature-set have since been expanded significantly +with the following generations. + + +Features and Integration +======================== + +Not much is currently known about SAM on 4th generation devices (Surface Pro +4, Surface Book 1), due to the use of a different communication interface +between host and EC (as detailed below). On 5th (Surface Pro 2017, Surface +Book 2, Surface Laptop 1) and later generation devices, SAM is responsible +for providing battery information (both current status and static values, +such as maximum capacity etc.), as well as an assortment of temperature +sensors (e.g. skin temperature) and cooling/performance-mode setting to the +host. On the Surface Book 2, specifically, it additionally provides an +interface for properly handling clipboard detachment (i.e. separating the +display part from the keyboard part of the device), on the Surface Laptop 1 +and 2 it is required for keyboard HID input. This HID subsystem has been +restructured for 7th generation devices and on those, specifically Surface +Laptop 3 and Surface Book 3, is responsible for all major HID input (i.e. +keyboard and touchpad). + +While features have not changed much on a coarse level since the 5th +generation, internal interfaces have undergone some rather large changes. On +5th and 6th generation devices, both battery and temperature information is +exposed to ACPI via a shim driver (referred to as Surface ACPI Notify, or +SAN), translating ACPI generic serial bus write-/read-accesses to SAM +requests. On 7th generation devices, this additional layer is gone and these +devices require a driver hooking directly into the SAM interface. Equally, +on newer generations, less devices are declared in ACPI, making them a bit +harder to discover and requiring us to hard-code a sort of device registry. +Due to this, a SSAM bus and subsystem with client devices +(:c:type:`struct ssam_device `) has been implemented. + + +Communication +============= + +The type of communication interface between host and EC depends on the +generation of the Surface device. On 4th generation devices, host and EC +communicate via HID, specifically using a HID-over-I2C device, whereas on +5th and later generations, communication takes place via a USART serial +device. In accordance to the drivers found on other operating systems, we +refer to the serial device and its driver as Surface Serial Hub (SSH). When +needed, we differentiate between both types of SAM by referring to them as +SAM-over-SSH and SAM-over-HID. + +Currently, this subsystem only supports SAM-over-SSH. The SSH communication +interface is described in more detail below. The HID interface has not been +reverse engineered yet and it is, at the moment, unclear how many (and +which) concepts of the SSH interface detailed below can be transferred to +it. + +Surface Serial Hub +------------------ + +As already elaborated above, the Surface Serial Hub (SSH) is the +communication interface for SAM on 5th- and all later-generation Surface +devices. On the highest level, communication can be separated into two main +types: Requests, messages sent from host to EC that may trigger a direct +response from the EC (explicitly associated with the request), and events +(sometimes also referred to as notifications), sent from EC to host without +being a direct response to a previous request. We may also refer to requests +without response as commands. In general, events need to be enabled via one +of multiple dedicated requests before they are sent by the EC. + +See :doc:`ssh` for a more technical protocol documentation and +:doc:`internal` for an overview of the internal driver architecture. diff --git a/Documentation/driver-api/surface_aggregator/ssh.rst b/Documentation/driver-api/surface_aggregator/ssh.rst new file mode 100644 index 000000000000..bf007d6c9873 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/ssh.rst @@ -0,0 +1,344 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. |u8| replace:: :c:type:`u8 ` +.. |u16| replace:: :c:type:`u16 ` +.. |TYPE| replace:: ``TYPE`` +.. |LEN| replace:: ``LEN`` +.. |SEQ| replace:: ``SEQ`` +.. |SYN| replace:: ``SYN`` +.. |NAK| replace:: ``NAK`` +.. |ACK| replace:: ``ACK`` +.. |DATA| replace:: ``DATA`` +.. |DATA_SEQ| replace:: ``DATA_SEQ`` +.. |DATA_NSQ| replace:: ``DATA_NSQ`` +.. |TC| replace:: ``TC`` +.. |TID| replace:: ``TID`` +.. |IID| replace:: ``IID`` +.. |RQID| replace:: ``RQID`` +.. |CID| replace:: ``CID`` + +=========================== +Surface Serial Hub Protocol +=========================== + +The Surface Serial Hub (SSH) is the central communication interface for the +embedded Surface Aggregator Module controller (SAM or EC), found on newer +Surface generations. We will refer to this protocol and interface as +SAM-over-SSH, as opposed to SAM-over-HID for the older generations. + +On Surface devices with SAM-over-SSH, SAM is connected to the host via UART +and defined in ACPI as device with ID ``MSHW0084``. On these devices, +significant functionality is provided via SAM, including access to battery +and power information and events, thermal read-outs and events, and many +more. For Surface Laptops, keyboard input is handled via HID directed +through SAM, on the Surface Laptop 3 and Surface Book 3 this also includes +touchpad input. + +Note that the standard disclaimer for this subsystem also applies to this +document: All of this has been reverse-engineered and may thus be erroneous +and/or incomplete. + +All CRCs used in the following are two-byte ``crc_ccitt_false(0xffff, ...)``. +All multi-byte values are little-endian, there is no implicit padding between +values. + + +SSH Packet Protocol: Definitions +================================ + +The fundamental communication unit of the SSH protocol is a frame +(:c:type:`struct ssh_frame `). A frame consists of the following +fields, packed together and in order: + +.. flat-table:: SSH Frame + :widths: 1 1 4 + :header-rows: 1 + + * - Field + - Type + - Description + + * - |TYPE| + - |u8| + - Type identifier of the frame. + + * - |LEN| + - |u16| + - Length of the payload associated with the frame. + + * - |SEQ| + - |u8| + - Sequence ID (see explanation below). + +Each frame structure is followed by a CRC over this structure. The CRC over +the frame structure (|TYPE|, |LEN|, and |SEQ| fields) is placed directly +after the frame structure and before the payload. The payload is followed by +its own CRC (over all payload bytes). If the payload is not present (i.e. +the frame has ``LEN=0``), the CRC of the payload is still present and will +evaluate to ``0xffff``. The |LEN| field does not include any of the CRCs, it +equals the number of bytes inbetween the CRC of the frame and the CRC of the +payload. + +Additionally, the following fixed two-byte sequences are used: + +.. flat-table:: SSH Byte Sequences + :widths: 1 1 4 + :header-rows: 1 + + * - Name + - Value + - Description + + * - |SYN| + - ``[0xAA, 0x55]`` + - Synchronization bytes. + +A message consists of |SYN|, followed by the frame (|TYPE|, |LEN|, |SEQ| and +CRC) and, if specified in the frame (i.e. ``LEN > 0``), payload bytes, +followed finally, regardless if the payload is present, the payload CRC. The +messages corresponding to an exchange are, in part, identified by having the +same sequence ID (|SEQ|), stored inside the frame (more on this in the next +section). The sequence ID is a wrapping counter. + +A frame can have the following types +(:c:type:`enum ssh_frame_type `): + +.. flat-table:: SSH Frame Types + :widths: 1 1 4 + :header-rows: 1 + + * - Name + - Value + - Short Description + + * - |NAK| + - ``0x04`` + - Sent on error in previously received message. + + * - |ACK| + - ``0x40`` + - Sent to acknowledge receival of |DATA| frame. + + * - |DATA_SEQ| + - ``0x80`` + - Sent to transfer data. Sequenced. + + * - |DATA_NSQ| + - ``0x00`` + - Same as |DATA_SEQ|, but does not need to be ACKed. + +Both |NAK|- and |ACK|-type frames are used to control flow of messages and +thus do not carry a payload. |DATA_SEQ|- and |DATA_NSQ|-type frames on the +other hand must carry a payload. The flow sequence and interaction of +different frame types will be described in more depth in the next section. + + +SSH Packet Protocol: Flow Sequence +================================== + +Each exchange begins with |SYN|, followed by a |DATA_SEQ|- or +|DATA_NSQ|-type frame, followed by its CRC, payload, and payload CRC. In +case of a |DATA_NSQ|-type frame, the exchange is then finished. In case of a +|DATA_SEQ|-type frame, the receiving party has to acknowledge receival of +the frame by responding with a message containing an |ACK|-type frame with +the same sequence ID of the |DATA| frame. In other words, the sequence ID of +the |ACK| frame specifies the |DATA| frame to be acknowledged. In case of an +error, e.g. an invalid CRC, the receiving party responds with a message +containing an |NAK|-type frame. As the sequence ID of the previous data +frame, for which an error is indicated via the |NAK| frame, cannot be relied +upon, the sequence ID of the |NAK| frame should not be used and is set to +zero. After receival of an |NAK| frame, the sending party should re-send all +outstanding (non-ACKed) messages. + +Sequence IDs are not synchronized between the two parties, meaning that they +are managed independently for each party. Identifying the messages +corresponding to a single exchange thus relies on the sequence ID as well as +the type of the message, and the context. Specifically, the sequence ID is +used to associate an ``ACK`` with its ``DATA_SEQ``-type frame, but not +``DATA_SEQ``- or ``DATA_NSQ``-type frames with other ``DATA``- type frames. + +An example exchange might look like this: + +:: + + tx: -- SYN FRAME(D) CRC(F) PAYLOAD CRC(P) ----------------------------- + rx: ------------------------------------- SYN FRAME(A) CRC(F) CRC(P) -- + +where both frames have the same sequence ID (``SEQ``). Here, ``FRAME(D)`` +indicates a |DATA_SEQ|-type frame, ``FRAME(A)`` an ``ACK``-type frame, +``CRC(F)`` the CRC over the previous frame, ``CRC(P)`` the CRC over the +previous payload. In case of an error, the exchange would look like this: + +:: + + tx: -- SYN FRAME(D) CRC(F) PAYLOAD CRC(P) ----------------------------- + rx: ------------------------------------- SYN FRAME(N) CRC(F) CRC(P) -- + +upon which the sender should re-send the message. ``FRAME(N)`` indicates an +|NAK|-type frame. Note that the sequence ID of the |NAK|-type frame is fixed +to zero. For |DATA_NSQ|-type frames, both exchanges are the same: + +:: + + tx: -- SYN FRAME(DATA_NSQ) CRC(F) PAYLOAD CRC(P) ---------------------- + rx: ------------------------------------------------------------------- + +Here, an error can be detected, but not corrected or indicated to the +sending party. These exchanges are symmetric, i.e. switching ``rx`` and +``tx`` results again in a valid exchange. Currently, no longer exchanges are +known. + + +Commands: Requests, Responses, and Events +========================================= + +Commands are sent as payload inside a data frame. Currently, this is the +only known payload type of |DATA| frames, with a payload-type value of +``0x80`` (:c:type:`SSH_PLD_TYPE_CMD `). + +The command-type payload (:c:type:`struct ssh_command `) +consists of an eight-byte command structure, followed by optional and +variable length command data. The length of this optional data is derived +from the frame payload length given in the corresponding frame, i.e. it is +``frame.len - sizeof(struct ssh_command)``. The command struct contains the +following fields, packed together and in order: + +.. flat-table:: SSH Command + :widths: 1 1 4 + :header-rows: 1 + + * - Field + - Type + - Description + + * - |TYPE| + - |u8| + - Type of the payload. For commands always ``0x80``. + + * - |TC| + - |u8| + - Target category. + + * - |TID| (out) + - |u8| + - Target ID for outgoing (host to EC) commands. + + * - |TID| (in) + - |u8| + - Target ID for incoming (EC to host) commands. + + * - |IID| + - |u8| + - Instance ID. + + * - |RQID| + - |u16| + - Request ID. + + * - |CID| + - |u8| + - Command ID. + +The command struct and data, in general, does not contain any failure +detection mechanism (e.g. CRCs), this is solely done on the frame level. + +Command-type payloads are used by the host to send commands and requests to +the EC as well as by the EC to send responses and events back to the host. +We differentiate between requests (sent by the host), responses (sent by the +EC in response to a request), and events (sent by the EC without a preceding +request). + +Commands and events are uniquely identified by their target category +(``TC``) and command ID (``CID``). The target category specifies a general +category for the command (e.g. system in general, vs. battery and AC, vs. +temperature, and so on), while the command ID specifies the command inside +that category. Only the combination of |TC| + |CID| is unique. Additionally, +commands have an instance ID (``IID``), which is used to differentiate +between different sub-devices. For example ``TC=3`` ``CID=1`` is a +request to get the temperature on a thermal sensor, where |IID| specifies +the respective sensor. If the instance ID is not used, it should be set to +zero. If instance IDs are used, they, in general, start with a value of one, +whereas zero may be used for instance independent queries, if applicable. A +response to a request should have the same target category, command ID, and +instance ID as the corresponding request. + +Responses are matched to their corresponding request via the request ID +(``RQID``) field. This is a 16 bit wrapping counter similar to the sequence +ID on the frames. Note that the sequence ID of the frames for a +request-response pair does not match. Only the request ID has to match. +Frame-protocol wise these are two separate exchanges, and may even be +separated, e.g. by an event being sent after the request but before the +response. Not all commands produce a response, and this is not detectable by +|TC| + |CID|. It is the responsibility of the issuing party to wait for a +response (or signal this to the communication framework, as is done in +SAN/ACPI via the ``SNC`` flag). + +Events are identified by unique and reserved request IDs. These IDs should +not be used by the host when sending a new request. They are used on the +host to, first, detect events and, second, match them with a registered +event handler. Request IDs for events are chosen by the host and directed to +the EC when setting up and enabling an event source (via the +enable-event-source request). The EC then uses the specified request ID for +events sent from the respective source. Note that an event should still be +identified by its target category, command ID, and, if applicable, instance +ID, as a single event source can send multiple different event types. In +general, however, a single target category should map to a single reserved +event request ID. + +Furthermore, requests, responses, and events have an associated target ID +(``TID``). This target ID is split into output (host to EC) and input (EC to +host) fields, with the respecting other field (e.g. output field on incoming +messages) set to zero. Two ``TID`` values are known: Primary (``0x01``) and +secondary (``0x02``). In general, the response to a request should have the +same ``TID`` value, however, the field (output vs. input) should be used in +accordance to the direction in which the response is sent (i.e. on the input +field, as responses are generally sent from the EC to the host). + +Note that, even though requests and events should be uniquely identifiable +by target category and command ID alone, the EC may require specific +target ID and instance ID values to accept a command. A command that is +accepted for ``TID=1``, for example, may not be accepted for ``TID=2`` +and vice versa. + + +Limitations and Observations +============================ + +The protocol can, in theory, handle up to ``U8_MAX`` frames in parallel, +with up to ``U16_MAX`` pending requests (neglecting request IDs reserved for +events). In practice, however, this is more limited. From our testing +(although via a python and thus a user-space program), it seems that the EC +can handle up to four requests (mostly) reliably in parallel at a certain +time. With five or more requests in parallel, consistent discarding of +commands (ACKed frame but no command response) has been observed. For five +simultaneous commands, this reproducibly resulted in one command being +dropped and four commands being handled. + +However, it has also been noted that, even with three requests in parallel, +occasional frame drops happen. Apart from this, with a limit of three +pending requests, no dropped commands (i.e. command being dropped but frame +carrying command being ACKed) have been observed. In any case, frames (and +possibly also commands) should be re-sent by the host if a certain timeout +is exceeded. This is done by the EC for frames with a timeout of one second, +up to two re-tries (i.e. three transmissions in total). The limit of +re-tries also applies to received NAKs, and, in a worst case scenario, can +lead to entire messages being dropped. + +While this also seems to work fine for pending data frames as long as no +transmission failures occur, implementation and handling of these seems to +depend on the assumption that there is only one non-acknowledged data frame. +In particular, the detection of repeated frames relies on the last sequence +number. This means that, if a frame that has been successfully received by +the EC is sent again, e.g. due to the host not receiving an |ACK|, the EC +will only detect this if it has the sequence ID of the last frame received +by the EC. As an example: Sending two frames with ``SEQ=0`` and ``SEQ=1`` +followed by a repetition of ``SEQ=0`` will not detect the second ``SEQ=0`` +frame as such, and thus execute the command in this frame each time it has +been received, i.e. twice in this example. Sending ``SEQ=0``, ``SEQ=1`` and +then repeating ``SEQ=1`` will detect the second ``SEQ=1`` as repetition of +the first one and ignore it, thus executing the contained command only once. + +In conclusion, this suggests a limit of at most one pending un-ACKed frame +(per party, effectively leading to synchronous communication regarding +frames) and at most three pending commands. The limit to synchronous frame +transfers seems to be consistent with behavior observed on Windows. diff --git a/Documentation/driver-api/thermal/sysfs-api.rst b/Documentation/driver-api/thermal/sysfs-api.rst index e7520cb439ac..29fdd817ddb0 100644 --- a/Documentation/driver-api/thermal/sysfs-api.rst +++ b/Documentation/driver-api/thermal/sysfs-api.rst @@ -54,7 +54,7 @@ temperature) and throttle appropriate devices. trips: the total number of trip points this thermal zone supports. mask: - Bit string: If 'n'th bit is set, then trip point 'n' is writeable. + Bit string: If 'n'th bit is set, then trip point 'n' is writable. devdata: device private data ops: @@ -406,7 +406,7 @@ Thermal cooling device sys I/F, created once it's registered:: |---stats/reset: Writing any value resets the statistics |---stats/time_in_state_ms: Time (msec) spent in various cooling states |---stats/total_trans: Total number of times cooling state is changed - |---stats/trans_table: Cooing state transition table + |---stats/trans_table: Cooling state transition table Then next two dynamic attributes are created/removed in pairs. They represent @@ -520,19 +520,6 @@ available_policies RW, Optional -passive - Attribute is only present for zones in which the passive cooling - policy is not supported by native thermal driver. Default is zero - and can be set to a temperature (in millidegrees) to enable a - passive trip point for the zone. Activation is done by polling with - an interval of 1 second. - - Unit: millidegrees Celsius - - Valid values: 0 (disabled) or greater than 1000 - - RW, Optional - emul_temp Interface to set the emulated temperature method in thermal zone (sensor). After setting this temperature, the thermal zone may pass @@ -779,5 +766,5 @@ emergency poweroff kicks in after the delay has elapsed and shuts down the system. If set to 0 emergency poweroff will not be supported. So a carefully -profiled non-zero positive value is a must for emergerncy poweroff to be +profiled non-zero positive value is a must for emergency poweroff to be triggered. diff --git a/Documentation/features/core/cBPF-JIT/arch-support.txt b/Documentation/features/core/cBPF-JIT/arch-support.txt index 399935616813..e59b5215402d 100644 --- a/Documentation/features/core/cBPF-JIT/arch-support.txt +++ b/Documentation/features/core/cBPF-JIT/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | TODO | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/core/eBPF-JIT/arch-support.txt b/Documentation/features/core/eBPF-JIT/arch-support.txt index 79409bfe0263..dcbd8679f514 100644 --- a/Documentation/features/core/eBPF-JIT/arch-support.txt +++ b/Documentation/features/core/eBPF-JIT/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/core/generic-idle-thread/arch-support.txt b/Documentation/features/core/generic-idle-thread/arch-support.txt index 9ea60e416efd..4efcba7b5239 100644 --- a/Documentation/features/core/generic-idle-thread/arch-support.txt +++ b/Documentation/features/core/generic-idle-thread/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | ok | diff --git a/Documentation/features/core/jump-labels/arch-support.txt b/Documentation/features/core/jump-labels/arch-support.txt index 894d9693b380..0c801d1bd2da 100644 --- a/Documentation/features/core/jump-labels/arch-support.txt +++ b/Documentation/features/core/jump-labels/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/core/tracehook/arch-support.txt b/Documentation/features/core/tracehook/arch-support.txt index cd3510e2eedb..af34308fce7f 100644 --- a/Documentation/features/core/tracehook/arch-support.txt +++ b/Documentation/features/core/tracehook/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | ok | | csky: | ok | | h8300: | TODO | | hexagon: | ok | diff --git a/Documentation/features/debug/KASAN/arch-support.txt b/Documentation/features/debug/KASAN/arch-support.txt index b2288dc14b72..c244ac7eee26 100644 --- a/Documentation/features/debug/KASAN/arch-support.txt +++ b/Documentation/features/debug/KASAN/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/debug/debug-vm-pgtable/arch-support.txt b/Documentation/features/debug/debug-vm-pgtable/arch-support.txt index 1c49723e7534..7aff505af706 100644 --- a/Documentation/features/debug/debug-vm-pgtable/arch-support.txt +++ b/Documentation/features/debug/debug-vm-pgtable/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | TODO | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/debug/gcov-profile-all/arch-support.txt b/Documentation/features/debug/gcov-profile-all/arch-support.txt index 7563a494ddb8..b39c1a5de3f3 100644 --- a/Documentation/features/debug/gcov-profile-all/arch-support.txt +++ b/Documentation/features/debug/gcov-profile-all/arch-support.txt @@ -10,14 +10,13 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | | ia64: | TODO | | m68k: | TODO | | microblaze: | ok | - | mips: | TODO | + | mips: | ok | | nds32: | TODO | | nios2: | TODO | | openrisc: | TODO | diff --git a/Documentation/features/debug/kcov/arch-support.txt b/Documentation/features/debug/kcov/arch-support.txt index ab0ee1c933c2..7e44013cc320 100644 --- a/Documentation/features/debug/kcov/arch-support.txt +++ b/Documentation/features/debug/kcov/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/debug/kgdb/arch-support.txt b/Documentation/features/debug/kgdb/arch-support.txt index bc45bac20442..2cb0576f9180 100644 --- a/Documentation/features/debug/kgdb/arch-support.txt +++ b/Documentation/features/debug/kgdb/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | ok | | hexagon: | ok | diff --git a/Documentation/features/debug/kmemleak/arch-support.txt b/Documentation/features/debug/kmemleak/arch-support.txt index 2db76807ec6f..e9ac415f8aec 100644 --- a/Documentation/features/debug/kmemleak/arch-support.txt +++ b/Documentation/features/debug/kmemleak/arch-support.txt @@ -10,8 +10,7 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | - | csky: | TODO | + | csky: | ok | | h8300: | TODO | | hexagon: | TODO | | ia64: | TODO | diff --git a/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt b/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt index 6225cfe0c5bf..96156e8802a7 100644 --- a/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt +++ b/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | TODO | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | @@ -23,7 +22,7 @@ | openrisc: | TODO | | parisc: | ok | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | TODO | | sparc: | TODO | diff --git a/Documentation/features/debug/kprobes/arch-support.txt b/Documentation/features/debug/kprobes/arch-support.txt index 371f0ac488f5..ee95ed61909a 100644 --- a/Documentation/features/debug/kprobes/arch-support.txt +++ b/Documentation/features/debug/kprobes/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | @@ -23,7 +22,7 @@ | openrisc: | TODO | | parisc: | ok | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | ok | | sparc: | ok | diff --git a/Documentation/features/debug/kretprobes/arch-support.txt b/Documentation/features/debug/kretprobes/arch-support.txt index 38e95251deed..612cb97d47b8 100644 --- a/Documentation/features/debug/kretprobes/arch-support.txt +++ b/Documentation/features/debug/kretprobes/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | @@ -23,7 +22,7 @@ | openrisc: | TODO | | parisc: | ok | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | ok | | sparc: | ok | diff --git a/Documentation/features/debug/optprobes/arch-support.txt b/Documentation/features/debug/optprobes/arch-support.txt index 7f4a20e6a12b..d6ff141a6122 100644 --- a/Documentation/features/debug/optprobes/arch-support.txt +++ b/Documentation/features/debug/optprobes/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | TODO | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/debug/stackprotector/arch-support.txt b/Documentation/features/debug/stackprotector/arch-support.txt index 3329559c8207..ad4de22a71ab 100644 --- a/Documentation/features/debug/stackprotector/arch-support.txt +++ b/Documentation/features/debug/stackprotector/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/debug/uprobes/arch-support.txt b/Documentation/features/debug/uprobes/arch-support.txt index 43cac6ee0c68..8bd5548a4485 100644 --- a/Documentation/features/debug/uprobes/arch-support.txt +++ b/Documentation/features/debug/uprobes/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | @@ -23,7 +22,7 @@ | openrisc: | TODO | | parisc: | TODO | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | TODO | | sparc: | ok | diff --git a/Documentation/features/debug/user-ret-profiler/arch-support.txt b/Documentation/features/debug/user-ret-profiler/arch-support.txt index d636ed0e679f..2a3fe812a5fa 100644 --- a/Documentation/features/debug/user-ret-profiler/arch-support.txt +++ b/Documentation/features/debug/user-ret-profiler/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | TODO | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/io/dma-contiguous/arch-support.txt b/Documentation/features/io/dma-contiguous/arch-support.txt index dfc93d074e3d..bece89586efa 100644 --- a/Documentation/features/io/dma-contiguous/arch-support.txt +++ b/Documentation/features/io/dma-contiguous/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/locking/cmpxchg-local/arch-support.txt b/Documentation/features/locking/cmpxchg-local/arch-support.txt index 1815c7fed06d..52bdda004f5c 100644 --- a/Documentation/features/locking/cmpxchg-local/arch-support.txt +++ b/Documentation/features/locking/cmpxchg-local/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/locking/lockdep/arch-support.txt b/Documentation/features/locking/lockdep/arch-support.txt index 940b0bd02957..a8cd163c8b7e 100644 --- a/Documentation/features/locking/lockdep/arch-support.txt +++ b/Documentation/features/locking/lockdep/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | ok | diff --git a/Documentation/features/locking/queued-rwlocks/arch-support.txt b/Documentation/features/locking/queued-rwlocks/arch-support.txt index 4dd5e554873f..8c85949752b3 100644 --- a/Documentation/features/locking/queued-rwlocks/arch-support.txt +++ b/Documentation/features/locking/queued-rwlocks/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/locking/queued-spinlocks/arch-support.txt b/Documentation/features/locking/queued-spinlocks/arch-support.txt index b16d4f71e5ce..5f4e1b3841af 100644 --- a/Documentation/features/locking/queued-spinlocks/arch-support.txt +++ b/Documentation/features/locking/queued-spinlocks/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/perf/kprobes-event/arch-support.txt b/Documentation/features/perf/kprobes-event/arch-support.txt index 04c17c2106a4..78f3fe080f0e 100644 --- a/Documentation/features/perf/kprobes-event/arch-support.txt +++ b/Documentation/features/perf/kprobes-event/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | ok | @@ -23,7 +22,7 @@ | openrisc: | TODO | | parisc: | ok | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | ok | | sparc: | ok | diff --git a/Documentation/features/perf/perf-regs/arch-support.txt b/Documentation/features/perf/perf-regs/arch-support.txt index e7450fbb8253..5bf3b1854a1f 100644 --- a/Documentation/features/perf/perf-regs/arch-support.txt +++ b/Documentation/features/perf/perf-regs/arch-support.txt @@ -10,14 +10,13 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | | ia64: | TODO | | m68k: | TODO | | microblaze: | TODO | - | mips: | TODO | + | mips: | ok | | nds32: | TODO | | nios2: | TODO | | openrisc: | TODO | diff --git a/Documentation/features/perf/perf-stackdump/arch-support.txt b/Documentation/features/perf/perf-stackdump/arch-support.txt index 98e79d128d9b..d88659bb4fc1 100644 --- a/Documentation/features/perf/perf-stackdump/arch-support.txt +++ b/Documentation/features/perf/perf-stackdump/arch-support.txt @@ -10,14 +10,13 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | | ia64: | TODO | | m68k: | TODO | | microblaze: | TODO | - | mips: | TODO | + | mips: | ok | | nds32: | TODO | | nios2: | TODO | | openrisc: | TODO | diff --git a/Documentation/features/sched/membarrier-sync-core/arch-support.txt b/Documentation/features/sched/membarrier-sync-core/arch-support.txt index 47e6903f47a5..883d33b265d6 100644 --- a/Documentation/features/sched/membarrier-sync-core/arch-support.txt +++ b/Documentation/features/sched/membarrier-sync-core/arch-support.txt @@ -33,7 +33,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/sched/numa-balancing/arch-support.txt b/Documentation/features/sched/numa-balancing/arch-support.txt index 964457ad26c1..9affb7c2c500 100644 --- a/Documentation/features/sched/numa-balancing/arch-support.txt +++ b/Documentation/features/sched/numa-balancing/arch-support.txt @@ -10,7 +10,6 @@ | arc: | .. | | arm: | .. | | arm64: | ok | - | c6x: | .. | | csky: | .. | | h8300: | .. | | hexagon: | .. | @@ -23,7 +22,7 @@ | openrisc: | .. | | parisc: | .. | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | .. | | sparc: | TODO | diff --git a/Documentation/features/seccomp/seccomp-filter/arch-support.txt b/Documentation/features/seccomp/seccomp-filter/arch-support.txt index eb3d74092c61..26eec58ab819 100644 --- a/Documentation/features/seccomp/seccomp-filter/arch-support.txt +++ b/Documentation/features/seccomp/seccomp-filter/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/time/arch-tick-broadcast/arch-support.txt b/Documentation/features/time/arch-tick-broadcast/arch-support.txt index 4d11cbb3c09b..8639fe8315f5 100644 --- a/Documentation/features/time/arch-tick-broadcast/arch-support.txt +++ b/Documentation/features/time/arch-tick-broadcast/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/time/clockevents/arch-support.txt b/Documentation/features/time/clockevents/arch-support.txt index 6863a3fbddad..9a81cb03b1fd 100644 --- a/Documentation/features/time/clockevents/arch-support.txt +++ b/Documentation/features/time/clockevents/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | TODO | | arm64: | ok | - | c6x: | ok | | csky: | ok | | h8300: | ok | | hexagon: | ok | diff --git a/Documentation/features/time/context-tracking/arch-support.txt b/Documentation/features/time/context-tracking/arch-support.txt index 52aea275aab7..4ed116c2ec39 100644 --- a/Documentation/features/time/context-tracking/arch-support.txt +++ b/Documentation/features/time/context-tracking/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/time/irq-time-acct/arch-support.txt b/Documentation/features/time/irq-time-acct/arch-support.txt index 6fc03deb1c38..bc30c15557c7 100644 --- a/Documentation/features/time/irq-time-acct/arch-support.txt +++ b/Documentation/features/time/irq-time-acct/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/time/virt-cpuacct/arch-support.txt b/Documentation/features/time/virt-cpuacct/arch-support.txt index e51f3af38e31..050de43bbbb9 100644 --- a/Documentation/features/time/virt-cpuacct/arch-support.txt +++ b/Documentation/features/time/virt-cpuacct/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/vm/ELF-ASLR/arch-support.txt b/Documentation/features/vm/ELF-ASLR/arch-support.txt index eccda0732474..99cb6d7f5005 100644 --- a/Documentation/features/vm/ELF-ASLR/arch-support.txt +++ b/Documentation/features/vm/ELF-ASLR/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/vm/PG_uncached/arch-support.txt b/Documentation/features/vm/PG_uncached/arch-support.txt index c74e3f8040e1..6cde38458596 100644 --- a/Documentation/features/vm/PG_uncached/arch-support.txt +++ b/Documentation/features/vm/PG_uncached/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | TODO | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/vm/THP/arch-support.txt b/Documentation/features/vm/THP/arch-support.txt index 1c0b95f2b40d..e8238cb2a4da 100644 --- a/Documentation/features/vm/THP/arch-support.txt +++ b/Documentation/features/vm/THP/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | .. | | csky: | .. | | h8300: | .. | | hexagon: | .. | diff --git a/Documentation/features/vm/TLB/arch-support.txt b/Documentation/features/vm/TLB/arch-support.txt index 30f75a79ce01..48a5ca548399 100644 --- a/Documentation/features/vm/TLB/arch-support.txt +++ b/Documentation/features/vm/TLB/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | TODO | - | c6x: | .. | | csky: | TODO | | h8300: | .. | | hexagon: | TODO | diff --git a/Documentation/features/vm/huge-vmap/arch-support.txt b/Documentation/features/vm/huge-vmap/arch-support.txt index c5ff3a427722..439fd9069b8b 100644 --- a/Documentation/features/vm/huge-vmap/arch-support.txt +++ b/Documentation/features/vm/huge-vmap/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/vm/ioremap_prot/arch-support.txt b/Documentation/features/vm/ioremap_prot/arch-support.txt index b5fb37c28cc6..9a0c8783b84d 100644 --- a/Documentation/features/vm/ioremap_prot/arch-support.txt +++ b/Documentation/features/vm/ioremap_prot/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | TODO | | arm64: | TODO | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/vm/pte_special/arch-support.txt b/Documentation/features/vm/pte_special/arch-support.txt index 13d0e1e17001..40b969f3a6bb 100644 --- a/Documentation/features/vm/pte_special/arch-support.txt +++ b/Documentation/features/vm/pte_special/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/filesystems/afs.rst b/Documentation/filesystems/afs.rst index 0abb155ac666..ca062a7f8ee2 100644 --- a/Documentation/filesystems/afs.rst +++ b/Documentation/filesystems/afs.rst @@ -109,7 +109,7 @@ Mountpoints AFS has a concept of mountpoints. In AFS terms, these are specially formatted symbolic links (of the same form as the "device name" passed to mount). kAFS presents these to the user as directories that have a follow-link capability -(ie: symbolic link semantics). If anyone attempts to access them, they will +(i.e.: symbolic link semantics). If anyone attempts to access them, they will automatically cause the target volume to be mounted (if possible) on that site. Automatically mounted filesystems will be automatically unmounted approximately @@ -144,7 +144,7 @@ looks up a cell of the same name, for example:: Proc Filesystem =============== -The AFS modules creates a "/proc/fs/afs/" directory and populates it: +The AFS module creates a "/proc/fs/afs/" directory and populates it: (*) A "cells" file that lists cells currently known to the afs module and their usage counts:: @@ -201,7 +201,7 @@ And then run as:: ./klog Assuming it's successful, this adds a key of type RxRPC, named for the service -and cell, eg: "afs@". This can be viewed with the keyctl program or +and cell, e.g.: "afs@". This can be viewed with the keyctl program or by cat'ing /proc/keys:: [root@andromeda ~]# keyctl show @@ -211,7 +211,7 @@ by cat'ing /proc/keys:: 111416553 --als--v 0 0 \_ rxrpc: afs@CAMBRIDGE.REDHAT.COM Currently the username, realm, password and proposed ticket lifetime are -compiled in to the program. +compiled into the program. It is not required to acquire a key before using AFS facilities, but if one is not acquired then all operations will be governed by the anonymous user parts diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt index 8fdb78f3c6c9..e03c20564f3a 100644 --- a/Documentation/filesystems/dax.txt +++ b/Documentation/filesystems/dax.txt @@ -83,20 +83,9 @@ Summary directories. This has runtime constraints and limitations that are described in 6) below. - 6. When changing the S_DAX policy via toggling the persistent FS_XFLAG_DAX flag, - the change in behaviour for existing regular files may not occur - immediately. If the change must take effect immediately, the administrator - needs to: - - a) stop the application so there are no active references to the data set - the policy change will affect - - b) evict the data set from kernel caches so it will be re-instantiated when - the application is restarted. This can be achieved by: - - i. drop-caches - ii. a filesystem unmount and mount cycle - iii. a system reboot + 6. When changing the S_DAX policy via toggling the persistent FS_XFLAG_DAX + flag, the change to existing regular files won't take effect until the + files are closed by all processes. Details diff --git a/Documentation/filesystems/index.rst b/Documentation/filesystems/index.rst index 7be9b46d85d9..1f76b1cb3348 100644 --- a/Documentation/filesystems/index.rst +++ b/Documentation/filesystems/index.rst @@ -83,6 +83,7 @@ Documentation for filesystem implementations. erofs ext2 ext3 + ext4/index f2fs gfs2 gfs2-uevents diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index c0f2c7586531..b7dcc86c92a4 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -126,9 +126,10 @@ prototypes:: int (*get)(const struct xattr_handler *handler, struct dentry *dentry, struct inode *inode, const char *name, void *buffer, size_t size); - int (*set)(const struct xattr_handler *handler, struct dentry *dentry, - struct inode *inode, const char *name, const void *buffer, - size_t size, int flags); + int (*set)(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, + struct dentry *dentry, struct inode *inode, const char *name, + const void *buffer, size_t size, int flags); locking rules: all may block diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst index 1f8cf8e10b34..0302035781be 100644 --- a/Documentation/filesystems/porting.rst +++ b/Documentation/filesystems/porting.rst @@ -717,6 +717,8 @@ be removed. Switch while you still can; the old one won't stay. **mandatory** ->setxattr() and xattr_handler.set() get dentry and inode passed separately. +The xattr_handler.set() gets passed the user namespace of the mount the inode +is seen from so filesystems can idmap the i_uid and i_gid accordingly. dentry might be yet to be attached to inode, so do _not_ use its ->d_inode in the instances. Rationale: !@#!@# security_d_instantiate() needs to be called before we attach dentry to inode and !@#!@##!@$!$#!@#$!@$!@$ smack @@ -881,3 +883,10 @@ For bvec based itererators bio_iov_iter_get_pages() now doesn't copy bvecs but uses the one provided. Anyone issuing kiocb-I/O should ensure that the bvec and page references stay until I/O has completed, i.e. until ->ki_complete() has been called or returned with non -EIOCBQUEUED code. + +--- + +**mandatory** + +mnt_want_write_file() can now only be paired with mnt_drop_write_file(), +whereas previously it could be paired with mnt_drop_write() as well. diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index 2fa69f710e2a..48fbfc336ebf 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -687,7 +687,14 @@ files are there, and which are missing. kcore Kernel core image (can be ELF or A.OUT(deprecated in 2.4)) kmsg Kernel messages ksyms Kernel symbol table - loadavg Load average of last 1, 5 & 15 minutes + loadavg Load average of last 1, 5 & 15 minutes; + number of processes currently runnable (running or on ready queue); + total number of processes in system; + last pid created. + All fields are separated by one space except "number of + processes currently runnable" and "total number of processes + in system", which are separated by a slash ('/'). Example: + 0.61 0.61 0.55 3/828 22084 locks Kernel locks meminfo Memory info misc Miscellaneous diff --git a/Documentation/filesystems/seq_file.rst b/Documentation/filesystems/seq_file.rst index 56856481dc8d..a6726082a7c2 100644 --- a/Documentation/filesystems/seq_file.rst +++ b/Documentation/filesystems/seq_file.rst @@ -217,6 +217,12 @@ between the calls to start() and stop(), so holding a lock during that time is a reasonable thing to do. The seq_file code will also avoid taking any other locks while the iterator is active. +The iterater value returned by start() or next() is guaranteed to be +passed to a subsequent next() or stop() call. This allows resources +such as locks that were taken to be reliably released. There is *no* +guarantee that the iterator will be passed to show(), though in practice +it often will be. + Formatted output ================ diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index ca52c82e5bb5..2049bbf5e388 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -112,7 +112,7 @@ members are defined: .. code-block:: c - struct file_system_operations { + struct file_system_type { const char *name; int fs_flags; struct dentry *(*mount) (struct file_system_type *, int, @@ -270,7 +270,10 @@ or bottom half). ->alloc_inode. ``dirty_inode`` - this method is called by the VFS to mark an inode dirty. + this method is called by the VFS when an inode is marked dirty. + This is specifically for the inode itself being marked dirty, + not its data. If the update needs to be persisted by fdatasync(), + then I_DIRTY_DATASYNC will be set in the flags argument. ``write_inode`` this method is called when the VFS needs to write an inode to @@ -415,28 +418,29 @@ As of kernel 2.6.22, the following members are defined: .. code-block:: c struct inode_operations { - int (*create) (struct inode *,struct dentry *, umode_t, bool); + int (*create) (struct user_namespace *, struct inode *,struct dentry *, umode_t, bool); struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int); int (*link) (struct dentry *,struct inode *,struct dentry *); int (*unlink) (struct inode *,struct dentry *); - int (*symlink) (struct inode *,struct dentry *,const char *); - int (*mkdir) (struct inode *,struct dentry *,umode_t); + int (*symlink) (struct user_namespace *, struct inode *,struct dentry *,const char *); + int (*mkdir) (struct user_namespace *, struct inode *,struct dentry *,umode_t); int (*rmdir) (struct inode *,struct dentry *); - int (*mknod) (struct inode *,struct dentry *,umode_t,dev_t); - int (*rename) (struct inode *, struct dentry *, + int (*mknod) (struct user_namespace *, struct inode *,struct dentry *,umode_t,dev_t); + int (*rename) (struct user_namespace *, struct inode *, struct dentry *, struct inode *, struct dentry *, unsigned int); int (*readlink) (struct dentry *, char __user *,int); const char *(*get_link) (struct dentry *, struct inode *, struct delayed_call *); - int (*permission) (struct inode *, int); + int (*permission) (struct user_namespace *, struct inode *, int); int (*get_acl)(struct inode *, int); - int (*setattr) (struct dentry *, struct iattr *); - int (*getattr) (const struct path *, struct kstat *, u32, unsigned int); + int (*setattr) (struct user_namespace *, struct dentry *, struct iattr *); + int (*getattr) (struct user_namespace *, const struct path *, struct kstat *, u32, unsigned int); ssize_t (*listxattr) (struct dentry *, char *, size_t); void (*update_time)(struct inode *, struct timespec *, int); int (*atomic_open)(struct inode *, struct dentry *, struct file *, unsigned open_flag, umode_t create_mode); - int (*tmpfile) (struct inode *, struct dentry *, umode_t); + int (*tmpfile) (struct user_namespace *, struct inode *, struct dentry *, umode_t); + int (*set_acl)(struct user_namespace *, struct inode *, struct posix_acl *, int); }; Again, all methods are called without any locks being held, unless diff --git a/Documentation/fpga/dfl.rst b/Documentation/fpga/dfl.rst index 0404fe6ffc74..c41ac76ffaae 100644 --- a/Documentation/fpga/dfl.rst +++ b/Documentation/fpga/dfl.rst @@ -501,6 +501,34 @@ Developer only needs to provide a sub feature driver with matched feature id. FME Partial Reconfiguration Sub Feature driver (see drivers/fpga/dfl-fme-pr.c) could be a reference. +Location of DFLs on a PCI Device +================================ +The original method for finding a DFL on a PCI device assumed the start of the +first DFL to offset 0 of bar 0. If the first node of the DFL is an FME, +then further DFLs in the port(s) are specified in FME header registers. +Alternatively, a PCIe vendor specific capability structure can be used to +specify the location of all the DFLs on the device, providing flexibility +for the type of starting node in the DFL. Intel has reserved the +VSEC ID of 0x43 for this purpose. The vendor specific +data begins with a 4 byte vendor specific register for the number of DFLs followed 4 byte +Offset/BIR vendor specific registers for each DFL. Bits 2:0 of Offset/BIR register +indicates the BAR, and bits 31:3 form the 8 byte aligned offset where bits 2:0 are +zero. +:: + + +----------------------------+ + |31 Number of DFLS 0| + +----------------------------+ + |31 Offset 3|2 BIR 0| + +----------------------------+ + . . . + +----------------------------+ + |31 Offset 3|2 BIR 0| + +----------------------------+ + +Being able to specify more than one DFL per BAR has been considered, but it +was determined the use case did not provide value. Specifying a single DFL +per BAR simplifies the implementation and allows for extra error checking. Open discussion =============== diff --git a/Documentation/gpu/drm-kms.rst b/Documentation/gpu/drm-kms.rst index 3c5ae4f6dfd2..87e5023e3f55 100644 --- a/Documentation/gpu/drm-kms.rst +++ b/Documentation/gpu/drm-kms.rst @@ -319,6 +319,15 @@ CRTC Functions Reference .. kernel-doc:: drivers/gpu/drm/drm_crtc.c :export: +Color Management Functions Reference +------------------------------------ + +.. kernel-doc:: drivers/gpu/drm/drm_color_mgmt.c + :export: + +.. kernel-doc:: include/drm/drm_color_mgmt.h + :internal: + Frame Buffer Abstraction ======================== @@ -370,6 +379,21 @@ Plane Functions Reference .. kernel-doc:: drivers/gpu/drm/drm_plane.c :export: +Plane Composition Functions Reference +------------------------------------- + +.. kernel-doc:: drivers/gpu/drm/drm_blend.c + :export: + +Plane Damage Tracking Functions Reference +----------------------------------------- + +.. kernel-doc:: drivers/gpu/drm/drm_damage_helper.c + :export: + +.. kernel-doc:: include/drm/drm_damage_helper.h + :internal: + Display Modes Function Reference ================================ @@ -436,6 +460,9 @@ KMS Locking KMS Properties ============== +This section of the documentation is primarily aimed at user-space developers. +For the driver APIs, see the other sections. + Property Types and Blob Property Support ---------------------------------------- @@ -466,39 +493,30 @@ Standard CRTC Properties .. kernel-doc:: drivers/gpu/drm/drm_crtc.c :doc: standard CRTC properties +Standard Plane Properties +------------------------- + +.. kernel-doc:: drivers/gpu/drm/drm_plane.c + :doc: standard plane properties + Plane Composition Properties ---------------------------- .. kernel-doc:: drivers/gpu/drm/drm_blend.c :doc: overview -.. kernel-doc:: drivers/gpu/drm/drm_blend.c - :export: - -FB_DAMAGE_CLIPS -~~~~~~~~~~~~~~~ +Damage Tracking Properties +-------------------------- .. kernel-doc:: drivers/gpu/drm/drm_damage_helper.c :doc: overview -.. kernel-doc:: drivers/gpu/drm/drm_damage_helper.c - :export: - -.. kernel-doc:: include/drm/drm_damage_helper.h - :internal: - Color Management Properties --------------------------- .. kernel-doc:: drivers/gpu/drm/drm_color_mgmt.c :doc: overview -.. kernel-doc:: drivers/gpu/drm/drm_color_mgmt.c - :export: - -.. kernel-doc:: include/drm/drm_color_mgmt.h - :internal: - Tile Group Property ------------------- diff --git a/Documentation/gpu/drm-uapi.rst b/Documentation/gpu/drm-uapi.rst index 7dce175f6d75..04bdc7a91d53 100644 --- a/Documentation/gpu/drm-uapi.rst +++ b/Documentation/gpu/drm-uapi.rst @@ -457,5 +457,8 @@ Userspace API Structures .. kernel-doc:: include/uapi/drm/drm_mode.h :doc: overview +.. kernel-doc:: include/uapi/drm/drm.h + :internal: + .. kernel-doc:: include/uapi/drm/drm_mode.h :internal: diff --git a/Documentation/gpu/i915.rst b/Documentation/gpu/i915.rst index 20868f3d0123..486c720f3890 100644 --- a/Documentation/gpu/i915.rst +++ b/Documentation/gpu/i915.rst @@ -428,7 +428,7 @@ User Batchbuffer Execution Logical Rings, Logical Ring Contexts and Execlists -------------------------------------------------- -.. kernel-doc:: drivers/gpu/drm/i915/gt/intel_lrc.c +.. kernel-doc:: drivers/gpu/drm/i915/gt/intel_execlists_submission.c :doc: Logical Rings, Logical Ring Contexts and Execlists Global GTT views diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst index 009d8e6c7e3c..40ccac61137e 100644 --- a/Documentation/gpu/todo.rst +++ b/Documentation/gpu/todo.rst @@ -23,6 +23,9 @@ Advanced: Tricky tasks that need fairly good understanding of the DRM subsystem and graphics topics. Generally need the relevant hardware for development and testing. +Expert: Only attempt these if you've successfully completed some tricky +refactorings already and are an expert in the specific area + Subsystem-wide refactorings =========================== @@ -168,6 +171,22 @@ Contact: Daniel Vetter, respective driver maintainers Level: Advanced +Move Buffer Object Locking to dma_resv_lock() +--------------------------------------------- + +Many drivers have their own per-object locking scheme, usually using +mutex_lock(). This causes all kinds of trouble for buffer sharing, since +depending which driver is the exporter and importer, the locking hierarchy is +reversed. + +To solve this we need one standard per-object locking mechanism, which is +dma_resv_lock(). This lock needs to be called as the outermost lock, with all +other driver specific per-object locks removed. The problem is tha rolling out +the actual change to the locking contract is a flag day, due to struct dma_buf +buffer sharing. + +Level: Expert + Convert logging to drm_* functions with drm_device paramater ------------------------------------------------------------ @@ -669,7 +688,7 @@ for fbdev. https://patchwork.freedesktop.org/patch/306579/ - [RFC PATCH v2 00/13] Kernel based bootsplash - https://lkml.org/lkml/2017/12/13/764 + https://lore.kernel.org/r/20171213194755.3409-1-mstaudt@suse.de Contact: Sam Ravnborg diff --git a/Documentation/gpu/vkms.rst b/Documentation/gpu/vkms.rst index 13bab1d93bb3..2c9b376da5ca 100644 --- a/Documentation/gpu/vkms.rst +++ b/Documentation/gpu/vkms.rst @@ -7,6 +7,88 @@ .. kernel-doc:: drivers/gpu/drm/vkms/vkms_drv.c :doc: vkms (Virtual Kernel Modesetting) +Setup +===== + +The VKMS driver can be setup with the following steps: + +To check if VKMS is loaded, run:: + + lsmod | grep vkms + +This should list the VKMS driver. If no output is obtained, then +you need to enable and/or load the VKMS driver. +Ensure that the VKMS driver has been set as a loadable module in your +kernel config file. Do:: + + make nconfig + + Go to `Device Drivers> Graphics support` + + Enable `Virtual KMS (EXPERIMENTAL)` + +Compile and build the kernel for the changes to get reflected. +Now, to load the driver, use:: + + sudo modprobe vkms + +On running the lsmod command now, the VKMS driver will appear listed. +You can also observe the driver being loaded in the dmesg logs. + +The VKMS driver has optional features to simulate different kinds of hardware, +which are exposed as module options. You can use the `modinfo` command +to see the module options for vkms:: + + modinfo vkms + +Module options are helpful when testing, and enabling modules +can be done while loading vkms. For example, to load vkms with cursor enabled, +use:: + + sudo modprobe vkms enable_cursor=1 + +To disable the driver, use :: + + sudo modprobe -r vkms + +Testing With IGT +================ + +The IGT GPU Tools is a test suite used specifically for debugging and +development of the DRM drivers. +The IGT Tools can be installed from +`here `_ . + +The tests need to be run without a compositor, so you need to switch to text +only mode. You can do this by:: + + sudo systemctl isolate multi-user.target + +To return to graphical mode, do:: + + sudo systemctl isolate graphical.target + +Once you are in text only mode, you can run tests using the --device switch +or IGT_DEVICE variable to specify the device filter for the driver we want +to test. IGT_DEVICE can also be used with the run-test.sh script to run the +tests for a specific driver:: + + sudo ./build/tests/ --device "sys:/sys/devices/platform/vkms" + sudo IGT_DEVICE="sys:/sys/devices/platform/vkms" ./build/tests/ + sudo IGT_DEVICE="sys:/sys/devices/platform/vkms" ./scripts/run-tests.sh -t + +For example, to test the functionality of the writeback library, +we can run the kms_writeback test:: + + sudo ./build/tests/kms_writeback --device "sys:/sys/devices/platform/vkms" + sudo IGT_DEVICE="sys:/sys/devices/platform/vkms" ./build/tests/kms_writeback + sudo IGT_DEVICE="sys:/sys/devices/platform/vkms" ./scripts/run-tests.sh -t kms_writeback + +You can also run subtests if you do not want to run the entire test:: + + sudo ./build/tests/kms_flip --run-subtest basic-plain-flip --device "sys:/sys/devices/platform/vkms" + sudo IGT_DEVICE="sys:/sys/devices/platform/vkms" ./build/tests/kms_flip --run-subtest basic-plain-flip + TODO ==== diff --git a/Documentation/hid/amd-sfh-hid.rst b/Documentation/hid/amd-sfh-hid.rst index 1f2fe29ccd4f..19ae94cde3b4 100644 --- a/Documentation/hid/amd-sfh-hid.rst +++ b/Documentation/hid/amd-sfh-hid.rst @@ -3,13 +3,13 @@ AMD Sensor Fusion Hub ===================== -AMD Sensor Fusion Hub (SFH) is part of an SOC starting from Ryzen based platforms. +AMD Sensor Fusion Hub (SFH) is part of an SOC starting from Ryzen-based platforms. The solution is working well on several OEM products. AMD SFH uses HID over PCIe bus. In terms of architecture it resembles ISH, however the major difference is all the HID reports are generated as part of the kernel driver. -1. Block Diagram -================ +Block Diagram +------------- :: @@ -45,20 +45,20 @@ the HID reports are generated as part of the kernel driver. AMD HID Transport Layer ----------------------- AMD SFH transport is also implemented as a bus. Each client application executing in the AMD MP2 is -registered as a device on this bus. Here: MP2 which is an ARM core connected to x86 for processing +registered as a device on this bus. Here, MP2 is an ARM core connected to x86 for processing sensor data. The layer, which binds each device (AMD SFH HID driver) identifies the device type and -registers with the hid core. Transport layer attach a constant "struct hid_ll_driver" object with +registers with the HID core. Transport layer attaches a constant "struct hid_ll_driver" object with each device. Once a device is registered with HID core, the callbacks provided via this struct are used by HID core to communicate with the device. AMD HID Transport layer implements the synchronous calls. AMD HID Client Layer -------------------- -This layer is responsible to implement HID request and descriptors. As firmware is OS agnostic, HID +This layer is responsible to implement HID requests and descriptors. As firmware is OS agnostic, HID client layer fills the HID request structure and descriptors. HID client layer is complex as it is -interface between MP2 PCIe layer and HID. HID client layer initialized the MP2 PCIe layer and holds -the instance of MP2 layer. It identifies the number of sensors connected using MP2-PCIe layer. Base -on that allocates the DRAM address for each and every sensor and pass it to MP2-PCIe driver.On -enumeration of each the sensor, client layer fills the HID Descriptor structure and HID input repor +interface between MP2 PCIe layer and HID. HID client layer initializes the MP2 PCIe layer and holds +the instance of MP2 layer. It identifies the number of sensors connected using MP2-PCIe layer. Based +on that allocates the DRAM address for each and every sensor and passes it to MP2-PCIe driver. On +enumeration of each sensor, client layer fills the HID Descriptor structure and HID input report structure. HID Feature report structure is optional. The report descriptor structure varies from sensor to sensor. @@ -72,7 +72,7 @@ The communication between X86 and MP2 is split into three parts. 2. Data transfer via DRAM. 3. Supported sensor info via P2C registers. -Commands are sent to MP2 using C2P Mailbox registers. Writing into C2P Message registers generate +Commands are sent to MP2 using C2P Mailbox registers. Writing into C2P Message registers generates interrupt to MP2. The client layer allocates the physical memory and the same is sent to MP2 via the PCI layer. MP2 firmware writes the command output to the access DRAM memory which the client layer has allocated. Firmware always writes minimum of 32 bytes into DRAM. So as a protocol driver diff --git a/Documentation/hid/hid-alps.rst b/Documentation/hid/hid-alps.rst index e2f4c4c11e3f..767c96bcbb7c 100644 --- a/Documentation/hid/hid-alps.rst +++ b/Documentation/hid/hid-alps.rst @@ -64,7 +64,7 @@ Case2 ReportID_3 TP Absolute Command Read/Write ------------------ -To read/write to RAM, need to send a commands to the device. +To read/write to RAM, need to send a command to the device. The command format is as below. @@ -80,7 +80,7 @@ Byte6 Value Byte Byte7 Checksum ===== ====================== -Command Byte is read=0xD1/write=0xD2 . +Command Byte is read=0xD1/write=0xD2. Address is read/write RAM address. diff --git a/Documentation/hid/hid-sensor.rst b/Documentation/hid/hid-sensor.rst index 758972e34971..c1c9b8d8dca6 100644 --- a/Documentation/hid/hid-sensor.rst +++ b/Documentation/hid/hid-sensor.rst @@ -48,12 +48,12 @@ for different sensors. For example an accelerometer can send X,Y and Z data, whe an ambient light sensor can send illumination data. So the implementation has two parts: -- Core hid driver +- Core HID driver - Individual sensor processing part (sensor drivers) Core driver ----------- -The core driver registers (hid-sensor-hub) registers as a HID driver. It parses +The core driver (hid-sensor-hub) registers as a HID driver. It parses report descriptors and identifies all the sensors present. It adds an MFD device with name HID-SENSOR-xxxx (where xxxx is usage id from the specification). @@ -95,14 +95,14 @@ Registration functions:: u32 usage_id, struct hid_sensor_hub_callbacks *usage_callback): -Registers callbacks for an usage id. The callback functions are not allowed +Registers callbacks for a usage id. The callback functions are not allowed to sleep:: int sensor_hub_remove_callback(struct hid_sensor_hub_device *hsdev, u32 usage_id): -Removes callbacks for an usage id. +Removes callbacks for a usage id. Parsing function:: @@ -166,7 +166,7 @@ This allows some differentiating use cases, where vendor can provide application Some common use cases are debug other sensors or to provide some events like keyboard attached/detached or lid open/close. -To allow application to utilize these sensors, here they are exported uses sysfs +To allow application to utilize these sensors, here they are exported using sysfs attribute groups, attributes and misc device interface. An example of this representation on sysfs:: @@ -207,9 +207,9 @@ An example of this representation on sysfs:: │   │   │   ├── input-1-200202-units │   │   │   ├── input-1-200202-value -Here there is a custom sensors with four fields, two feature and two inputs. +Here there is a custom sensor with four fields: two feature and two inputs. Each field is represented by a set of attributes. All fields except the "value" -are read only. The value field is a RW field. +are read only. The value field is a read-write field. Example:: @@ -237,6 +237,6 @@ These reports are pushed using misc device interface in a FIFO order:: │   │   │   ├── 10:53 -> ../HID-SENSOR-2000e1.6.auto │   ├── HID-SENSOR-2000e1.6.auto -Each reports can be of variable length preceded by a header. This header -consist of a 32 bit usage id, 64 bit time stamp and 32 bit length field of raw +Each report can be of variable length preceded by a header. This header +consists of a 32-bit usage id, 64-bit time stamp and 32-bit length field of raw data. diff --git a/Documentation/hid/hid-transport.rst b/Documentation/hid/hid-transport.rst index 0fe526f36db6..6f1692da296c 100644 --- a/Documentation/hid/hid-transport.rst +++ b/Documentation/hid/hid-transport.rst @@ -12,8 +12,8 @@ Bluetooth, I2C and user-space I/O drivers. The HID subsystem is designed as a bus. Any I/O subsystem may provide HID devices and register them with the HID bus. HID core then loads generic device -drivers on top of it. The transport drivers are responsible of raw data -transport and device setup/management. HID core is responsible of +drivers on top of it. The transport drivers are responsible for raw data +transport and device setup/management. HID core is responsible for report-parsing, report interpretation and the user-space API. Device specifics and quirks are handled by all layers depending on the quirk. @@ -67,7 +67,7 @@ Transport drivers attach a constant "struct hid_ll_driver" object with each device. Once a device is registered with HID core, the callbacks provided via this struct are used by HID core to communicate with the device. -Transport drivers are responsible of detecting device failures and unplugging. +Transport drivers are responsible for detecting device failures and unplugging. HID core will operate a device as long as it is registered regardless of any device failures. Once transport drivers detect unplug or failure events, they must unregister the device from HID core and HID core will stop using the @@ -101,7 +101,7 @@ properties in common. channel. Any unrequested incoming or outgoing data report must be sent on this channel and is never acknowledged by the remote side. Devices usually send their input events on this channel. Outgoing events are normally - not send via intr, except if high throughput is required. + not sent via intr, except if high throughput is required. - Control Channel (ctrl): The ctrl channel is used for synchronous requests and device management. Unrequested data input events must not be sent on this channel and are normally ignored. Instead, devices only send management @@ -161,7 +161,7 @@ allowed on the intr channel and are the only means of data there. payload may be blocked by the underlying transport driver if the specification does not allow them. - SET_REPORT: A SET_REPORT request has a report ID plus data as payload. It is - sent from host to device and a device must update it's current report state + sent from host to device and a device must update its current report state according to the given data. Any of the 3 report types can be used. However, INPUT reports as payload might be blocked by the underlying transport driver if the specification does not allow them. @@ -294,7 +294,7 @@ The available HID callbacks are: void (*request) (struct hid_device *hdev, struct hid_report *report, int reqtype) - Send an HID request on the ctrl channel. "report" contains the report that + Send a HID request on the ctrl channel. "report" contains the report that should be sent and "reqtype" the request type. Request-type can be HID_REQ_SET_REPORT or HID_REQ_GET_REPORT. diff --git a/Documentation/hid/hiddev.rst b/Documentation/hid/hiddev.rst index 9b28a97c03e6..caebc6266603 100644 --- a/Documentation/hid/hiddev.rst +++ b/Documentation/hid/hiddev.rst @@ -27,7 +27,7 @@ the following:: --> hiddev.c ----> POWER / MONITOR CONTROL In addition, other subsystems (apart from USB) can potentially feed -events into the input subsystem, but these have no effect on the hid +events into the input subsystem, but these have no effect on the HID device interface. Using the HID Device Interface @@ -73,7 +73,7 @@ The hiddev API uses a read() interface, and a set of ioctl() calls. HID devices exchange data with the host computer using data bundles called "reports". Each report is divided into "fields", each of which can have one or more "usages". In the hid-core, -each one of these usages has a single signed 32 bit value. +each one of these usages has a single signed 32-bit value. read(): ------- @@ -113,7 +113,7 @@ HIDIOCAPPLICATION - (none) This ioctl call returns the HID application usage associated with the -hid device. The third argument to ioctl() specifies which application +HID device. The third argument to ioctl() specifies which application index to get. This is useful when the device has more than one application collection. If the index is invalid (greater or equal to the number of application collections this device has) the ioctl @@ -181,7 +181,7 @@ looked up by type (input, output or feature) and id, so these fields must be filled in by the user. The ID can be absolute -- the actual report id as reported by the device -- or relative -- HID_REPORT_ID_FIRST for the first report, and (HID_REPORT_ID_NEXT | -report_id) for the next report after report_id. Without a-priori +report_id) for the next report after report_id. Without a priori information about report ids, the right way to use this ioctl is to use the relative IDs above to enumerate the valid IDs. The ioctl returns non-zero when there is no more next ID. The real report ID is @@ -200,7 +200,7 @@ HIDIOCGUCODE - struct hiddev_usage_ref (read/write) Returns the usage_code in a hiddev_usage_ref structure, given that -given its report type, report id, field index, and index within the +its report type, report id, field index, and index within the field have already been filled into the structure. HIDIOCGUSAGE diff --git a/Documentation/hid/hidraw.rst b/Documentation/hid/hidraw.rst index f41c1f0f6252..b717ee5cdaef 100644 --- a/Documentation/hid/hidraw.rst +++ b/Documentation/hid/hidraw.rst @@ -21,7 +21,7 @@ Hidraw is the only alternative, short of writing a custom kernel driver, for these non-conformant devices. A benefit of hidraw is that its use by userspace applications is independent -of the underlying hardware type. Currently, Hidraw is implemented for USB +of the underlying hardware type. Currently, hidraw is implemented for USB and Bluetooth. In the future, as new hardware bus types are developed which use the HID specification, hidraw will be expanded to add support for these new bus types. @@ -31,9 +31,10 @@ create hidraw device nodes. Udev will typically create the device nodes directly under /dev (eg: /dev/hidraw0). As this location is distribution- and udev rule-dependent, applications should use libudev to locate hidraw devices attached to the system. There is a tutorial on libudev with a -working example at: +working example at:: http://www.signal11.us/oss/udev/ + https://web.archive.org/web/2019*/www.signal11.us The HIDRAW API --------------- diff --git a/Documentation/hid/intel-ish-hid.rst b/Documentation/hid/intel-ish-hid.rst index d4785cf6eefd..f6ce44ff611d 100644 --- a/Documentation/hid/intel-ish-hid.rst +++ b/Documentation/hid/intel-ish-hid.rst @@ -4,19 +4,19 @@ Intel Integrated Sensor Hub (ISH) A sensor hub enables the ability to offload sensor polling and algorithm processing to a dedicated low power co-processor. This allows the core -processor to go into low power modes more often, resulting in the increased +processor to go into low power modes more often, resulting in increased battery life. -There are many vendors providing external sensor hubs confirming to HID -Sensor usage tables, and used in several tablets, 2 in 1 convertible laptops -and embedded products. Linux had this support since Linux 3.9. +There are many vendors providing external sensor hubs conforming to HID +Sensor usage tables. These may be found in tablets, 2-in-1 convertible laptops +and embedded products. Linux has had this support since Linux 3.9. Intel® introduced integrated sensor hubs as a part of the SoC starting from Cherry Trail and now supported on multiple generations of CPU packages. There are many commercial devices already shipped with Integrated Sensor Hubs (ISH). -These ISH also comply to HID sensor specification, but the difference is the +These ISH also comply to HID sensor specification, but the difference is the transport protocol used for communication. The current external sensor hubs -mainly use HID over i2C or USB. But ISH doesn't use either i2c or USB. +mainly use HID over I2C or USB. But ISH doesn't use either I2C or USB. 1. Overview =========== @@ -35,7 +35,7 @@ for a very high speed communication:: ----------------- ---------------------- PCI PCI ----------------- ---------------------- - |Host controller| --> | ISH processor | + |Host controller| --> | ISH processor | ----------------- ---------------------- USB Link ----------------- ---------------------- @@ -50,13 +50,13 @@ applications implemented in the firmware. The ISH allows multiple sensor management applications executing in the firmware. Like USB endpoints the messaging can be to/from a client. As part of enumeration process, these clients are identified. These clients can be simple -HID sensor applications, sensor calibration application or senor firmware -update application. +HID sensor applications, sensor calibration applications or sensor firmware +update applications. The implementation model is similar, like USB bus, ISH transport is also implemented as a bus. Each client application executing in the ISH processor is registered as a device on this bus. The driver, which binds each device -(ISH HID driver) identifies the device type and registers with the hid core. +(ISH HID driver) identifies the device type and registers with the HID core. 2. ISH Implementation: Block Diagram ==================================== @@ -104,7 +104,7 @@ is registered as a device on this bus. The driver, which binds each device The ISH is exposed as "Non-VGA unclassified PCI device" to the host. The PCI product and vendor IDs are changed from different generations of processors. So -the source code which enumerate drivers needs to update from generation to +the source code which enumerates drivers needs to update from generation to generation. 3.2 Inter Processor Communication (IPC) driver @@ -112,41 +112,42 @@ generation. Location: drivers/hid/intel-ish-hid/ipc -The IPC message used memory mapped I/O. The registers are defined in +The IPC message uses memory mapped I/O. The registers are defined in hw-ish-regs.h. 3.2.1 IPC/FW message types ^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are two types of messages, one for management of link and other messages -are to and from transport layers. +There are two types of messages, one for management of link and another for +messages to and from transport layers. TX and RX of Transport messages ............................... -A set of memory mapped register offers support of multi byte messages TX and -RX (E.g.IPC_REG_ISH2HOST_MSG, IPC_REG_HOST2ISH_MSG). The IPC layer maintains -internal queues to sequence messages and send them in order to the FW. +A set of memory mapped register offers support of multi-byte messages TX and +RX (e.g. IPC_REG_ISH2HOST_MSG, IPC_REG_HOST2ISH_MSG). The IPC layer maintains +internal queues to sequence messages and send them in order to the firmware. Optionally the caller can register handler to get notification of completion. -A door bell mechanism is used in messaging to trigger processing in host and +A doorbell mechanism is used in messaging to trigger processing in host and client firmware side. When ISH interrupt handler is called, the ISH2HOST doorbell register is used by host drivers to determine that the interrupt is for ISH. Each side has 32 32-bit message registers and a 32-bit doorbell. Doorbell -register has the following format: -Bits 0..6: fragment length (7 bits are used) -Bits 10..13: encapsulated protocol -Bits 16..19: management command (for IPC management protocol) -Bit 31: doorbell trigger (signal H/W interrupt to the other side) -Other bits are reserved, should be 0. +register has the following format:: + + Bits 0..6: fragment length (7 bits are used) + Bits 10..13: encapsulated protocol + Bits 16..19: management command (for IPC management protocol) + Bit 31: doorbell trigger (signal H/W interrupt to the other side) + Other bits are reserved, should be 0. 3.2.2 Transport layer interface ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To abstract HW level IPC communication, a set of callbacks are registered. +To abstract HW level IPC communication, a set of callbacks is registered. The transport layer uses them to send and receive messages. -Refer to struct ishtp_hw_ops for callbacks. +Refer to struct ishtp_hw_ops for callbacks. 3.3 ISH Transport layer ----------------------- @@ -158,7 +159,7 @@ Location: drivers/hid/intel-ish-hid/ishtp/ The transport layer is a bi-directional protocol, which defines: - Set of commands to start, stop, connect, disconnect and flow control -(ishtp/hbm.h) for details +(see ishtp/hbm.h for details) - A flow control mechanism to avoid buffer overflows This protocol resembles bus messages described in the following document: @@ -168,14 +169,14 @@ specifications/dcmi-hi-1-0-spec.pdf "Chapter 7: Bus Message Layer" 3.3.2 Connection and Flow Control Mechanism ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Each FW client and a protocol is identified by an UUID. In order to communicate +Each FW client and a protocol is identified by a UUID. In order to communicate to a FW client, a connection must be established using connect request and response bus messages. If successful, a pair (host_client_id and fw_client_id) will identify the connection. Once connection is established, peers send each other flow control bus messages independently. Every peer may send a message only if it has received a -flow-control credit before. Once it sent a message, it may not send another one +flow-control credit before. Once it has sent a message, it may not send another one before receiving the next flow control credit. Either side can send disconnect request bus message to end communication. Also the link will be dropped if major FW reset occurs. @@ -209,7 +210,7 @@ and DMA_XFER_ACK act as ownership indicators. At initial state all outgoing memory belongs to the sender (TX to host, RX to FW), DMA_XFER transfers ownership on the region that contains ISHTP message to the receiving side, DMA_XFER_ACK returns ownership to the sender. A sender -needs not wait for previous DMA_XFER to be ack'ed, and may send another message +need not wait for previous DMA_XFER to be ack'ed, and may send another message as long as remaining continuous memory in its ownership is enough. In principle, multiple DMA_XFER and DMA_XFER_ACK messages may be sent at once (up to IPC MTU), thus allowing for interrupt throttling. @@ -219,8 +220,8 @@ fragments and via IPC otherwise. 3.3.4 Ring Buffers ^^^^^^^^^^^^^^^^^^ -When a client initiate a connection, a ring or RX and TX buffers are allocated. -The size of ring can be specified by the client. HID client set 16 and 32 for +When a client initiates a connection, a ring of RX and TX buffers is allocated. +The size of ring can be specified by the client. HID client sets 16 and 32 for TX and RX buffers respectively. On send request from client, the data to be sent is copied to one of the send ring buffer and scheduled to be sent using bus message protocol. These buffers are required because the FW may have not @@ -230,10 +231,10 @@ to send. Same thing holds true on receive side and flow control is required. 3.3.5 Host Enumeration ^^^^^^^^^^^^^^^^^^^^^^ -The host enumeration bus command allow discovery of clients present in the FW. +The host enumeration bus command allows discovery of clients present in the FW. There can be multiple sensor clients and clients for calibration function. -To ease in implantation and allow independent driver handle each client +To ease implementation and allow independent drivers to handle each client, this transport layer takes advantage of Linux Bus driver model. Each client is registered as device on the transport bus (ishtp bus). @@ -270,7 +271,7 @@ The ISHTP client driver is responsible for: The functionality in these drivers is the same as an external sensor hub. Refer to Documentation/hid/hid-sensor.rst for HID sensor -Documentation/ABI/testing/sysfs-bus-iio for IIO ABIs to user space +Documentation/ABI/testing/sysfs-bus-iio for IIO ABIs to user space. 3.6 End to End HID transport Sequence Diagram --------------------------------------------- @@ -341,9 +342,10 @@ Documentation/ABI/testing/sysfs-bus-iio for IIO ABIs to user space 3.7 ISH Debugging ----------------- -To debug ISH, event tracing mechanism is used. To enable debug logs -echo 1 > /sys/kernel/debug/tracing/events/intel_ish/enable -cat sys/kernel/debug/tracing/trace +To debug ISH, event tracing mechanism is used. To enable debug logs:: + + echo 1 > /sys/kernel/debug/tracing/events/intel_ish/enable + cat sys/kernel/debug/tracing/trace 3.8 ISH IIO sysfs Example on Lenovo thinkpad Yoga 260 ----------------------------------------------------- diff --git a/Documentation/hid/uhid.rst b/Documentation/hid/uhid.rst index b18cb96c885f..2243a6b75914 100644 --- a/Documentation/hid/uhid.rst +++ b/Documentation/hid/uhid.rst @@ -3,7 +3,7 @@ UHID - User-space I/O driver support for HID subsystem ====================================================== UHID allows user-space to implement HID transport drivers. Please see -hid-transport.txt for an introduction into HID transport drivers. This document +hid-transport.rst for an introduction into HID transport drivers. This document relies heavily on the definitions declared there. With UHID, a user-space transport driver can create kernel hid-devices for each @@ -15,7 +15,7 @@ There is an example user-space application in ./samples/uhid/uhid-example.c The UHID API ------------ -UHID is accessed through a character misc-device. The minor-number is allocated +UHID is accessed through a character misc-device. The minor number is allocated dynamically so you need to rely on udev (or similar) to create the device node. This is /dev/uhid by default. @@ -45,23 +45,23 @@ The "type" field defines the payload. For each type, there is a payload-structure available in the union "u" (except for empty payloads). This payload contains management and/or device data. -The first thing you should do is sending an UHID_CREATE2 event. This will -register the device. UHID will respond with an UHID_START event. You can now +The first thing you should do is send a UHID_CREATE2 event. This will +register the device. UHID will respond with a UHID_START event. You can now start sending data to and reading data from UHID. However, unless UHID sends the UHID_OPEN event, the internally attached HID Device Driver has no user attached. That is, you might put your device asleep unless you receive the UHID_OPEN event. If you receive the UHID_OPEN event, you should start I/O. If the last -user closes the HID device, you will receive an UHID_CLOSE event. This may be -followed by an UHID_OPEN event again and so on. There is no need to perform +user closes the HID device, you will receive a UHID_CLOSE event. This may be +followed by a UHID_OPEN event again and so on. There is no need to perform reference-counting in user-space. That is, you will never receive multiple -UHID_OPEN events without an UHID_CLOSE event. The HID subsystem performs +UHID_OPEN events without a UHID_CLOSE event. The HID subsystem performs ref-counting for you. You may decide to ignore UHID_OPEN/UHID_CLOSE, though. I/O is allowed even though the device may have no users. If you want to send data on the interrupt channel to the HID subsystem, you send -an HID_INPUT2 event with your raw data payload. If the kernel wants to send data -on the interrupt channel to the device, you will read an UHID_OUTPUT event. +a HID_INPUT2 event with your raw data payload. If the kernel wants to send data +on the interrupt channel to the device, you will read a UHID_OUTPUT event. Data requests on the control channel are currently limited to GET_REPORT and SET_REPORT (no other data reports on the control channel are defined so far). Those requests are always synchronous. That means, the kernel sends @@ -71,7 +71,7 @@ the response via UHID_GET_REPORT_REPLY and UHID_SET_REPORT_REPLY to the kernel. The kernel blocks internal driver-execution during such round-trips (times out after a hard-coded period). -If your device disconnects, you should send an UHID_DESTROY event. This will +If your device disconnects, you should send a UHID_DESTROY event. This will unregister the device. You can now send UHID_CREATE2 again to register a new device. If you close() the fd, the device is automatically unregistered and destroyed @@ -125,7 +125,7 @@ UHID_START: This is sent when the HID device is started. Consider this as an answer to UHID_CREATE2. This is always the first event that is sent. Note that this event might not be available immediately after write(UHID_CREATE2) returns. - Device drivers might required delayed setups. + Device drivers might require delayed setups. This event contains a payload of type uhid_start_req. The "dev_flags" field describes special behaviors of a device. The following flags are defined: @@ -149,7 +149,7 @@ UHID_STOP: reloaded/changed the device driver loaded on your HID device (or some other maintenance actions happened). - You can usually ignored any UHID_STOP events safely. + You can usually ignore any UHID_STOP events safely. UHID_OPEN: This is sent when the HID device is opened. That is, the data that the HID @@ -166,17 +166,17 @@ UHID_OUTPUT: This is sent if the HID device driver wants to send raw data to the I/O device on the interrupt channel. You should read the payload and forward it to the device. The payload is of type "struct uhid_output_req". - This may be received even though you haven't received UHID_OPEN, yet. + This may be received even though you haven't received UHID_OPEN yet. UHID_GET_REPORT: This event is sent if the kernel driver wants to perform a GET_REPORT request - on the control channeld as described in the HID specs. The report-type and + on the control channel as described in the HID specs. The report-type and report-number are available in the payload. The kernel serializes GET_REPORT requests so there will never be two in parallel. However, if you fail to respond with a UHID_GET_REPORT_REPLY, the request might silently time out. - Once you read a GET_REPORT request, you shall forward it to the hid device and - remember the "id" field in the payload. Once your hid device responds to the + Once you read a GET_REPORT request, you shall forward it to the HID device and + remember the "id" field in the payload. Once your HID device responds to the GET_REPORT (or if it fails), you must send a UHID_GET_REPORT_REPLY to the kernel with the exact same "id" as in the request. If the request already timed out, the kernel will ignore the response silently. The "id" field is @@ -184,7 +184,7 @@ UHID_GET_REPORT: UHID_SET_REPORT: This is the SET_REPORT equivalent of UHID_GET_REPORT. On receipt, you shall - send a SET_REPORT request to your hid device. Once it replies, you must tell + send a SET_REPORT request to your HID device. Once it replies, you must tell the kernel about it via UHID_SET_REPORT_REPLY. The same restrictions as for UHID_GET_REPORT apply. diff --git a/Documentation/hwmon/ab8500.rst b/Documentation/hwmon/ab8500.rst deleted file mode 100644 index 33f93a9cec04..000000000000 --- a/Documentation/hwmon/ab8500.rst +++ /dev/null @@ -1,26 +0,0 @@ -Kernel driver ab8500 -==================== - -Supported chips: - - * ST-Ericsson AB8500 - - Prefix: 'ab8500' - - Addresses scanned: - - - Datasheet: http://www.stericsson.com/developers/documentation.jsp - -Authors: - - Martin Persson - - Hongbo Zhang - -Description ------------ - -See also Documentation/hwmon/abx500.rst. This is the ST-Ericsson AB8500 specific -driver. - -Currently only the AB8500 internal sensor and one external sensor for battery -temperature are monitored. Other GPADC channels can also be monitored if needed -in future. diff --git a/Documentation/hwmon/abx500.rst b/Documentation/hwmon/abx500.rst deleted file mode 100644 index 3d88b2ce0f00..000000000000 --- a/Documentation/hwmon/abx500.rst +++ /dev/null @@ -1,32 +0,0 @@ -Kernel driver abx500 -==================== - -Supported chips: - - * ST-Ericsson ABx500 series - - Prefix: 'abx500' - - Addresses scanned: - - - Datasheet: http://www.stericsson.com/developers/documentation.jsp - -Authors: - Martin Persson - Hongbo Zhang - -Description ------------ - -Every ST-Ericsson Ux500 SOC consists of both ABx500 and DBx500 physically, -this is kernel hwmon driver for ABx500. - -There are some GPADCs inside ABx500 which are designed for connecting to -thermal sensors, and there is also a thermal sensor inside ABx500 too, which -raises interrupt when critical temperature reached. - -This abx500 is a common layer which can monitor all of the sensors, every -specific abx500 chip has its special configurations in its own file, e.g. some -sensors can be configured invisible if they are not available on that chip, and -the corresponding gpadc_addr should be set to 0, thus this sensor won't be -polled. diff --git a/Documentation/hwmon/aht10.rst b/Documentation/hwmon/aht10.rst new file mode 100644 index 000000000000..482262ca117c --- /dev/null +++ b/Documentation/hwmon/aht10.rst @@ -0,0 +1,46 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Kernel driver aht10 +===================== + +Supported chips: + + * Aosong AHT10 + + Prefix: 'aht10' + + Addresses scanned: None + + Datasheet: + + Chinese: http://www.aosong.com/userfiles/files/media/AHT10%E4%BA%A7%E5%93%81%E6%89%8B%E5%86%8C%20A3%2020201210.pdf + English: https://server4.eca.ir/eshop/AHT10/Aosong_AHT10_en_draft_0c.pdf + +Author: Johannes Cornelis Draaijer + + +Description +----------- + +The AHT10 is a Temperature and Humidity sensor + +The address of this i2c device may only be 0x38 + +Usage Notes +----------- + +This driver does not probe for AHT10 devices, as there is no reliable +way to determine if an i2c chip is or isn't an AHT10. The device has +to be instantiated explicitly with the address 0x38. See +Documentation/i2c/instantiating-devices.rst for details. + +Sysfs entries +------------- + +=============== ============================================ +temp1_input Measured temperature in millidegrees Celcius +humidity1_input Measured humidity in %H +update_interval The minimum interval for polling the sensor, + in milliseconds. Writable. Must be at + least 2000. +=============== ============================================ diff --git a/Documentation/hwmon/ina2xx.rst b/Documentation/hwmon/ina2xx.rst index f78a5cd44c4c..27d2e39bc8ac 100644 --- a/Documentation/hwmon/ina2xx.rst +++ b/Documentation/hwmon/ina2xx.rst @@ -74,7 +74,7 @@ bus supply voltage. The shunt value in micro-ohms can be set via platform data or device tree at compile-time or via the shunt_resistor attribute in sysfs at run-time. Please -refer to the Documentation/devicetree/bindings/hwmon/ina2xx.txt for bindings +refer to the Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml for bindings if the device tree is used. Additionally ina226 supports update_interval attribute as described in diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index fcb870ce6286..8d5a2df1ecb6 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -18,10 +18,8 @@ Hardware Monitoring Kernel Drivers .. toctree:: :maxdepth: 1 - ab8500 abituguru abituguru3 - abx500 acpi_power_meter ad7314 adc128d818 @@ -39,6 +37,7 @@ Hardware Monitoring Kernel Drivers adt7462 adt7470 adt7475 + aht10 amc6821 amd_energy asb100 @@ -178,6 +177,7 @@ Hardware Monitoring Kernel Drivers tmp401 tmp421 tmp513 + tps23861 tps40422 tps53679 twl4030-madc-hwmon diff --git a/Documentation/hwmon/max16601.rst b/Documentation/hwmon/max16601.rst index 346e74674c51..92c0a7d7808c 100644 --- a/Documentation/hwmon/max16601.rst +++ b/Documentation/hwmon/max16601.rst @@ -5,6 +5,14 @@ Kernel driver max16601 Supported chips: + * Maxim MAX16508 + + Prefix: 'max16508' + + Addresses scanned: - + + Datasheet: Not published + * Maxim MAX16601 Prefix: 'max16601' @@ -19,8 +27,8 @@ Author: Guenter Roeck Description ----------- -This driver supports the MAX16601 VR13.HC Dual-Output Voltage Regulator -Chipset. +This driver supports the MAX16508 VR13 Dual-Output Voltage Regulator +as well as the MAX16601 VR13.HC Dual-Output Voltage Regulator chipsets. The driver is a client driver to the core PMBus driver. Please see Documentation/hwmon/pmbus.rst for details on PMBus client drivers. @@ -45,115 +53,76 @@ Sysfs entries The following attributes are supported. -======================= ======================================================= -in1_label "vin1" -in1_input VCORE input voltage. -in1_alarm Input voltage alarm. +=============================== =============================================== +in1_label "vin1" +in1_input VCORE input voltage. +in1_alarm Input voltage alarm. -in2_label "vout1" -in2_input VCORE output voltage. -in2_alarm Output voltage alarm. +in2_label "vout1" +in2_input VCORE output voltage. +in2_alarm Output voltage alarm. -curr1_label "iin1" -curr1_input VCORE input current, derived from duty cycle and output - current. -curr1_max Maximum input current. -curr1_max_alarm Current high alarm. +curr1_label "iin1" +curr1_input VCORE input current, derived from duty cycle + and output current. +curr1_max Maximum input current. +curr1_max_alarm Current high alarm. -curr2_label "iin1.0" -curr2_input VCORE phase 0 input current. +curr[P+2]_label "iin1.P" +curr[P+2]_input VCORE phase P input current. -curr3_label "iin1.1" -curr3_input VCORE phase 1 input current. +curr[N+2]_label "iin2" +curr[N+2]_input VCORE input current, derived from sensor + element. + 'N' is the number of enabled/populated phases. -curr4_label "iin1.2" -curr4_input VCORE phase 2 input current. +curr[N+3]_label "iin3" +curr[N+3]_input VSA input current. -curr5_label "iin1.3" -curr5_input VCORE phase 3 input current. +curr[N+4]_label "iout1" +curr[N+4]_input VCORE output current. +curr[N+4]_crit Critical output current. +curr[N+4]_crit_alarm Output current critical alarm. +curr[N+4]_max Maximum output current. +curr[N+4]_max_alarm Output current high alarm. -curr6_label "iin1.4" -curr6_input VCORE phase 4 input current. +curr[N+P+5]_label "iout1.P" +curr[N+P+5]_input VCORE phase P output current. -curr7_label "iin1.5" -curr7_input VCORE phase 5 input current. +curr[2*N+5]_label "iout3" +curr[2*N+5]_input VSA output current. +curr[2*N+5]_highest Historical maximum VSA output current. +curr[2*N+5]_reset_history Write any value to reset curr21_highest. +curr[2*N+5]_crit Critical output current. +curr[2*N+5]_crit_alarm Output current critical alarm. +curr[2*N+5]_max Maximum output current. +curr[2*N+5]_max_alarm Output current high alarm. -curr8_label "iin1.6" -curr8_input VCORE phase 6 input current. +power1_label "pin1" +power1_input Input power, derived from duty cycle and output + current. +power1_alarm Input power alarm. -curr9_label "iin1.7" -curr9_input VCORE phase 7 input current. +power2_label "pin2" +power2_input Input power, derived from input current sensor. -curr10_label "iin2" -curr10_input VCORE input current, derived from sensor element. +power3_label "pout" +power3_input Output power. -curr11_label "iin3" -curr11_input VSA input current. +temp1_input VCORE temperature. +temp1_crit Critical high temperature. +temp1_crit_alarm Chip temperature critical high alarm. +temp1_max Maximum temperature. +temp1_max_alarm Chip temperature high alarm. -curr12_label "iout1" -curr12_input VCORE output current. -curr12_crit Critical output current. -curr12_crit_alarm Output current critical alarm. -curr12_max Maximum output current. -curr12_max_alarm Output current high alarm. +temp2_input TSENSE_0 temperature +temp3_input TSENSE_1 temperature +temp4_input TSENSE_2 temperature +temp5_input TSENSE_3 temperature -curr13_label "iout1.0" -curr13_input VCORE phase 0 output current. - -curr14_label "iout1.1" -curr14_input VCORE phase 1 output current. - -curr15_label "iout1.2" -curr15_input VCORE phase 2 output current. - -curr16_label "iout1.3" -curr16_input VCORE phase 3 output current. - -curr17_label "iout1.4" -curr17_input VCORE phase 4 output current. - -curr18_label "iout1.5" -curr18_input VCORE phase 5 output current. - -curr19_label "iout1.6" -curr19_input VCORE phase 6 output current. - -curr20_label "iout1.7" -curr20_input VCORE phase 7 output current. - -curr21_label "iout3" -curr21_input VSA output current. -curr21_highest Historical maximum VSA output current. -curr21_reset_history Write any value to reset curr21_highest. -curr21_crit Critical output current. -curr21_crit_alarm Output current critical alarm. -curr21_max Maximum output current. -curr21_max_alarm Output current high alarm. - -power1_label "pin1" -power1_input Input power, derived from duty cycle and output current. -power1_alarm Input power alarm. - -power2_label "pin2" -power2_input Input power, derived from input current sensor. - -power3_label "pout" -power3_input Output power. - -temp1_input VCORE temperature. -temp1_crit Critical high temperature. -temp1_crit_alarm Chip temperature critical high alarm. -temp1_max Maximum temperature. -temp1_max_alarm Chip temperature high alarm. - -temp2_input TSENSE_0 temperature -temp3_input TSENSE_1 temperature -temp4_input TSENSE_2 temperature -temp5_input TSENSE_3 temperature - -temp6_input VSA temperature. -temp6_crit Critical high temperature. -temp6_crit_alarm Chip temperature critical high alarm. -temp6_max Maximum temperature. -temp6_max_alarm Chip temperature high alarm. -======================= ======================================================= +temp6_input VSA temperature. +temp6_crit Critical high temperature. +temp6_crit_alarm Chip temperature critical high alarm. +temp6_max Maximum temperature. +temp6_max_alarm Chip temperature high alarm. +=============================== =============================================== diff --git a/Documentation/hwmon/nct6683.rst b/Documentation/hwmon/nct6683.rst index 8646ad519fcd..2e1408d174bd 100644 --- a/Documentation/hwmon/nct6683.rst +++ b/Documentation/hwmon/nct6683.rst @@ -61,5 +61,6 @@ Board Firmware version Intel DH87RL NCT6683D EC firmware version 1.0 build 04/03/13 Intel DH87MC NCT6683D EC firmware version 1.0 build 04/03/13 Intel DB85FL NCT6683D EC firmware version 1.0 build 04/03/13 +ASRock X570 NCT6683D EC firmware version 1.0 build 06/28/19 MSI B550 NCT6687D EC firmware version 1.0 build 05/07/20 =============== =============================================== diff --git a/Documentation/hwmon/tps23861.rst b/Documentation/hwmon/tps23861.rst new file mode 100644 index 000000000000..46d121ff3f31 --- /dev/null +++ b/Documentation/hwmon/tps23861.rst @@ -0,0 +1,41 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Kernel driver tps23861 +====================== + +Supported chips: + * Texas Instruments TPS23861 + + Prefix: 'tps23861' + + Datasheet: https://www.ti.com/lit/gpn/tps23861 + +Author: Robert Marko + +Description +----------- + +This driver supports hardware monitoring for Texas Instruments TPS23861 PoE PSE. + +TPS23861 is a quad port IEEE802.3at PSE controller with optional I2C control +and monitoring capabilities. + +TPS23861 offers three modes of operation: Auto, Semi-Auto and Manual. + +This driver only supports the Auto mode of operation providing monitoring +as well as enabling/disabling the four ports. + +Sysfs entries +------------- + +======================= ===================================================================== +in[0-3]_input Voltage on ports [1-4] +in[0-3]_label "Port[1-4]" +in4_input IC input voltage +in4_label "Input" +temp1_input IC die temperature +temp1_label "Die" +curr[1-4]_input Current on ports [1-4] +in[1-4]_label "Port[1-4]" +in[0-3]_enable Enable/disable ports [1-4] +======================= ===================================================================== diff --git a/Documentation/i2c/slave-testunit-backend.rst b/Documentation/i2c/slave-testunit-backend.rst index 2c38e64f0bac..ecfc2abec32d 100644 --- a/Documentation/i2c/slave-testunit-backend.rst +++ b/Documentation/i2c/slave-testunit-backend.rst @@ -22,8 +22,9 @@ Instantiating the device is regular. Example for bus 0, address 0x30: After that, you will have a write-only device listening. Reads will just return an 8-bit version number of the testunit. When writing, the device consists of 4 -8-bit registers and all must be written to start a testcase, i.e. you must -always write 4 bytes to the device. The registers are: +8-bit registers and, except for some "partial" commands, all registers must be +written to start a testcase, i.e. you usually write 4 bytes to the device. The +registers are: 0x00 CMD - which test to trigger 0x01 DATAL - configuration byte 1 for the test @@ -67,3 +68,21 @@ status word is currently ignored in the Linux Kernel. Example to send a notification after 10ms: # i2cset -y 0 0x30 0x02 0x42 0x64 0x01 i + +0x03 SMBUS_BLOCK_PROC_CALL (partial command) + DATAL - must be '1', i.e. one further byte will be written + DATAH - number of bytes to be sent back + DELAY - not applicable, partial command! + +This test will respond to a block process call as defined by the SMBus +specification. The one data byte written specifies how many bytes will be sent +back in the following read transfer. Note that in this read transfer, the +testunit will prefix the length of the bytes to follow. So, if your host bus +driver emulates SMBus calls like the majority does, it needs to support the +I2C_M_RECV_LEN flag of an i2c_msg. This is a good testcase for it. The returned +data consists of the length first, and then of an array of bytes from length-1 +to 0. Here is an example which emulates i2c_smbus_block_process_call() using +i2ctransfer (you need i2c-tools v4.2 or later): + +# i2ctransfer -y 0 w3@0x30 0x03 0x01 0x10 r? +0x10 0x0f 0x0e 0x0d 0x0c 0x0b 0x0a 0x09 0x08 0x07 0x06 0x05 0x04 0x03 0x02 0x01 0x00 diff --git a/Documentation/iio/ep93xx_adc.rst b/Documentation/iio/ep93xx_adc.rst index 4fd8dea3f6b8..0af0e9040457 100644 --- a/Documentation/iio/ep93xx_adc.rst +++ b/Documentation/iio/ep93xx_adc.rst @@ -13,7 +13,7 @@ touchscreen/ADC module. ==================== Numbering scheme for channels 0..4 is defined in EP9301 and EP9302 datasheets. -EP9307, EP9312 and EP9312 have 3 channels more (total 8), but the numbering is +EP9307, EP9312 and EP9315 have 3 channels more (total 8), but the numbering is not defined. So the last three are numbered randomly, let's say. Assuming ep93xx_adc is IIO device0, you'd find the following entries under diff --git a/Documentation/index.rst b/Documentation/index.rst index 5888e8a7272f..31f2adc8542d 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -171,17 +171,6 @@ implementation. x86/index xtensa/index -Filesystem Documentation ------------------------- - -The documentation in this section are provided by specific filesystem -subprojects. - -.. toctree:: - :maxdepth: 2 - - filesystems/ext4/index - Other documentation ------------------- diff --git a/Documentation/input/event-codes.rst b/Documentation/input/event-codes.rst index b24b5343f5eb..3118fc1c1e26 100644 --- a/Documentation/input/event-codes.rst +++ b/Documentation/input/event-codes.rst @@ -236,6 +236,21 @@ A few EV_ABS codes have special meanings: - Used to describe multitouch input events. Please see multi-touch-protocol.txt for details. +* ABS_PRESSURE/ABS_MT_PRESSURE: + + - For touch devices, many devices converted contact size into pressure. + A finger flattens with pressure, causing a larger contact area and thus + pressure and contact size are directly related. This is not the case + for other devices, for example digitizers and touchpads with a true + pressure sensor ("pressure pads"). + + A device should set the resolution of the axis to indicate whether the + pressure is in measurable units. If the resolution is zero, the + pressure data is in arbitrary units. If the resolution is nonzero, the + pressure data is in units/gram. For example, a value of 10 with a + resolution of 1 represents 10 gram, a value of 10 with a resolution on + 1000 represents 10 microgram. + EV_SW ----- diff --git a/Documentation/input/multi-touch-protocol.rst b/Documentation/input/multi-touch-protocol.rst index 307fe22d9668..21c1e6a22888 100644 --- a/Documentation/input/multi-touch-protocol.rst +++ b/Documentation/input/multi-touch-protocol.rst @@ -260,6 +260,10 @@ ABS_MT_PRESSURE of TOUCH and WIDTH for pressure-based devices or any device with a spatial signal intensity distribution. + If the resolution is zero, the pressure data is in arbitrary units. + If the resolution is nonzero, the pressure data is in units/gram. See + :ref:`input-event-codes` for details. + ABS_MT_DISTANCE The distance, in surface units, between the contact and the surface. Zero distance means the contact is touching the surface. A positive number means diff --git a/Documentation/kbuild/makefiles.rst b/Documentation/kbuild/makefiles.rst index 910c6303c7ea..db3af0b45baf 100644 --- a/Documentation/kbuild/makefiles.rst +++ b/Documentation/kbuild/makefiles.rst @@ -12,7 +12,7 @@ This document describes the Linux kernel Makefiles. --- 3.1 Goal definitions --- 3.2 Built-in object goals - obj-y --- 3.3 Loadable module goals - obj-m - --- 3.4 Objects which export symbols + --- 3.4 --- 3.5 Library file goals - lib-y --- 3.6 Descending down in directories --- 3.7 Non-builtin vmlinux targets - extra-y @@ -247,12 +247,6 @@ more details, with real examples. kbuild will build an ext2.o file for you out of the individual parts and then link this into built-in.a, as you would expect. -3.4 Objects which export symbols --------------------------------- - - No special notation is required in the makefiles for - modules exporting symbols. - 3.5 Library file goals - lib-y ------------------------------ @@ -461,10 +455,8 @@ more details, with real examples. # drivers/scsi/Makefile CFLAGS_aha152x.o = -DAHA152X_STAT -DAUTOCONF - CFLAGS_gdth.o = # -DDEBUG_GDTH=2 -D__SERIAL__ -D__COM2__ \ - -DGDTH_STATISTICS - These two lines specify compilation flags for aha152x.o and gdth.o. + This line specify compilation flags for aha152x.o. $(AFLAGS_$@) is a similar feature for source files in assembly languages. diff --git a/Documentation/kernel-hacking/hacking.rst b/Documentation/kernel-hacking/hacking.rst index eed2136d847f..451523424942 100644 --- a/Documentation/kernel-hacking/hacking.rst +++ b/Documentation/kernel-hacking/hacking.rst @@ -346,8 +346,8 @@ routine. Before inventing your own cache of often-used objects consider using a slab cache in ``include/linux/slab.h`` -:c:func:`current()` -------------------- +:c:macro:`current` +------------------ Defined in ``include/asm/current.h`` diff --git a/Documentation/kernel-hacking/locking.rst b/Documentation/kernel-hacking/locking.rst index c3448929a824..ed1284c6f078 100644 --- a/Documentation/kernel-hacking/locking.rst +++ b/Documentation/kernel-hacking/locking.rst @@ -958,7 +958,7 @@ grabs a read lock, searches a list, fails to find what it wants, drops the read lock, grabs a write lock and inserts the object has a race condition. -If you don't see why, please stay the fuck away from my code. +If you don't see why, please stay away from my code. Racing Timers: A Kernel Pastime ------------------------------- diff --git a/Documentation/livepatch/index.rst b/Documentation/livepatch/index.rst index 525944063be7..43cce5fad705 100644 --- a/Documentation/livepatch/index.rst +++ b/Documentation/livepatch/index.rst @@ -13,6 +13,7 @@ Kernel Livepatching module-elf-format shadow-vars system-state + reliable-stacktrace .. only:: subproject and html diff --git a/Documentation/livepatch/livepatch.rst b/Documentation/livepatch/livepatch.rst index c2c598c4ead8..68e3651e8af9 100644 --- a/Documentation/livepatch/livepatch.rst +++ b/Documentation/livepatch/livepatch.rst @@ -6,20 +6,7 @@ This document outlines basic information about kernel livepatching. .. Table of Contents: - 1. Motivation - 2. Kprobes, Ftrace, Livepatching - 3. Consistency model - 4. Livepatch module - 4.1. New functions - 4.2. Metadata - 5. Livepatch life-cycle - 5.1. Loading - 5.2. Enabling - 5.3. Replacing - 5.4. Disabling - 5.5. Removing - 6. Sysfs - 7. Limitations +.. contents:: :local: 1. Motivation diff --git a/Documentation/livepatch/module-elf-format.rst b/Documentation/livepatch/module-elf-format.rst index 8c6b894c4661..dbe9b400e39f 100644 --- a/Documentation/livepatch/module-elf-format.rst +++ b/Documentation/livepatch/module-elf-format.rst @@ -7,14 +7,8 @@ This document outlines the Elf format requirements that livepatch modules must f .. Table of Contents - 1. Background and motivation - 2. Livepatch modinfo field - 3. Livepatch relocation sections - 3.1 Livepatch relocation section format - 4. Livepatch symbols - 4.1 A livepatch module's symbol table - 4.2 Livepatch symbol format - 5. Symbol table and Elf section access +.. contents:: :local: + 1. Background and motivation ============================ diff --git a/Documentation/livepatch/reliable-stacktrace.rst b/Documentation/livepatch/reliable-stacktrace.rst new file mode 100644 index 000000000000..67459d2ca2af --- /dev/null +++ b/Documentation/livepatch/reliable-stacktrace.rst @@ -0,0 +1,309 @@ +=================== +Reliable Stacktrace +=================== + +This document outlines basic information about reliable stacktracing. + +.. Table of Contents: + +.. contents:: :local: + +1. Introduction +=============== + +The kernel livepatch consistency model relies on accurately identifying which +functions may have live state and therefore may not be safe to patch. One way +to identify which functions are live is to use a stacktrace. + +Existing stacktrace code may not always give an accurate picture of all +functions with live state, and best-effort approaches which can be helpful for +debugging are unsound for livepatching. Livepatching depends on architectures +to provide a *reliable* stacktrace which ensures it never omits any live +functions from a trace. + + +2. Requirements +=============== + +Architectures must implement one of the reliable stacktrace functions. +Architectures using CONFIG_ARCH_STACKWALK must implement +'arch_stack_walk_reliable', and other architectures must implement +'save_stack_trace_tsk_reliable'. + +Principally, the reliable stacktrace function must ensure that either: + +* The trace includes all functions that the task may be returned to, and the + return code is zero to indicate that the trace is reliable. + +* The return code is non-zero to indicate that the trace is not reliable. + +.. note:: + In some cases it is legitimate to omit specific functions from the trace, + but all other functions must be reported. These cases are described in + futher detail below. + +Secondly, the reliable stacktrace function must be robust to cases where +the stack or other unwind state is corrupt or otherwise unreliable. The +function should attempt to detect such cases and return a non-zero error +code, and should not get stuck in an infinite loop or access memory in +an unsafe way. Specific cases are described in further detail below. + + +3. Compile-time analysis +======================== + +To ensure that kernel code can be correctly unwound in all cases, +architectures may need to verify that code has been compiled in a manner +expected by the unwinder. For example, an unwinder may expect that +functions manipulate the stack pointer in a limited way, or that all +functions use specific prologue and epilogue sequences. Architectures +with such requirements should verify the kernel compilation using +objtool. + +In some cases, an unwinder may require metadata to correctly unwind. +Where necessary, this metadata should be generated at build time using +objtool. + + +4. Considerations +================= + +The unwinding process varies across architectures, their respective procedure +call standards, and kernel configurations. This section describes common +details that architectures should consider. + +4.1 Identifying successful termination +-------------------------------------- + +Unwinding may terminate early for a number of reasons, including: + +* Stack or frame pointer corruption. + +* Missing unwind support for an uncommon scenario, or a bug in the unwinder. + +* Dynamically generated code (e.g. eBPF) or foreign code (e.g. EFI runtime + services) not following the conventions expected by the unwinder. + +To ensure that this does not result in functions being omitted from the trace, +even if not caught by other checks, it is strongly recommended that +architectures verify that a stacktrace ends at an expected location, e.g. + +* Within a specific function that is an entry point to the kernel. + +* At a specific location on a stack expected for a kernel entry point. + +* On a specific stack expected for a kernel entry point (e.g. if the + architecture has separate task and IRQ stacks). + +4.2 Identifying unwindable code +------------------------------- + +Unwinding typically relies on code following specific conventions (e.g. +manipulating a frame pointer), but there can be code which may not follow these +conventions and may require special handling in the unwinder, e.g. + +* Exception vectors and entry assembly. + +* Procedure Linkage Table (PLT) entries and veneer functions. + +* Trampoline assembly (e.g. ftrace, kprobes). + +* Dynamically generated code (e.g. eBPF, optprobe trampolines). + +* Foreign code (e.g. EFI runtime services). + +To ensure that such cases do not result in functions being omitted from a +trace, it is strongly recommended that architectures positively identify code +which is known to be reliable to unwind from, and reject unwinding from all +other code. + +Kernel code including modules and eBPF can be distinguished from foreign code +using '__kernel_text_address()'. Checking for this also helps to detect stack +corruption. + +There are several ways an architecture may identify kernel code which is deemed +unreliable to unwind from, e.g. + +* Placing such code into special linker sections, and rejecting unwinding from + any code in these sections. + +* Identifying specific portions of code using bounds information. + +4.3 Unwinding across interrupts and exceptions +---------------------------------------------- + +At function call boundaries the stack and other unwind state is expected to be +in a consistent state suitable for reliable unwinding, but this may not be the +case part-way through a function. For example, during a function prologue or +epilogue a frame pointer may be transiently invalid, or during the function +body the return address may be held in an arbitrary general purpose register. +For some architectures this may change at runtime as a result of dynamic +instrumentation. + +If an interrupt or other exception is taken while the stack or other unwind +state is in an inconsistent state, it may not be possible to reliably unwind, +and it may not be possible to identify whether such unwinding will be reliable. +See below for examples. + +Architectures which cannot identify when it is reliable to unwind such cases +(or where it is never reliable) must reject unwinding across exception +boundaries. Note that it may be reliable to unwind across certain +exceptions (e.g. IRQ) but unreliable to unwind across other exceptions +(e.g. NMI). + +Architectures which can identify when it is reliable to unwind such cases (or +have no such cases) should attempt to unwind across exception boundaries, as +doing so can prevent unnecessarily stalling livepatch consistency checks and +permits livepatch transitions to complete more quickly. + +4.4 Rewriting of return addresses +--------------------------------- + +Some trampolines temporarily modify the return address of a function in order +to intercept when that function returns with a return trampoline, e.g. + +* An ftrace trampoline may modify the return address so that function graph + tracing can intercept returns. + +* A kprobes (or optprobes) trampoline may modify the return address so that + kretprobes can intercept returns. + +When this happens, the original return address will not be in its usual +location. For trampolines which are not subject to live patching, where an +unwinder can reliably determine the original return address and no unwind state +is altered by the trampoline, the unwinder may report the original return +address in place of the trampoline and report this as reliable. Otherwise, an +unwinder must report these cases as unreliable. + +Special care is required when identifying the original return address, as this +information is not in a consistent location for the duration of the entry +trampoline or return trampoline. For example, considering the x86_64 +'return_to_handler' return trampoline: + +.. code-block:: none + + SYM_CODE_START(return_to_handler) + UNWIND_HINT_EMPTY + subq $24, %rsp + + /* Save the return values */ + movq %rax, (%rsp) + movq %rdx, 8(%rsp) + movq %rbp, %rdi + + call ftrace_return_to_handler + + movq %rax, %rdi + movq 8(%rsp), %rdx + movq (%rsp), %rax + addq $24, %rsp + JMP_NOSPEC rdi + SYM_CODE_END(return_to_handler) + +While the traced function runs its return address on the stack points to +the start of return_to_handler, and the original return address is stored in +the task's cur_ret_stack. During this time the unwinder can find the return +address using ftrace_graph_ret_addr(). + +When the traced function returns to return_to_handler, there is no longer a +return address on the stack, though the original return address is still stored +in the task's cur_ret_stack. Within ftrace_return_to_handler(), the original +return address is removed from cur_ret_stack and is transiently moved +arbitrarily by the compiler before being returned in rax. The return_to_handler +trampoline moves this into rdi before jumping to it. + +Architectures might not always be able to unwind such sequences, such as when +ftrace_return_to_handler() has removed the address from cur_ret_stack, and the +location of the return address cannot be reliably determined. + +It is recommended that architectures unwind cases where return_to_handler has +not yet been returned to, but architectures are not required to unwind from the +middle of return_to_handler and can report this as unreliable. Architectures +are not required to unwind from other trampolines which modify the return +address. + +4.5 Obscuring of return addresses +--------------------------------- + +Some trampolines do not rewrite the return address in order to intercept +returns, but do transiently clobber the return address or other unwind state. + +For example, the x86_64 implementation of optprobes patches the probed function +with a JMP instruction which targets the associated optprobe trampoline. When +the probe is hit, the CPU will branch to the optprobe trampoline, and the +address of the probed function is not held in any register or on the stack. + +Similarly, the arm64 implementation of DYNAMIC_FTRACE_WITH_REGS patches traced +functions with the following: + +.. code-block:: none + + MOV X9, X30 + BL + +The MOV saves the link register (X30) into X9 to preserve the return address +before the BL clobbers the link register and branches to the trampoline. At the +start of the trampoline, the address of the traced function is in X9 rather +than the link register as would usually be the case. + +Architectures must either ensure that unwinders either reliably unwind +such cases, or report the unwinding as unreliable. + +4.6 Link register unreliability +------------------------------- + +On some other architectures, 'call' instructions place the return address into a +link register, and 'return' instructions consume the return address from the +link register without modifying the register. On these architectures software +must save the return address to the stack prior to making a function call. Over +the duration of a function call, the return address may be held in the link +register alone, on the stack alone, or in both locations. + +Unwinders typically assume the link register is always live, but this +assumption can lead to unreliable stack traces. For example, consider the +following arm64 assembly for a simple function: + +.. code-block:: none + + function: + STP X29, X30, [SP, -16]! + MOV X29, SP + BL + LDP X29, X30, [SP], #16 + RET + +At entry to the function, the link register (x30) points to the caller, and the +frame pointer (X29) points to the caller's frame including the caller's return +address. The first two instructions create a new stackframe and update the +frame pointer, and at this point the link register and the frame pointer both +describe this function's return address. A trace at this point may describe +this function twice, and if the function return is being traced, the unwinder +may consume two entries from the fgraph return stack rather than one entry. + +The BL invokes 'other_function' with the link register pointing to this +function's LDR and the frame pointer pointing to this function's stackframe. +When 'other_function' returns, the link register is left pointing at the BL, +and so a trace at this point could result in 'function' appearing twice in the +backtrace. + +Similarly, a function may deliberately clobber the LR, e.g. + +.. code-block:: none + + caller: + STP X29, X30, [SP, -16]! + MOV X29, SP + ADR LR, + BLR LR + LDP X29, X30, [SP], #16 + RET + +The ADR places the address of 'callee' into the LR, before the BLR branches to +this address. If a trace is made immediately after the ADR, 'callee' will +appear to be the parent of 'caller', rather than the child. + +Due to cases such as the above, it may only be possible to reliably consume a +link register value at a function call boundary. Architectures where this is +the case must reject unwinding across exception boundaries unless they can +reliably identify when the LR or stack value should be used (e.g. using +metadata generated by objtool). diff --git a/Documentation/power/freezing-of-tasks.rst b/Documentation/power/freezing-of-tasks.rst index 8bd693399834..53b6a56c4635 100644 --- a/Documentation/power/freezing-of-tasks.rst +++ b/Documentation/power/freezing-of-tasks.rst @@ -134,7 +134,7 @@ Generally speaking, there is a couple of reasons to use the freezing of tasks: safeguards against race conditions that might occur in such a case. Although Linus Torvalds doesn't like the freezing of tasks, he said this in one -of the discussions on LKML (http://lkml.org/lkml/2007/4/27/608): +of the discussions on LKML (https://lore.kernel.org/r/alpine.LFD.0.98.0704271801020.9964@woody.linux-foundation.org): "RJW:> Why we freeze tasks at all or why we freeze kernel threads? diff --git a/Documentation/powerpc/syscall64-abi.rst b/Documentation/powerpc/syscall64-abi.rst index cf9b2857c72a..dabee3729e5a 100644 --- a/Documentation/powerpc/syscall64-abi.rst +++ b/Documentation/powerpc/syscall64-abi.rst @@ -46,25 +46,38 @@ stack frame LR and CR save fields are not used. Register preservation rules --------------------------- -Register preservation rules match the ELF ABI calling sequence with the -following differences: +Register preservation rules match the ELF ABI calling sequence with some +differences. -+------------------------------------------------------------------------+ -| For the sc instruction, differences with the ELF ABI | -+--------------+--------------+------------------------------------------+ -| r0 | Volatile | (System call number.) | -| rr3 | Volatile | (Parameter 1, and return value.) | -| rr4-r8 | Volatile | (Parameters 2-6.) | -| rcr0 | Volatile | (cr0.SO is the return error condition.) | -| rcr1, cr5-7 | Nonvolatile | | -| rlr | Nonvolatile | | -+--------------+--------------+------------------------------------------+ -| For the scv 0 instruction, differences with the ELF ABI | -+--------------+--------------+------------------------------------------+ -| r0 | Volatile | (System call number.) | -| r3 | Volatile | (Parameter 1, and return value.) | -| r4-r8 | Volatile | (Parameters 2-6.) | -+--------------+--------------+------------------------------------------+ +For the sc instruction, the differences from the ELF ABI are as follows: + ++--------------+--------------------+-----------------------------------------+ +| Register | Preservation Rules | Purpose | ++==============+====================+=========================================+ +| r0 | Volatile | (System call number.) | ++--------------+--------------------+-----------------------------------------+ +| r3 | Volatile | (Parameter 1, and return value.) | ++--------------+--------------------+-----------------------------------------+ +| r4-r8 | Volatile | (Parameters 2-6.) | ++--------------+--------------------+-----------------------------------------+ +| cr0 | Volatile | (cr0.SO is the return error condition.) | ++--------------+--------------------+-----------------------------------------+ +| cr1, cr5-7 | Nonvolatile | | ++--------------+--------------------+-----------------------------------------+ +| lr | Nonvolatile | | ++--------------+--------------------+-----------------------------------------+ + +For the scv 0 instruction, the differences from the ELF ABI are as follows: + ++--------------+--------------------+-----------------------------------------+ +| Register | Preservation Rules | Purpose | ++==============+====================+=========================================+ +| r0 | Volatile | (System call number.) | ++--------------+--------------------+-----------------------------------------+ +| r3 | Volatile | (Parameter 1, and return value.) | ++--------------+--------------------+-----------------------------------------+ +| r4-r8 | Volatile | (Parameters 2-6.) | ++--------------+--------------------+-----------------------------------------+ All floating point and vector data registers as well as control and status registers are nonvolatile. diff --git a/Documentation/process/4.Coding.rst b/Documentation/process/4.Coding.rst index 0825dc496f22..1f0d81f44e14 100644 --- a/Documentation/process/4.Coding.rst +++ b/Documentation/process/4.Coding.rst @@ -242,7 +242,7 @@ and try to avoid "fixes" which make the warning go away without addressing its cause. Note that not all compiler warnings are enabled by default. Build the -kernel with "make EXTRA_CFLAGS=-W" to get the full set. +kernel with "make KCFLAGS=-W" to get the full set. The kernel provides several configuration options which turn on debugging features; most of these are found in the "kernel hacking" submenu. Several diff --git a/Documentation/process/adding-syscalls.rst b/Documentation/process/adding-syscalls.rst index a3ecb236576c..906c47f1a9e5 100644 --- a/Documentation/process/adding-syscalls.rst +++ b/Documentation/process/adding-syscalls.rst @@ -501,7 +501,7 @@ table, but not from elsewhere in the kernel. If the syscall functionality is useful to be used within the kernel, needs to be shared between an old and a new syscall, or needs to be shared between a syscall and its compatibility variant, it should be implemented by means of a "helper" function (such as -``kern_xyzzy()``). This kernel function may then be called within the +``ksys_xyzzy()``). This kernel function may then be called within the syscall stub (``sys_xyzzy()``), the compatibility syscall stub (``compat_sys_xyzzy()``), and/or other kernel code. @@ -548,18 +548,18 @@ References and Sources https://lwn.net/Articles/486306/ - Recommendation from Andrew Morton that all related information for a new system call should come in the same email thread: - https://lkml.org/lkml/2014/7/24/641 + https://lore.kernel.org/r/20140724144747.3041b208832bbdf9fbce5d96@linux-foundation.org - Recommendation from Michael Kerrisk that a new system call should come with - a man page: https://lkml.org/lkml/2014/6/13/309 + a man page: https://lore.kernel.org/r/CAKgNAkgMA39AfoSoA5Pe1r9N+ZzfYQNvNPvcRN7tOvRb8+v06Q@mail.gmail.com - Suggestion from Thomas Gleixner that x86 wire-up should be in a separate - commit: https://lkml.org/lkml/2014/11/19/254 + commit: https://lore.kernel.org/r/alpine.DEB.2.11.1411191249560.3909@nanos - Suggestion from Greg Kroah-Hartman that it's good for new system calls to - come with a man-page & selftest: https://lkml.org/lkml/2014/3/19/710 + come with a man-page & selftest: https://lore.kernel.org/r/20140320025530.GA25469@kroah.com - Discussion from Michael Kerrisk of new system call vs. :manpage:`prctl(2)` extension: - https://lkml.org/lkml/2014/6/3/411 + https://lore.kernel.org/r/CAHO5Pa3F2MjfTtfNxa8LbnkeeU8=YJ+9tDqxZpw7Gz59E-4AUg@mail.gmail.com - Suggestion from Ingo Molnar that system calls that involve multiple arguments should encapsulate those arguments in a struct, which includes a - size field for future extensibility: https://lkml.org/lkml/2015/7/30/117 + size field for future extensibility: https://lore.kernel.org/r/20150730083831.GA22182@gmail.com - Numbering oddities arising from (re-)use of O_* numbering space flags: - commit 75069f2b5bfb ("vfs: renumber FMODE_NONOTIFY and add to uniqueness @@ -569,9 +569,9 @@ References and Sources - commit bb458c644a59 ("Safer ABI for O_TMPFILE") - Discussion from Matthew Wilcox about restrictions on 64-bit arguments: - https://lkml.org/lkml/2008/12/12/187 + https://lore.kernel.org/r/20081212152929.GM26095@parisc-linux.org - Recommendation from Greg Kroah-Hartman that unknown flags should be - policed: https://lkml.org/lkml/2014/7/17/577 + policed: https://lore.kernel.org/r/20140717193330.GB4703@kroah.com - Recommendation from Linus Torvalds that x32 system calls should prefer compatibility with 64-bit versions rather than 32-bit versions: - https://lkml.org/lkml/2011/8/31/244 + https://lore.kernel.org/r/CA+55aFxfmwfB7jbbrXxa=K7VBYPfAvmu3XOkGrLbB1UFjX1+Ew@mail.gmail.com diff --git a/Documentation/process/coding-style.rst b/Documentation/process/coding-style.rst index 98227226c4e5..42969ab37b34 100644 --- a/Documentation/process/coding-style.rst +++ b/Documentation/process/coding-style.rst @@ -69,9 +69,26 @@ something to hide: if (condition) do_this; do_something_everytime; +Don't use commas to avoid using braces: + +.. code-block:: c + + if (condition) + do_this(), do_that(); + +Always uses braces for multiple statements: + +.. code-block:: c + + if (condition) { + do_this(); + do_that(); + } + Don't put multiple assignments on a single line either. Kernel coding style is super simple. Avoid tricky expressions. + Outside of comments, documentation and except in Kconfig, spaces are never used for indentation, and the above example is deliberately broken. @@ -306,8 +323,7 @@ that counts the number of active users, you should call that Encoding the type of a function into the name (so-called Hungarian notation) is asinine - the compiler knows the types anyway and can check -those, and it only confuses the programmer. No wonder Microsoft makes buggy -programs. +those, and it only confuses the programmer. LOCAL variable names should be short, and to the point. If you have some random integer loop counter, it should probably be called ``i``. diff --git a/Documentation/process/howto.rst b/Documentation/process/howto.rst index 7a5c105e34d4..e4beeca57e5f 100644 --- a/Documentation/process/howto.rst +++ b/Documentation/process/howto.rst @@ -342,16 +342,10 @@ Adventurous testers are very welcome to runtime-test the linux-next. Bug Reporting ------------- -https://bugzilla.kernel.org is where the Linux kernel developers track kernel -bugs. Users are encouraged to report all bugs that they find in this -tool. For details on how to use the kernel bugzilla, please see: - - https://bugzilla.kernel.org/page.cgi?id=faq.html - The file 'Documentation/admin-guide/reporting-issues.rst' in the main kernel -source directory has a good template for how to report a possible kernel bug, -and details what kind of information is needed by the kernel developers to help -track down the problem. +source directory describes how to report a possible kernel bug, and details +what kind of information is needed by the kernel developers to help track +down the problem. Managing bug reports @@ -364,7 +358,13 @@ improve your skills, and other developers will be aware of your presence. Fixing bugs is one of the best ways to get merits among other developers, because not many people like wasting time fixing other people's bugs. -To work in the already reported bug reports, go to https://bugzilla.kernel.org. +To work on already reported bug reports, find a subsystem you are interested in. +Check the MAINTAINERS file where bugs for that subsystem get reported to; often +it will be a mailing list, rarely a bugtracker. Search the archives of said +place for recent reports and help where you see fit. You may also want to check +https://bugzilla.kernel.org for bug reports; only a handful of kernel subsystems +use it actively for reporting or tracking, nevertheless bugs for the whole +kernel get filed there. Mailing lists diff --git a/Documentation/process/magic-number.rst b/Documentation/process/magic-number.rst index c6dfe060ec2f..fa5a62f4150c 100644 --- a/Documentation/process/magic-number.rst +++ b/Documentation/process/magic-number.rst @@ -99,7 +99,6 @@ USB_SERIAL_PORT_MAGIC 0x7301 usb_serial_port ``drivers/usb/se CG_MAGIC 0x00090255 ufs_cylinder_group ``include/linux/ufs_fs.h`` RPORT_MAGIC 0x00525001 r_port ``drivers/char/rocket_int.h`` LSEMAGIC 0x05091998 lse ``drivers/fc4/fc.c`` -GDTIOCTL_MAGIC 0x06030f07 gdth_iowr_str ``drivers/scsi/gdth_ioctl.h`` RIEBL_MAGIC 0x09051990 ``drivers/net/atarilance.c`` NBD_REQUEST_MAGIC 0x12560953 nbd_request ``include/linux/nbd.h`` RED_MAGIC2 0x170fc2a5 (any) ``mm/slab.c`` @@ -142,7 +141,6 @@ PWC_MAGIC 0x89DC10AB pwc_device ``drivers/usb/me NBD_REPLY_MAGIC 0x96744668 nbd_reply ``include/linux/nbd.h`` ENI155_MAGIC 0xa54b872d midway_eprom ``drivers/atm/eni.h`` CODA_MAGIC 0xC0DAC0DA coda_file_info ``fs/coda/coda_fs_i.h`` -DPMEM_MAGIC 0xc0ffee11 gdt_pci_sram ``drivers/scsi/gdth.h`` YAM_MAGIC 0xF10A7654 yam_port ``drivers/net/hamradio/yam.c`` CCB_MAGIC 0xf2691ad2 ccb ``drivers/scsi/ncr53c8xx.c`` QUEUE_MAGIC_FREE 0xf7e1c9a3 queue_entry ``drivers/scsi/arm/queue.c`` diff --git a/Documentation/process/submit-checklist.rst b/Documentation/process/submit-checklist.rst index 230ee42f872f..b1bc2d37bd0a 100644 --- a/Documentation/process/submit-checklist.rst +++ b/Documentation/process/submit-checklist.rst @@ -89,30 +89,28 @@ and elsewhere regarding submitting Linux kernel patches. Patches that change userspace interfaces should be CCed to linux-api@vger.kernel.org. -19) Check that it all passes ``make headers_check``. - -20) Has been checked with injection of at least slab and page-allocation +19) Has been checked with injection of at least slab and page-allocation failures. See ``Documentation/fault-injection/``. If the new code is substantial, addition of subsystem-specific fault injection might be appropriate. -21) Newly-added code has been compiled with ``gcc -W`` (use - ``make EXTRA_CFLAGS=-W``). This will generate lots of noise, but is good +20) Newly-added code has been compiled with ``gcc -W`` (use + ``make KCFLAGS=-W``). This will generate lots of noise, but is good for finding bugs like "warning: comparison between signed and unsigned". -22) Tested after it has been merged into the -mm patchset to make sure +21) Tested after it has been merged into the -mm patchset to make sure that it still works with all of the other queued patches and various changes in the VM, VFS, and other subsystems. -23) All memory barriers {e.g., ``barrier()``, ``rmb()``, ``wmb()``} need a +22) All memory barriers {e.g., ``barrier()``, ``rmb()``, ``wmb()``} need a comment in the source code that explains the logic of what they are doing and why. -24) If any ioctl's are added by the patch, then also update +23) If any ioctl's are added by the patch, then also update ``Documentation/userspace-api/ioctl/ioctl-number.rst``. -25) If your modified source code depends on or uses any of the kernel +24) If your modified source code depends on or uses any of the kernel APIs or features that are related to the following ``Kconfig`` symbols, then test multiple builds with the related ``Kconfig`` symbols disabled and/or ``=m`` (if that option is available) [not all of these at the diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst index 5ba54120bef7..8c991c863628 100644 --- a/Documentation/process/submitting-patches.rst +++ b/Documentation/process/submitting-patches.rst @@ -556,6 +556,11 @@ which stable kernel versions should receive your fix. This is the preferred method for indicating a bug fixed by the patch. See :ref:`describe_changes` for more details. +Note: Attaching a Fixes: tag does not subvert the stable kernel rules +process nor the requirement to Cc: stable@vger.kernel.org on all stable +patch candidates. For more information, please read +:ref:`Documentation/process/stable-kernel-rules.rst ` + .. _the_canonical_patch_format: The canonical patch format @@ -679,6 +684,26 @@ generates appropriate diffstats by default.) See more details on the proper patch format in the following references. +Backtraces in commit mesages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Backtraces help document the call chain leading to a problem. However, +not all backtraces are helpful. For example, early boot call chains are +unique and obvious. Copying the full dmesg output verbatim, however, +adds distracting information like timestamps, module lists, register and +stack dumps. + +Therefore, the most useful backtraces should distill the relevant +information from the dump, which makes it easier to focus on the real +issue. Here is an example of a well-trimmed backtrace:: + + unchecked MSR access error: WRMSR to 0xd51 (tried to write 0x0000000000000064) + at rIP: 0xffffffffae059994 (native_write_msr+0x4/0x20) + Call Trace: + mba_wrmsr + update_domains + rdtgroup_mkdir + .. _explicit_in_reply_to: Explicit In-Reply-To headers @@ -769,13 +794,13 @@ Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer". NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people! - + Kernel Documentation/process/coding-style.rst: :ref:`Documentation/process/coding-style.rst ` Linus Torvalds's mail on the canonical patch format: - + Andi Kleen, "On submitting kernel patches" Some strategies to get difficult or controversial changes in. diff --git a/Documentation/scheduler/sched-bwc.rst b/Documentation/scheduler/sched-bwc.rst index 9801d6b284b1..845eee659199 100644 --- a/Documentation/scheduler/sched-bwc.rst +++ b/Documentation/scheduler/sched-bwc.rst @@ -2,8 +2,9 @@ CFS Bandwidth Control ===================== -[ This document only discusses CPU bandwidth control for SCHED_NORMAL. - The SCHED_RT case is covered in Documentation/scheduler/sched-rt-group.rst ] +.. note:: + This document only discusses CPU bandwidth control for SCHED_NORMAL. + The SCHED_RT case is covered in Documentation/scheduler/sched-rt-group.rst CFS bandwidth control is a CONFIG_FAIR_GROUP_SCHED extension which allows the specification of the maximum CPU bandwidth available to a group or hierarchy. @@ -25,9 +26,15 @@ Management ---------- Quota and period are managed within the cpu subsystem via cgroupfs. -cpu.cfs_quota_us: the total available run-time within a period (in microseconds) -cpu.cfs_period_us: the length of a period (in microseconds) -cpu.stat: exports throttling statistics [explained further below] +.. note:: + The cgroupfs files described in this section are only applicable + to cgroup v1. For cgroup v2, see + :ref:`Documentation/admin-guide/cgroupv2.rst `. + +- cpu.cfs_quota_us: the total available run-time within a period (in + microseconds) +- cpu.cfs_period_us: the length of a period (in microseconds) +- cpu.stat: exports throttling statistics [explained further below] The default values are:: diff --git a/Documentation/scheduler/sched-deadline.rst b/Documentation/scheduler/sched-deadline.rst index 14a2f7bf63fe..9d9be52f221a 100644 --- a/Documentation/scheduler/sched-deadline.rst +++ b/Documentation/scheduler/sched-deadline.rst @@ -707,7 +707,7 @@ Deadline Task Scheduling and how to prevent non-root users "cheat" the system? As already discussed, we are planning also to merge this work with the EDF - throttling patches [https://lkml.org/lkml/2010/2/23/239] but we still are in + throttling patches [https://lore.kernel.org/r/cover.1266931410.git.fabio@helm.retis] but we still are in the preliminary phases of the merge and we really seek feedback that would help us decide on the direction it should take. diff --git a/Documentation/scheduler/sched-design-CFS.rst b/Documentation/scheduler/sched-design-CFS.rst index a96c72651877..59b2d1fb4dc4 100644 --- a/Documentation/scheduler/sched-design-CFS.rst +++ b/Documentation/scheduler/sched-design-CFS.rst @@ -34,9 +34,9 @@ In CFS the virtual runtime is expressed and tracked via the per-task p->se.vruntime (nanosec-unit) value. This way, it's possible to accurately timestamp and measure the "expected CPU time" a task should have gotten. -[ small detail: on "ideal" hardware, at any time all tasks would have the same - p->se.vruntime value --- i.e., tasks would execute simultaneously and no task - would ever get "out of balance" from the "ideal" share of CPU time. ] + Small detail: on "ideal" hardware, at any time all tasks would have the same + p->se.vruntime value --- i.e., tasks would execute simultaneously and no task + would ever get "out of balance" from the "ideal" share of CPU time. CFS's task picking logic is based on this p->se.vruntime value and it is thus very simple: it always tries to run the task with the smallest p->se.vruntime diff --git a/Documentation/scsi/libsas.rst b/Documentation/scsi/libsas.rst index 7216b5d25800..6589dfefbc02 100644 --- a/Documentation/scsi/libsas.rst +++ b/Documentation/scsi/libsas.rst @@ -189,13 +189,8 @@ num_phys The event interface:: /* LLDD calls these to notify the class of an event. */ - void (*notify_ha_event)(struct sas_ha_struct *, enum ha_event); - void (*notify_port_event)(struct sas_phy *, enum port_event); - void (*notify_phy_event)(struct sas_phy *, enum phy_event); - -When sas_register_ha() returns, those are set and can be -called by the LLDD to notify the SAS layer of such events -the SAS layer. + void sas_notify_port_event(struct sas_phy *, enum port_event, gfp_t); + void sas_notify_phy_event(struct sas_phy *, enum phy_event, gfp_t); The port notification:: diff --git a/Documentation/scsi/scsi-parameters.rst b/Documentation/scsi/scsi-parameters.rst index dea5803f5c62..c42c55e1e25e 100644 --- a/Documentation/scsi/scsi-parameters.rst +++ b/Documentation/scsi/scsi-parameters.rst @@ -38,9 +38,6 @@ parameters may be changed at runtime by the command See drivers/scsi/BusLogic.c, comment before function BusLogic_ParseDriverOptions(). - gdth= [HW,SCSI] - See header of drivers/scsi/gdth.c. - gvp11= [HW,SCSI] ips= [HW,SCSI] Adaptec / IBM ServeRAID controller diff --git a/Documentation/security/keys/core.rst b/Documentation/security/keys/core.rst index aa0081685ee1..b3ed5c581034 100644 --- a/Documentation/security/keys/core.rst +++ b/Documentation/security/keys/core.rst @@ -1040,8 +1040,8 @@ The keyctl syscall functions are: "key" is the ID of the key to be watched. - "queue_fd" is a file descriptor referring to an open "/dev/watch_queue" - which manages the buffer into which notifications will be delivered. + "queue_fd" is a file descriptor referring to an open pipe which + manages the buffer into which notifications will be delivered. "filter" is either NULL to remove a watch or a filter specification to indicate what events are required from the key. diff --git a/Documentation/security/lsm-development.rst b/Documentation/security/lsm-development.rst index 31d92bc5fdd2..ac53e5065f79 100644 --- a/Documentation/security/lsm-development.rst +++ b/Documentation/security/lsm-development.rst @@ -2,7 +2,7 @@ Linux Security Module Development ================================= -Based on https://lkml.org/lkml/2007/10/26/215, +Based on https://lore.kernel.org/r/20071026073721.618b4778@laptopd505.fenrus.org, a new LSM is accepted into the kernel when its intent (a description of what it tries to protect against and in what cases one would expect to use it) has been appropriately documented in ``Documentation/admin-guide/LSM/``. diff --git a/Documentation/sphinx/automarkup.py b/Documentation/sphinx/automarkup.py index 953b24b6e2b4..acf5473002f3 100644 --- a/Documentation/sphinx/automarkup.py +++ b/Documentation/sphinx/automarkup.py @@ -51,7 +51,7 @@ RE_typedef = re.compile(r'\b(typedef)\s+([a-zA-Z_]\w+)', flags=ascii_p3) # Detects a reference to a documentation page of the form Documentation/... with # an optional extension # -RE_doc = re.compile(r'\bDocumentation(/[\w\-_/]+)(\.\w+)*') +RE_doc = re.compile(r'(\bDocumentation/)?((\.\./)*[\w\-/]+)\.(rst|txt)') RE_namespace = re.compile(r'^\s*..\s*c:namespace::\s*(\S+)\s*$') @@ -234,7 +234,10 @@ def markup_doc_ref(docname, app, match): # # Go through the dance of getting an xref out of the std domain # - target = match.group(1) + absolute = match.group(1) + target = match.group(2) + if absolute: + target = "/" + target xref = None pxref = addnodes.pending_xref('', refdomain = 'std', reftype = 'doc', reftarget = target, modname = None, diff --git a/Documentation/sphinx/cdomain.py b/Documentation/sphinx/cdomain.py index 014a5229e57a..ca8ac9e59ded 100644 --- a/Documentation/sphinx/cdomain.py +++ b/Documentation/sphinx/cdomain.py @@ -236,13 +236,7 @@ class CObject(Base_CObject): indextext = self.get_index_text(name) if indextext: - if major == 1 and minor < 4: - # indexnode's tuple changed in 1.4 - # https://github.com/sphinx-doc/sphinx/commit/e6a5a3a92e938fcd75866b4227db9e0524d58f7c - self.indexnode['entries'].append( - ('single', indextext, targetname, '')) - else: - self.indexnode['entries'].append( + self.indexnode['entries'].append( ('single', indextext, targetname, '', None)) class CDomain(Base_CDomain): diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index f3da859c9878..efe760e410c4 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -45,17 +45,7 @@ from docutils import nodes, statemachine from docutils.statemachine import ViewList from docutils.parsers.rst import directives, Directive from docutils.utils.error_reporting import ErrorString - -# -# AutodocReporter is only good up to Sphinx 1.7 -# -import sphinx - -Use_SSI = sphinx.__version__[:3] >= '1.7' -if Use_SSI: - from sphinx.util.docutils import switch_source_input -else: - from sphinx.ext.autodoc import AutodocReporter +from sphinx.util.docutils import switch_source_input __version__ = '1.0' @@ -179,16 +169,5 @@ class KernelCmd(Directive): return node.children def do_parse(self, content, node): - if Use_SSI: - with switch_source_input(self.state, content): - self.state.nested_parse(content, 0, node, match_titles=1) - else: - buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter - - self.state.memo.title_styles = [] - self.state.memo.section_level = 0 - self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter) - try: - self.state.nested_parse(content, 0, node, match_titles=1) - finally: - self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf + with switch_source_input(self.state, content): + self.state.nested_parse(content, 0, node, match_titles=1) diff --git a/Documentation/sphinx/kernel_feat.py b/Documentation/sphinx/kernel_feat.py index 2fee04f1dedd..c91ea2b27697 100644 --- a/Documentation/sphinx/kernel_feat.py +++ b/Documentation/sphinx/kernel_feat.py @@ -42,17 +42,7 @@ from docutils import nodes, statemachine from docutils.statemachine import ViewList from docutils.parsers.rst import directives, Directive from docutils.utils.error_reporting import ErrorString - -# -# AutodocReporter is only good up to Sphinx 1.7 -# -import sphinx - -Use_SSI = sphinx.__version__[:3] >= '1.7' -if Use_SSI: - from sphinx.util.docutils import switch_source_input -else: - from sphinx.ext.autodoc import AutodocReporter +from sphinx.util.docutils import switch_source_input __version__ = '1.0' @@ -154,16 +144,7 @@ class KernelFeat(Directive): buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter - if Use_SSI: - with switch_source_input(self.state, content): - self.state.nested_parse(content, 0, node, match_titles=1) - else: - self.state.memo.title_styles = [] - self.state.memo.section_level = 0 - self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter) - try: - self.state.nested_parse(content, 0, node, match_titles=1) - finally: - self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf + with switch_source_input(self.state, content): + self.state.nested_parse(content, 0, node, match_titles=1) return node.children diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py index e9857ab904f1..8189c33b9dda 100644 --- a/Documentation/sphinx/kerneldoc.py +++ b/Documentation/sphinx/kerneldoc.py @@ -37,18 +37,8 @@ import glob from docutils import nodes, statemachine from docutils.statemachine import ViewList from docutils.parsers.rst import directives, Directive - -# -# AutodocReporter is only good up to Sphinx 1.7 -# import sphinx - -Use_SSI = sphinx.__version__[:3] >= '1.7' -if Use_SSI: - from sphinx.util.docutils import switch_source_input -else: - from sphinx.ext.autodoc import AutodocReporter - +from sphinx.util.docutils import switch_source_input import kernellog __version__ = '1.0' @@ -163,18 +153,8 @@ class KernelDocDirective(Directive): return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))] def do_parse(self, result, node): - if Use_SSI: - with switch_source_input(self.state, result): - self.state.nested_parse(result, 0, node, match_titles=1) - else: - save = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter - self.state.memo.reporter = AutodocReporter(result, self.state.memo.reporter) - self.state.memo.title_styles, self.state.memo.section_level = [], 0 - try: - self.state.nested_parse(result, 0, node, match_titles=1) - finally: - self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = save - + with switch_source_input(self.state, result): + self.state.nested_parse(result, 0, node, match_titles=1) def setup(app): app.add_config_value('kerneldoc_bin', None, 'env') diff --git a/Documentation/sphinx/kernellog.py b/Documentation/sphinx/kernellog.py index 8ac7d274f542..0bc00c138cad 100644 --- a/Documentation/sphinx/kernellog.py +++ b/Documentation/sphinx/kernellog.py @@ -4,29 +4,19 @@ # only goes back to 1.6. So here's a wrapper layer to keep around for # as long as we support 1.4. # +# We don't support 1.4 anymore, but we'll keep the wrappers around until +# we change all the code to not use them anymore :) +# import sphinx +from sphinx.util import logging -if sphinx.__version__[:3] >= '1.6': - UseLogging = True - from sphinx.util import logging - logger = logging.getLogger('kerneldoc') -else: - UseLogging = False +logger = logging.getLogger('kerneldoc') def warn(app, message): - if UseLogging: - logger.warning(message) - else: - app.warn(message) + logger.warning(message) def verbose(app, message): - if UseLogging: - logger.verbose(message) - else: - app.verbose(message) + logger.verbose(message) def info(app, message): - if UseLogging: - logger.info(message) - else: - app.info(message) + logger.info(message) diff --git a/Documentation/sphinx/kfigure.py b/Documentation/sphinx/kfigure.py index 788704886eec..3c78828330be 100644 --- a/Documentation/sphinx/kfigure.py +++ b/Documentation/sphinx/kfigure.py @@ -49,26 +49,14 @@ import os from os import path import subprocess from hashlib import sha1 -import sys - from docutils import nodes from docutils.statemachine import ViewList from docutils.parsers.rst import directives from docutils.parsers.rst.directives import images import sphinx - from sphinx.util.nodes import clean_astext -from six import iteritems - import kernellog -PY3 = sys.version_info[0] == 3 - -if PY3: - _unicode = str -else: - _unicode = unicode - # Get Sphinx version major, minor, patch = sphinx.version_info[:3] if major == 1 and minor > 3: @@ -540,7 +528,7 @@ def add_kernel_figure_to_std_domain(app, doctree): docname = app.env.docname labels = std.data["labels"] - for name, explicit in iteritems(doctree.nametypes): + for name, explicit in doctree.nametypes.items(): if not explicit: continue labelid = doctree.nameids[name] diff --git a/Documentation/sphinx/maintainers_include.py b/Documentation/sphinx/maintainers_include.py index dc8fed48d3c2..328b3631a585 100755 --- a/Documentation/sphinx/maintainers_include.py +++ b/Documentation/sphinx/maintainers_include.py @@ -61,8 +61,6 @@ class MaintainersInclude(Include): field_content = "" for line in open(path): - if sys.version_info.major == 2: - line = unicode(line, 'utf-8') # Have we reached the end of the preformatted Descriptions text? if descriptions and line.startswith('Maintainers'): descriptions = False diff --git a/Documentation/sphinx/requirements.txt b/Documentation/sphinx/requirements.txt index 5030d346d23b..489f6626de67 100644 --- a/Documentation/sphinx/requirements.txt +++ b/Documentation/sphinx/requirements.txt @@ -1,4 +1,3 @@ docutils Sphinx==2.4.4 sphinx_rtd_theme -six diff --git a/Documentation/sphinx/rstFlatTable.py b/Documentation/sphinx/rstFlatTable.py index 2019a55f6b18..a3eea0bbe6ba 100755 --- a/Documentation/sphinx/rstFlatTable.py +++ b/Documentation/sphinx/rstFlatTable.py @@ -42,8 +42,6 @@ u""" # imports # ============================================================================== -import sys - from docutils import nodes from docutils.parsers.rst import directives, roles from docutils.parsers.rst.directives.tables import Table @@ -55,14 +53,6 @@ from docutils.utils import SystemMessagePropagation __version__ = '1.0' -PY3 = sys.version_info[0] == 3 -PY2 = sys.version_info[0] == 2 - -if PY3: - # pylint: disable=C0103, W0622 - unicode = str - basestring = str - # ============================================================================== def setup(app): # ============================================================================== diff --git a/Documentation/timers/timers-howto.rst b/Documentation/timers/timers-howto.rst index afb0a43b8cdf..5c169e3d29a8 100644 --- a/Documentation/timers/timers-howto.rst +++ b/Documentation/timers/timers-howto.rst @@ -75,7 +75,7 @@ NON-ATOMIC CONTEXT: - Why not msleep for (1ms - 20ms)? Explained originally here: - http://lkml.org/lkml/2007/8/3/250 + https://lore.kernel.org/r/15327.1186166232@lwn.net msleep(1~20) may not do what the caller intends, and will often sleep longer (~20 ms actual sleep for any diff --git a/Documentation/trace/coresight/coresight.rst b/Documentation/trace/coresight/coresight.rst index 0b73acb44efa..169749efd8d1 100644 --- a/Documentation/trace/coresight/coresight.rst +++ b/Documentation/trace/coresight/coresight.rst @@ -512,6 +512,38 @@ The --itrace option controls the type and frequency of synthesized events Note that only 64-bit programs are currently supported - further work is required to support instruction decode of 32-bit Arm programs. +2.2) Tracing PID + +The kernel can be built to write the PID value into the PE ContextID registers. +For a kernel running at EL1, the PID is stored in CONTEXTIDR_EL1. A PE may +implement Arm Virtualization Host Extensions (VHE), which the kernel can +run at EL2 as a virtualisation host; in this case, the PID value is stored in +CONTEXTIDR_EL2. + +perf provides PMU formats that program the ETM to insert these values into the +trace data; the PMU formats are defined as below: + + "contextid1": Available on both EL1 kernel and EL2 kernel. When the + kernel is running at EL1, "contextid1" enables the PID + tracing; when the kernel is running at EL2, this enables + tracing the PID of guest applications. + + "contextid2": Only usable when the kernel is running at EL2. When + selected, enables PID tracing on EL2 kernel. + + "contextid": Will be an alias for the option that enables PID + tracing. I.e, + contextid == contextid1, on EL1 kernel. + contextid == contextid2, on EL2 kernel. + +perf will always enable PID tracing at the relevant EL, this is accomplished by +automatically enable the "contextid" config - but for EL2 it is possible to make +specific adjustments using configs "contextid1" and "contextid2", E.g. if a user +wants to trace PIDs for both host and guest, the two configs "contextid1" and +"contextid2" can be set at the same time: + + perf record -e cs_etm/contextid1,contextid2/u -- vm + Generating coverage files for Feedback Directed Optimization: AutoFDO --------------------------------------------------------------------- diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst index 87cf5c010d5d..62c98e9bbdd9 100644 --- a/Documentation/trace/ftrace.rst +++ b/Documentation/trace/ftrace.rst @@ -1159,6 +1159,12 @@ Here are the available options: This simulates the original behavior of the trace file. When the file is closed, tracing will be enabled again. + hash-ptr + When set, "%p" in the event printk format displays the + hashed pointer value instead of real address. + This will be useful if you want to find out which hashed + value is corresponding to the real value in trace log. + record-cmd When any event or tracer is enabled, a hook is enabled in the sched_switch trace point to fill comm cache diff --git a/Documentation/translations/it_IT/process/4.Coding.rst b/Documentation/translations/it_IT/process/4.Coding.rst index a5e36aa60448..8012fe9497ae 100644 --- a/Documentation/translations/it_IT/process/4.Coding.rst +++ b/Documentation/translations/it_IT/process/4.Coding.rst @@ -256,7 +256,7 @@ e cercate di evitare le "riparazioni" che fan sparire l'avvertimento senza però averne trovato la causa. Tenete a mente che non tutti gli avvertimenti sono disabilitati di default. -Costruite il kernel con "make EXTRA_CFLAGS=-W" per ottenerli tutti. +Costruite il kernel con "make KCFLAGS=-W" per ottenerli tutti. Il kernel fornisce differenti opzioni che abilitano funzionalità di debugging; molti di queste sono trovano all'interno del sotto menu "kernel hacking". diff --git a/Documentation/translations/it_IT/process/adding-syscalls.rst b/Documentation/translations/it_IT/process/adding-syscalls.rst index bff0a82bf127..c478b6e8c292 100644 --- a/Documentation/translations/it_IT/process/adding-syscalls.rst +++ b/Documentation/translations/it_IT/process/adding-syscalls.rst @@ -611,21 +611,21 @@ Riferimenti e fonti https://lwn.net/Articles/486306/ - Raccomandazioni da Andrew Morton circa il fatto che tutte le informazioni su una nuova chiamata di sistema dovrebbero essere contenute nello stesso - filone di discussione di email: https://lkml.org/lkml/2014/7/24/641 + filone di discussione di email: https://lore.kernel.org/r/20140724144747.3041b208832bbdf9fbce5d96@linux-foundation.org - Raccomandazioni da Michael Kerrisk circa il fatto che le nuove chiamate di - sistema dovrebbero avere una pagina man: https://lkml.org/lkml/2014/6/13/309 + sistema dovrebbero avere una pagina man: https://lore.kernel.org/r/CAKgNAkgMA39AfoSoA5Pe1r9N+ZzfYQNvNPvcRN7tOvRb8+v06Q@mail.gmail.com - Consigli da Thomas Gleixner sul fatto che il collegamento all'architettura x86 dovrebbe avvenire in un *commit* differente: - https://lkml.org/lkml/2014/11/19/254 + https://lore.kernel.org/r/alpine.DEB.2.11.1411191249560.3909@nanos - Consigli da Greg Kroah-Hartman circa la bontà d'avere una pagina man e un programma di auto-verifica per le nuove chiamate di sistema: - https://lkml.org/lkml/2014/3/19/710 + https://lore.kernel.org/r/20140320025530.GA25469@kroah.com - Discussione di Michael Kerrisk sulle nuove chiamate di sistema contro - le estensioni :manpage:`prctl(2)`: https://lkml.org/lkml/2014/6/3/411 + le estensioni :manpage:`prctl(2)`: https://lore.kernel.org/r/CAHO5Pa3F2MjfTtfNxa8LbnkeeU8=YJ+9tDqxZpw7Gz59E-4AUg@mail.gmail.com - Consigli da Ingo Molnar che le chiamate di sistema con più argomenti dovrebbero incapsularli in una struttura che includa un argomento *size* per garantire l'estensibilità futura: - https://lkml.org/lkml/2015/7/30/117 + https://lore.kernel.org/r/20150730083831.GA22182@gmail.com - Un certo numero di casi strani emersi dall'uso (riuso) dei flag O_*: - commit 75069f2b5bfb ("vfs: renumber FMODE_NONOTIFY and add to uniqueness @@ -635,9 +635,9 @@ Riferimenti e fonti - commit bb458c644a59 ("Safer ABI for O_TMPFILE") - Discussion from Matthew Wilcox about restrictions on 64-bit arguments: - https://lkml.org/lkml/2008/12/12/187 + https://lore.kernel.org/r/20081212152929.GM26095@parisc-linux.org - Raccomandazioni da Greg Kroah-Hartman sul fatto che i flag sconosciuti dovrebbero - essere controllati: https://lkml.org/lkml/2014/7/17/577 + essere controllati: https://lore.kernel.org/r/20140717193330.GB4703@kroah.com - Raccomandazioni da Linus Torvalds che le chiamate di sistema x32 dovrebbero favorire la compatibilità con le versioni a 64-bit piuttosto che quelle a 32-bit: - https://lkml.org/lkml/2011/8/31/244 + https://lore.kernel.org/r/CA+55aFxfmwfB7jbbrXxa=K7VBYPfAvmu3XOkGrLbB1UFjX1+Ew@mail.gmail.com diff --git a/Documentation/translations/it_IT/process/submit-checklist.rst b/Documentation/translations/it_IT/process/submit-checklist.rst index 3e575502690f..614fc17d9086 100644 --- a/Documentation/translations/it_IT/process/submit-checklist.rst +++ b/Documentation/translations/it_IT/process/submit-checklist.rst @@ -104,7 +104,7 @@ sottomissione delle patch, in particolare l'iniezione di fallimenti specifici per il sottosistema. 22) Il nuovo codice è stato compilato con ``gcc -W`` (usate - ``make EXTRA_CFLAGS=-W``). Questo genererà molti avvisi, ma è ottimo + ``make KCFLAGS=-W``). Questo genererà molti avvisi, ma è ottimo per scovare bachi come "warning: comparison between signed and unsigned". 23) La patch è stata verificata dopo essere stata inclusa nella serie di patch diff --git a/Documentation/translations/it_IT/process/submitting-patches.rst b/Documentation/translations/it_IT/process/submitting-patches.rst index 966cd3242a60..ae00352346ed 100644 --- a/Documentation/translations/it_IT/process/submitting-patches.rst +++ b/Documentation/translations/it_IT/process/submitting-patches.rst @@ -731,13 +731,13 @@ Greg Kroah-Hartman, "Come scocciare un manutentore di un sottosistema" No!!!! Basta gigantesche bombe patch alle persone sulla lista linux-kernel@vger.kernel.org! - + Kernel Documentation/translations/it_IT/process/coding-style.rst: :ref:`Documentation/translations/it_IT/process/coding-style.rst ` E-mail di Linus Torvalds sul formato canonico di una patch: - + Andi Kleen, "Su come sottomettere patch del kernel" Alcune strategie su come sottomettere modifiche toste o controverse. diff --git a/Documentation/translations/ja_JP/SubmittingPatches b/Documentation/translations/ja_JP/SubmittingPatches index dd0c3280ba5a..6854f5add72e 100644 --- a/Documentation/translations/ja_JP/SubmittingPatches +++ b/Documentation/translations/ja_JP/SubmittingPatches @@ -702,13 +702,13 @@ Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer". NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people! - + Kernel Documentation/process/coding-style.rst: Linus Torvalds's mail on the canonical patch format: - + Andi Kleen, "On submitting kernel patches" Some strategies to get difficult or controversial changes in. diff --git a/Documentation/translations/ko_KR/howto.rst b/Documentation/translations/ko_KR/howto.rst index 240d29be38f2..787f1e85f8a0 100644 --- a/Documentation/translations/ko_KR/howto.rst +++ b/Documentation/translations/ko_KR/howto.rst @@ -345,7 +345,7 @@ https://bugzilla.kernel.org 는 리눅스 ì»¤ë„ ê°œë°œìžë“¤ì´ 커ë„ì˜ ë²„ https://bugzilla.kernel.org/page.cgi?id=faq.html -ë©”ì¸ ì»¤ë„ ì†ŒìŠ¤ ë””ë ‰í† ë¦¬ì— ìžˆëŠ” :ref:`admin-guide/reporting-bugs.rst ` +ë©”ì¸ ì»¤ë„ ì†ŒìŠ¤ ë””ë ‰í† ë¦¬ì— ìžˆëŠ” 'Documentation/admin-guide/reporting-issues.rst' 파ì¼ì€ ì»¤ë„ ë²„ê·¸ë¼ê³  ìƒê°ë˜ëŠ” ê²ƒì„ ë³´ê³ í•˜ëŠ” ë°©ë²•ì— ê´€í•œ ì¢‹ì€ í…œí”Œë¦¿ì´ë©° 문제를 추ì í•˜ê¸° 위해서 ì»¤ë„ ê°œë°œìžë“¤ì´ 필요로 하는 ì •ë³´ê°€ 무엇들ì¸ì§€ë¥¼ ìƒì„¸ížˆ 설명하고 있다. @@ -583,7 +583,7 @@ Patì´ë¼ëŠ” ì´ë¦„ì„ ê°€ì§„ ì—¬ìžê°€ ìžˆì„ ìˆ˜ë„ ìžˆëŠ” 것ì´ë‹¤. 리눅 "The Perfect Patch" - http://www.ozlabs.org/~akpm/stuff/tpp.txt + https://www.ozlabs.org/~akpm/stuff/tpp.txt ì´ ëª¨ë“  ê²ƒì„ í•˜ëŠ” ê²ƒì€ ë§¤ìš° 어려운 ì¼ì´ë‹¤. 완벽히 소화하는 ë°ëŠ” ì ì–´ë„ ëª‡ë…„ì´ diff --git a/Documentation/translations/ko_KR/index.rst b/Documentation/translations/ko_KR/index.rst index 27995c4233de..b9e27d20b039 100644 --- a/Documentation/translations/ko_KR/index.rst +++ b/Documentation/translations/ko_KR/index.rst @@ -10,3 +10,18 @@ :maxdepth: 1 howto + + +리눅스 ì»¤ë„ ë©”ëª¨ë¦¬ 배리어 +------------------------- + +.. raw:: latex + + \footnotesize + +.. include:: ./memory-barriers.txt + :literal: + +.. raw:: latex + + \normalsize diff --git a/Documentation/translations/zh_CN/admin-guide/cpu-load.rst b/Documentation/translations/zh_CN/admin-guide/cpu-load.rst index c972731c0e57..a73400a054ff 100644 --- a/Documentation/translations/zh_CN/admin-guide/cpu-load.rst +++ b/Documentation/translations/zh_CN/admin-guide/cpu-load.rst @@ -95,7 +95,7 @@ Linux通过``/proc/stat``å’Œ``/proc/uptime``导出å„ç§ä¿¡æ¯ï¼Œç”¨æˆ·ç©ºé—´å·¥ å‚考 --- -- http://lkml.org/lkml/2007/2/12/6 +- https://lore.kernel.org/r/loom.20070212T063225-663@post.gmane.org - Documentation/filesystems/proc.rst (1.8) diff --git a/Documentation/translations/zh_CN/arm/Booting b/Documentation/translations/zh_CN/arm/Booting index c3d26ce5f6de..5ecea0767893 100644 --- a/Documentation/translations/zh_CN/arm/Booting +++ b/Documentation/translations/zh_CN/arm/Booting @@ -124,7 +124,7 @@ bootloader 必须传递一个系统内存的ä½ç½®å’Œæœ€å°å€¼ï¼Œä»¥åŠæ ¹æ–‡ä»¶ bootloader 必须以 64bit 地å€å¯¹é½çš„å½¢å¼åŠ è½½ä¸€ä¸ªè®¾å¤‡æ ‘映åƒ(dtb)到系统 RAM 中，并用å¯åŠ¨æ•°æ®åˆå§‹åŒ–它。dtb æ ¼å¼åœ¨æ–‡æ¡£ -Documentation/devicetree/booting-without-of.rst 中。内核将会在 +https://www.devicetree.org/specifications/ 中。内核将会在 dtb 物ç†åœ°å€å¤„查找 dtb 魔数值（0xd00dfeed），以确定 dtb 是å¦å·²ç»ä»£æ›¿ 标签列表被传递进æ¥ã€‚ diff --git a/Documentation/translations/zh_CN/iio/ep93xx_adc.rst b/Documentation/translations/zh_CN/iio/ep93xx_adc.rst new file mode 100644 index 000000000000..7e91d2197867 --- /dev/null +++ b/Documentation/translations/zh_CN/iio/ep93xx_adc.rst @@ -0,0 +1,46 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../iio/ep93xx_adc` +:Translator: Yanteng Si + +.. _cn_iio_ep93xx_adc: + + +================================== +æ€ç¿é€»è¾‘ EP93xx 模拟数字转æ¢å™¨é©±åŠ¨ +================================== + +1. 概述 +======= + +该驱动åŒæ—¶é€‚用于具有5通é“模拟数字转æ¢å™¨çš„低端 (EP9301, Ep9302) 设备和10é€šé“ +触摸å±/模拟数字转æ¢å™¨çš„高端设备(EP9307, EP9312, EP9315)。 + +2. 通é“ç¼–å· +=========== + +EP9301å’ŒEP9302æ•°æ®è¡¨å®šä¹‰äº†é€šé“0..4çš„ç¼–å·æ–¹æ¡ˆã€‚虽然EP9307, EP9312å’ŒEP9315多 +了3个通é“（一共8个），但是编å·å¹¶æ²¡æœ‰å®šä¹‰ã€‚所以说最åŽä¸‰ä¸ªé€šé“是éšæœºç¼–å·çš„。 + +如果ep93xx_adc是IIO设备0，您将在以下ä½ç½®æ‰¾åˆ°æ¡ç›® +/sys/bus/iio/devices/iio:device0/: + + +-----------------+---------------+ + | sysfs å…¥å£ | ball/pin å称 | + +=================+===============+ + | in_voltage0_raw | YM | + +-----------------+---------------+ + | in_voltage1_raw | SXP | + +-----------------+---------------+ + | in_voltage2_raw | SXM | + +-----------------+---------------+ + | in_voltage3_raw | SYP | + +-----------------+---------------+ + | in_voltage4_raw | SYM | + +-----------------+---------------+ + | in_voltage5_raw | XP | + +-----------------+---------------+ + | in_voltage6_raw | XM | + +-----------------+---------------+ + | in_voltage7_raw | YP | + +-----------------+---------------+ diff --git a/Documentation/translations/zh_CN/iio/iio_configfs.rst b/Documentation/translations/zh_CN/iio/iio_configfs.rst new file mode 100644 index 000000000000..274488e8dce4 --- /dev/null +++ b/Documentation/translations/zh_CN/iio/iio_configfs.rst @@ -0,0 +1,102 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../iio/iio_configfs` +:Translator: Yanteng Si + +.. _cn_iio_configfs: + + +===================== +工业 IIO configfsæ”¯æŒ +===================== + +1. 概述 +======= + +Configfs是一ç§å†…核对象的基于文件系统的管ç†ç³»ç»Ÿï¼ŒIIO使用一些å¯ä»¥é€šè¿‡ +configfsè½»æ¾é…置的对象（例如：设备，触å‘器）。 + +关于configfs是如何è¿è¡Œçš„，请查阅Documentation/filesystems/configfs.rst +了解更多信æ¯ã€‚ + +2. 用法 +======= +为了使configfs支æŒIIO，我们需è¦åœ¨ç¼–译时选中configçš„CONFIG_IIO_CONFIGFS +选项。 + +然åŽï¼ŒæŒ‚è½½configfs文件系统(通常在 /config directory目录下）：： + + $ mkdir/config + $ mount -t configfs none/config + +此时，将创建所有默认IIO组，并å¯ä»¥åœ¨/ config / iio下对其进行访问。 下一章 +将介ç»å¯ç”¨çš„IIOé…置对象。 + +3. 软件触å‘器 +============= + +IIO默认configfs组之一是“触å‘器â€ç»„。 挂载configfsåŽå¯ä»¥è‡ªåŠ¨è®¿é—®å®ƒï¼Œå¹¶ä¸”å¯ +以在/config/iio/triggers下找到。 + +IIO软件触å‘器为创建多ç§è§¦å‘器类型æ供了支æŒã€‚ 通常在include/linux/iio +/sw_trigger.h：中的接å£ä¸‹å°†æ–°çš„触å‘器类型实现为å•ç‹¬çš„内核模å—： +:: + + /* + * drivers/iio/trigger/iio-trig-sample.c + * 一ç§æ–°è§¦å‘器类型的内核模å—实例 + */ + #include + + + static struct iio_sw_trigger *iio_trig_sample_probe(const char *name) + { + /* + * 这将分é…并注册一个IIO触å‘器以åŠå…¶ä»–触å‘器类型特性的åˆå§‹åŒ–。 + */ + } + + static int iio_trig_sample_remove(struct iio_sw_trigger *swt) + { + /* + * 这会废弃iio_trig_sample_probe中的æ“作 + */ + } + + static const struct iio_sw_trigger_ops iio_trig_sample_ops = { + .probe = iio_trig_sample_probe, + .remove = iio_trig_sample_remove, + }; + + static struct iio_sw_trigger_type iio_trig_sample = { + .name = "trig-sample", + .owner = THIS_MODULE, + .ops = &iio_trig_sample_ops, + }; + +module_iio_sw_trigger_driver(iio_trig_sample); + +æ¯ç§è§¦å‘器类型在/config/iio/triggers下都有其自己的目录。 加载iio-trig-sample +模å—将创建“ trig-sampleâ€è§¦å‘器类型目录/config/iio/triggers/trig-sample. + +我们支æŒä»¥ä¸‹ä¸­æ–­æºï¼ˆè§¦å‘器类型） + + * hrtimer,ä½¿ç”¨é«˜åˆ†è¾¨çŽ‡å®šæ—¶å™¨ä½œä¸ºä¸­æ–­æº + +3.1 Hrtimer触å‘å™¨åˆ›å»ºä¸Žé”€æ¯ +--------------------------- + +加载iio-trig-hrtimer模å—将注册hrtimer触å‘器类型，从而å…许用户在 +/config/iio/triggers/hrtimer下创建hrtimer触å‘器。 + +例如:: + + $ mkdir /config/iio/triggers/hrtimer/instance1 + $ rmdir /config/iio/triggers/hrtimer/instance1 + +æ¯ä¸ªè§¦å‘器å¯ä»¥å…·æœ‰ä¸€ä¸ªæˆ–多个独特的触å‘器类型的属性。 + +3.2 "hrtimer" 触å‘器类型属性 +---------------------------- + +"hrtimerâ€è§¦å‘器类型没有æ¥è‡ª/config dir的任何å¯é…置属性。 diff --git a/Documentation/translations/zh_CN/iio/index.rst b/Documentation/translations/zh_CN/iio/index.rst new file mode 100644 index 000000000000..7087076a10f6 --- /dev/null +++ b/Documentation/translations/zh_CN/iio/index.rst @@ -0,0 +1,20 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../iio/index` +:Translator: Yanteng Si + +.. _cn_iio_index: + + +======== +工业 I/O +======== + +.. toctree:: + :maxdepth: 1 + + iio_configfs + + ep93xx_adc diff --git a/Documentation/translations/zh_CN/mips/booting.rst b/Documentation/translations/zh_CN/mips/booting.rst new file mode 100644 index 000000000000..96453e1b962e --- /dev/null +++ b/Documentation/translations/zh_CN/mips/booting.rst @@ -0,0 +1,31 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../mips/booting` +:Translator: Yanteng Si + +.. _cn_booting: + +BMIPS设备树引导 +------------------------ + + 一些bootloadersåªæ”¯æŒåœ¨å†…核镜åƒå¼€å§‹åœ°å€å¤„çš„å•ä¸€å…¥å£ç‚¹ã€‚而其它 + bootloaders将跳转到ELF的开始地å€å¤„。两ç§æ–¹æ¡ˆéƒ½æ”¯æŒçš„；因为 + CONFIG_BOOT_RAW=y and CONFIG_NO_EXCEPT_FILL=y, 所以第一æ¡æŒ‡ä»¤ + 会立å³è·³è½¬åˆ°kernel_entry()å…¥å£å¤„执行。 + + 与arch/arm情况(b)类似，dt感知的引导加载程åºéœ€è¦è®¾ç½®ä»¥ä¸‹å¯„存器: + + a0 : 0 + + a1 : 0xffffffff + + a2 : RAM中指å‘设备树å—的物ç†æŒ‡é’ˆ(在chapterII中定义)。 + 设备树å¯ä»¥ä½äºŽå‰512MB物ç†åœ°å€ç©ºé—´(0x00000000 - + 0x1fffffff)的任何ä½ç½®ï¼Œä»¥64ä½è¾¹ç•Œå¯¹é½ã€‚ + + 传统bootloadersä¸ä¼šä½¿ç”¨è¿™æ ·çš„约定，并且它们ä¸ä¼ å…¥DTå—。 + 在这ç§æƒ…况下，Linux将通过选中CONFIG_DT_*查找DTB。 + + 以上约定åªåœ¨32ä½ç³»ç»Ÿä¸­å®šä¹‰ï¼Œå› ä¸ºç›®å‰æ²¡æœ‰ä»»ä½•64ä½çš„BMIPS实现。 diff --git a/Documentation/translations/zh_CN/mips/features.rst b/Documentation/translations/zh_CN/mips/features.rst new file mode 100644 index 000000000000..93d93d06b1b3 --- /dev/null +++ b/Documentation/translations/zh_CN/mips/features.rst @@ -0,0 +1,10 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../mips/features` +:Translator: Yanteng Si + +.. _cn_features: + +.. kernel-feat:: $srctree/Documentation/features mips diff --git a/Documentation/translations/zh_CN/mips/index.rst b/Documentation/translations/zh_CN/mips/index.rst new file mode 100644 index 000000000000..b85033f9d67c --- /dev/null +++ b/Documentation/translations/zh_CN/mips/index.rst @@ -0,0 +1,26 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../mips/index` +:Translator: Yanteng Si + +=========================== +MIPS特性文档 +=========================== + +.. toctree:: + :maxdepth: 2 + :numbered: + + booting + ingenic-tcu + + features + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/translations/zh_CN/mips/ingenic-tcu.rst b/Documentation/translations/zh_CN/mips/ingenic-tcu.rst new file mode 100644 index 000000000000..f04ba407384a --- /dev/null +++ b/Documentation/translations/zh_CN/mips/ingenic-tcu.rst @@ -0,0 +1,69 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../mips/ingenic-tcu` +:Translator: Yanteng Si + +.. _cn_ingenic-tcu: + +=============================================== +å›æ­£ JZ47xx SoC定时器/计数器硬件å•å…ƒ +=============================================== + +å›æ­£ JZ47xx SoC中的定时器/计数器å•å…ƒ(TCU)是一个多功能硬件å—。它有多达 +8个通é“，å¯ä»¥ç”¨ä½œè®¡æ•°å™¨ï¼Œè®¡æ—¶å™¨ï¼Œæˆ–脉冲宽度调制器。 + +- JZ4725B, JZ4750, JZ4755 åªæœ‰ï¼–个TCU通é“。其它SoC都有８个通é“。 + +- JZ4725B引入了一个独立的通é“，称为æ“作系统计时器(OST)。这是一个32ä½å¯ + 编程定时器。在JZ4760BåŠä»¥ä¸Šåž‹å·ä¸Šï¼Œå®ƒæ˜¯64ä½çš„。 + +- æ¯ä¸ªTCU通é“都有自己的时钟æºï¼Œå¯ä»¥é€šè¿‡ TCSR 寄存器设置通é“的父级时钟 + æºï¼ˆpclkã€extã€rtc）ã€å¼€å…³ä»¥åŠåˆ†é¢‘。 + + - 看门狗和OST硬件模å—在它们的寄存器空间中也有相åŒå½¢å¼çš„TCSR寄存器。 + - 用于关闭/å¼€å¯çš„ TCU 寄存器也å¯ä»¥å…³é—­/å¼€å¯çœ‹é—¨ç‹—å’Œ OST 时钟。 + +- æ¯ä¸ªTCU通é“在两ç§æ¨¡å¼çš„其中一ç§æ¨¡å¼ä¸‹è¿è¡Œï¼š + + - æ¨¡å¼ TCU1：通é“无法在ç¡çœ æ¨¡å¼ä¸‹è¿è¡Œï¼Œä½†æ›´æ˜“于æ“作。 + - æ¨¡å¼ TCU2：通é“å¯ä»¥åœ¨ç¡çœ æ¨¡å¼ä¸‹è¿è¡Œï¼Œä½†æ“作比 TCU1 通é“å¤æ‚一些。 + +- æ¯ä¸ª TCU 通é“的模å¼å–决于使用的SoC： + + - 在最è€çš„SoC（高于JZ4740），八个通é“都è¿è¡Œåœ¨TCU1模å¼ã€‚ + - 在 JZ4725B，通é“5è¿è¡Œåœ¨TCU2,其它通é“则è¿è¡Œåœ¨TCU1。 + - 在最新的SoC（JZ4750åŠä¹‹åŽï¼‰ï¼Œé€šé“1-2è¿è¡Œåœ¨TCU2，其它通é“则è¿è¡Œ + 在TCU1。 + +- æ¯ä¸ªé€šé“都å¯ä»¥ç”Ÿæˆä¸­æ–­ã€‚有些通é“共享一æ¡ä¸­æ–­çº¿ï¼Œè€Œæœ‰äº›æ²¡æœ‰ï¼Œå…¶åœ¨SoCåž‹ + å·ä¹‹é—´çš„å˜æ›´ï¼š + + - 在很è€çš„SoC（JZ4740åŠæ›´ä½Žï¼‰ï¼Œé€šé“0和通é“1有它们自己的中断线；通 + é“2-7共享最åŽä¸€æ¡ä¸­æ–­çº¿ã€‚ + - 在 JZ4725B，通é“0有它自己的中断线；通é“1-5共享一æ¡ä¸­æ–­çº¿ï¼›OST + 使用最åŽä¸€æ¡ä¸­æ–­çº¿ã€‚ + - 在比较新的SoC（JZ4750åŠä»¥åŽï¼‰ï¼Œé€šé“5有它自己的中断线；通 + é“0-4和（如果是8通é“）6-7全部共享一æ¡ä¸­æ–­çº¿ï¼›OST使用最åŽä¸€æ¡ä¸­ + 断线。 + +实现 +==== + +TCU硬件的功能分布在多个驱动程åºï¼š + +============== =================================== +时钟 drivers/clk/ingenic/tcu.c +中断 drivers/irqchip/irq-ingenic-tcu.c +定时器 drivers/clocksource/ingenic-timer.c +OST drivers/clocksource/ingenic-ost.c +脉冲宽度调制器 drivers/pwm/pwm-jz4740.c +看门狗 drivers/watchdog/jz4740_wdt.c +============== =================================== + +因为å¯ä»¥ä»Žç›¸åŒçš„寄存器控制属于ä¸åŒé©±åŠ¨ç¨‹åºå’Œæ¡†æž¶çš„TCUçš„å„ç§åŠŸèƒ½ï¼Œæ‰€ä»¥ +所有这些驱动程åºéƒ½é€šè¿‡ç›¸åŒçš„控制总线通用接å£è®¿é—®å®ƒä»¬çš„寄存器。 + +有关TCU驱动程åºçš„设备树绑定的更多信æ¯ï¼Œè¯·å‚阅: +Documentation/devicetree/bindings/timer/ingenic,tcu.yaml. diff --git a/Documentation/translations/zh_CN/process/4.Coding.rst b/Documentation/translations/zh_CN/process/4.Coding.rst index 959a06ba025c..66cd8ee07606 100644 --- a/Documentation/translations/zh_CN/process/4.Coding.rst +++ b/Documentation/translations/zh_CN/process/4.Coding.rst @@ -165,7 +165,7 @@ Linus对这个问题给出了最佳答案: 通常，这些警告都指å‘真正的问题。æ交以供审阅的代ç é€šå¸¸ä¸ä¼šäº§ç”Ÿä»»ä½•ç¼–译器警告。 在消除警告时，注æ„了解真正的原因，并尽é‡é¿å…“修å¤â€ï¼Œä½¿è­¦å‘Šæ¶ˆå¤±è€Œä¸è§£å†³å…¶åŽŸå› ã€‚ -请注æ„，并éžæ‰€æœ‰ç¼–译器警告都默认å¯ç”¨ã€‚使用“make EXTRA_CFLAGS=-Wâ€æž„建内核以 +请注æ„，并éžæ‰€æœ‰ç¼–译器警告都默认å¯ç”¨ã€‚使用“make KCFLAGS=-Wâ€æž„建内核以 获得完整集åˆã€‚ 内核æ供了几个é…置选项，å¯ä»¥æ‰“开调试功能；大多数é…置选项ä½äºŽâ€œkernel hacking†diff --git a/Documentation/translations/zh_CN/process/submitting-patches.rst b/Documentation/translations/zh_CN/process/submitting-patches.rst index 2e7dbaad4028..4fc6d16f5196 100644 --- a/Documentation/translations/zh_CN/process/submitting-patches.rst +++ b/Documentation/translations/zh_CN/process/submitting-patches.rst @@ -668,13 +668,13 @@ Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer". NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people! - + Kernel Documentation/process/coding-style.rst: :ref:`Documentation/translations/zh_CN/process/coding-style.rst ` Linus Torvalds's mail on the canonical patch format: - + Andi Kleen, "On submitting kernel patches" Some strategies to get difficult or controversial changes in. diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index a4c75a28c839..599bd4493944 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -157,7 +157,6 @@ Code Seq# Include File Comments 'I' all linux/isdn.h conflict! 'I' 00-0F drivers/isdn/divert/isdn_divert.h conflict! 'I' 40-4F linux/mISDNif.h conflict! -'J' 00-1F drivers/scsi/gdth_ioctl.h 'K' all linux/kd.h 'L' 00-1F linux/loop.h conflict! 'L' 10-1F drivers/scsi/mpt3sas/mpt3sas_ctl.h conflict! @@ -180,6 +179,7 @@ Code Seq# Include File Comments 'R' 00-1F linux/random.h conflict! 'R' 01 linux/rfkill.h conflict! 'R' C0-DF net/bluetooth/rfcomm.h +'R' E0 uapi/linux/fsl_mc.h 'S' all linux/cdrom.h conflict! 'S' 80-81 scsi/scsi_ioctl.h conflict! 'S' 82-FF scsi/scsi.h conflict! @@ -319,11 +319,14 @@ Code Seq# Include File Comments 0xA0 all linux/sdp/sdp.h Industrial Device Project 0xA1 0 linux/vtpm_proxy.h TPM Emulator Proxy Driver +0xA2 all uapi/linux/acrn.h ACRN hypervisor 0xA3 80-8F Port ACL in development: 0xA3 90-9F linux/dtlk.h 0xA4 00-1F uapi/linux/tee.h Generic TEE subsystem 0xA4 00-1F uapi/asm/sgx.h +0xA5 01 linux/surface_aggregator/cdev.h Microsoft Surface Platform System Aggregator + 0xAA 00-3F linux/uapi/linux/userfaultfd.h 0xAB 00-1F linux/nbd.h 0xAC 00-1F linux/raw.h @@ -352,6 +355,7 @@ Code Seq# Include File Comments 0xCC 00-0F drivers/misc/ibmvmc.h pseries VMC driver 0xCD 01 linux/reiserfs_fs.h +0xCE 01-02 uapi/linux/cxl_mem.h Compute Express Link Memory Devices 0xCF 02 fs/cifs/ioctl.c 0xDB 00-0F drivers/char/mwave/mwavepub.h 0xDD 00-3F ZFCP device driver see drivers/s390/scsi/ diff --git a/Documentation/virt/acrn/cpuid.rst b/Documentation/virt/acrn/cpuid.rst new file mode 100644 index 000000000000..65fa4b9c1798 --- /dev/null +++ b/Documentation/virt/acrn/cpuid.rst @@ -0,0 +1,46 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +ACRN CPUID bits +=============== + +A guest VM running on an ACRN hypervisor can check some of its features using +CPUID. + +ACRN cpuid functions are: + +function: 0x40000000 + +returns:: + + eax = 0x40000010 + ebx = 0x4e524341 + ecx = 0x4e524341 + edx = 0x4e524341 + +Note that this value in ebx, ecx and edx corresponds to the string +"ACRNACRNACRN". The value in eax corresponds to the maximum cpuid function +present in this leaf, and will be updated if more functions are added in the +future. + +function: define ACRN_CPUID_FEATURES (0x40000001) + +returns:: + + ebx, ecx, edx + eax = an OR'ed group of (1 << flag) + +where ``flag`` is defined as below: + +================================= =========== ================================ +flag value meaning +================================= =========== ================================ +ACRN_FEATURE_PRIVILEGED_VM 0 guest VM is a privileged VM +================================= =========== ================================ + +function: 0x40000010 + +returns:: + + ebx, ecx, edx + eax = (Virtual) TSC frequency in kHz. diff --git a/Documentation/virt/acrn/index.rst b/Documentation/virt/acrn/index.rst new file mode 100644 index 000000000000..b5f793e73df5 --- /dev/null +++ b/Documentation/virt/acrn/index.rst @@ -0,0 +1,12 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +ACRN Hypervisor +=============== + +.. toctree:: + :maxdepth: 1 + + introduction + io-request + cpuid diff --git a/Documentation/virt/acrn/introduction.rst b/Documentation/virt/acrn/introduction.rst new file mode 100644 index 000000000000..f8d081bc084d --- /dev/null +++ b/Documentation/virt/acrn/introduction.rst @@ -0,0 +1,43 @@ +.. SPDX-License-Identifier: GPL-2.0 + +ACRN Hypervisor Introduction +============================ + +The ACRN Hypervisor is a Type 1 hypervisor, running directly on bare-metal +hardware. It has a privileged management VM, called Service VM, to manage User +VMs and do I/O emulation. + +ACRN userspace is an application running in the Service VM that emulates +devices for a User VM based on command line configurations. ACRN Hypervisor +Service Module (HSM) is a kernel module in the Service VM which provides +hypervisor services to the ACRN userspace. + +Below figure shows the architecture. + +:: + + Service VM User VM + +----------------------------+ | +------------------+ + | +--------------+ | | | | + | |ACRN userspace| | | | | + | +--------------+ | | | | + |-----------------ioctl------| | | | ... + |kernel space +----------+ | | | | + | | HSM | | | | Drivers | + | +----------+ | | | | + +--------------------|-------+ | +------------------+ + +---------------------hypercall----------------------------------------+ + | ACRN Hypervisor | + +----------------------------------------------------------------------+ + | Hardware | + +----------------------------------------------------------------------+ + +ACRN userspace allocates memory for the User VM, configures and initializes the +devices used by the User VM, loads the virtual bootloader, initializes the +virtual CPU state and handles I/O request accesses from the User VM. It uses +ioctls to communicate with the HSM. HSM implements hypervisor services by +interacting with the ACRN Hypervisor via hypercalls. HSM exports a char device +interface (/dev/acrn_hsm) to userspace. + +The ACRN hypervisor is open for contribution from anyone. The source repo is +available at https://github.com/projectacrn/acrn-hypervisor. diff --git a/Documentation/virt/acrn/io-request.rst b/Documentation/virt/acrn/io-request.rst new file mode 100644 index 000000000000..6cc3ea0fa1f5 --- /dev/null +++ b/Documentation/virt/acrn/io-request.rst @@ -0,0 +1,97 @@ +.. SPDX-License-Identifier: GPL-2.0 + +I/O request handling +==================== + +An I/O request of a User VM, which is constructed by the hypervisor, is +distributed by the ACRN Hypervisor Service Module to an I/O client +corresponding to the address range of the I/O request. Details of I/O request +handling are described in the following sections. + +1. I/O request +-------------- + +For each User VM, there is a shared 4-KByte memory region used for I/O requests +communication between the hypervisor and Service VM. An I/O request is a +256-byte structure buffer, which is 'struct acrn_io_request', that is filled by +an I/O handler of the hypervisor when a trapped I/O access happens in a User +VM. ACRN userspace in the Service VM first allocates a 4-KByte page and passes +the GPA (Guest Physical Address) of the buffer to the hypervisor. The buffer is +used as an array of 16 I/O request slots with each I/O request slot being 256 +bytes. This array is indexed by vCPU ID. + +2. I/O clients +-------------- + +An I/O client is responsible for handling User VM I/O requests whose accessed +GPA falls in a certain range. Multiple I/O clients can be associated with each +User VM. There is a special client associated with each User VM, called the +default client, that handles all I/O requests that do not fit into the range of +any other clients. The ACRN userspace acts as the default client for each User +VM. + +Below illustration shows the relationship between I/O requests shared buffer, +I/O requests and I/O clients. + +:: + + +------------------------------------------------------+ + | Service VM | + |+--------------------------------------------------+ | + || +----------------------------------------+ | | + || | shared page ACRN userspace | | | + || | +-----------------+ +------------+ | | | + || +----+->| acrn_io_request |<-+ default | | | | + || | | | +-----------------+ | I/O client | | | | + || | | | | ... | +------------+ | | | + || | | | +-----------------+ | | | + || | +-|--------------------------------------+ | | + ||---|----|-----------------------------------------| | + || | | kernel | | + || | | +----------------------+ | | + || | | | +-------------+ HSM | | | + || | +--------------+ | | | | + || | | | I/O clients | | | | + || | | | | | | | + || | | +-------------+ | | | + || | +----------------------+ | | + |+---|----------------------------------------------+ | + +----|-------------------------------------------------+ + | + +----|-------------------------------------------------+ + | +-+-----------+ | + | | I/O handler | ACRN Hypervisor | + | +-------------+ | + +------------------------------------------------------+ + +3. I/O request state transition +------------------------------- + +The state transitions of an ACRN I/O request are as follows. + +:: + + FREE -> PENDING -> PROCESSING -> COMPLETE -> FREE -> ... + +- FREE: this I/O request slot is empty +- PENDING: a valid I/O request is pending in this slot +- PROCESSING: the I/O request is being processed +- COMPLETE: the I/O request has been processed + +An I/O request in COMPLETE or FREE state is owned by the hypervisor. HSM and +ACRN userspace are in charge of processing the others. + +4. Processing flow of I/O requests +---------------------------------- + +a. The I/O handler of the hypervisor will fill an I/O request with PENDING + state when a trapped I/O access happens in a User VM. +b. The hypervisor makes an upcall, which is a notification interrupt, to + the Service VM. +c. The upcall handler schedules a worker to dispatch I/O requests. +d. The worker looks for the PENDING I/O requests, assigns them to different + registered clients based on the address of the I/O accesses, updates + their state to PROCESSING, and notifies the corresponding client to handle. +e. The notified client handles the assigned I/O requests. +f. The HSM updates I/O requests states to COMPLETE and notifies the hypervisor + of the completion via hypercalls. diff --git a/Documentation/virt/index.rst b/Documentation/virt/index.rst index 350f5c869b56..edea7fea95a8 100644 --- a/Documentation/virt/index.rst +++ b/Documentation/virt/index.rst @@ -12,6 +12,7 @@ Linux Virtualization Support paravirt_ops guest-halt-polling ne_overview + acrn/index .. only:: html and subproject diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 45fd862ac128..aed52b0fc16e 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -4519,6 +4519,7 @@ KVM_GET_SUPPORTED_CPUID ioctl because some of them intersect with KVM feature leaves (0x40000000, 0x40000001). Currently, the following list of CPUID leaves are returned: + - HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS - HYPERV_CPUID_INTERFACE - HYPERV_CPUID_VERSION @@ -4543,6 +4544,7 @@ userspace should not expect to get any particular value there. Note, vcpu version of KVM_GET_SUPPORTED_HV_CPUID is currently deprecated. Unlike system ioctl which exposes all supported feature bits unconditionally, vcpu version has the following quirks: + - HYPERV_CPUID_NESTED_FEATURES leaf and HV_X64_ENLIGHTENED_VMCS_RECOMMENDED feature bit are only exposed when Enlightened VMCS was previously enabled on the corresponding vCPU (KVM_CAP_HYPERV_ENLIGHTENED_VMCS). diff --git a/Documentation/vm/arch_pgtable_helpers.rst b/Documentation/vm/arch_pgtable_helpers.rst index f3591ee3aaa8..552567d863b8 100644 --- a/Documentation/vm/arch_pgtable_helpers.rst +++ b/Documentation/vm/arch_pgtable_helpers.rst @@ -50,7 +50,7 @@ PTE Page Table Helpers +---------------------------+--------------------------------------------------+ | pte_mkwrite | Creates a writable PTE | +---------------------------+--------------------------------------------------+ -| pte_mkwrprotect | Creates a write protected PTE | +| pte_wrprotect | Creates a write protected PTE | +---------------------------+--------------------------------------------------+ | pte_mkspecial | Creates a special PTE | +---------------------------+--------------------------------------------------+ @@ -120,7 +120,7 @@ PMD Page Table Helpers +---------------------------+--------------------------------------------------+ | pmd_mkwrite | Creates a writable PMD | +---------------------------+--------------------------------------------------+ -| pmd_mkwrprotect | Creates a write protected PMD | +| pmd_wrprotect | Creates a write protected PMD | +---------------------------+--------------------------------------------------+ | pmd_mkspecial | Creates a special PMD | +---------------------------+--------------------------------------------------+ @@ -186,7 +186,7 @@ PUD Page Table Helpers +---------------------------+--------------------------------------------------+ | pud_mkwrite | Creates a writable PUD | +---------------------------+--------------------------------------------------+ -| pud_mkwrprotect | Creates a write protected PUD | +| pud_wrprotect | Creates a write protected PUD | +---------------------------+--------------------------------------------------+ | pud_mkdevmap | Creates a ZONE_DEVICE mapped PUD | +---------------------------+--------------------------------------------------+ @@ -224,7 +224,7 @@ HugeTLB Page Table Helpers +---------------------------+--------------------------------------------------+ | huge_pte_mkwrite | Creates a writable HugeTLB | +---------------------------+--------------------------------------------------+ -| huge_pte_mkwrprotect | Creates a write protected HugeTLB | +| huge_pte_wrprotect | Creates a write protected HugeTLB | +---------------------------+--------------------------------------------------+ | huge_ptep_get_and_clear | Clears a HugeTLB | +---------------------------+--------------------------------------------------+ diff --git a/Documentation/vm/split_page_table_lock.rst b/Documentation/vm/split_page_table_lock.rst index ff51f4a5494d..c08919662704 100644 --- a/Documentation/vm/split_page_table_lock.rst +++ b/Documentation/vm/split_page_table_lock.rst @@ -32,7 +32,7 @@ There are helpers to lock/unlock a table and other accessor functions: Split page table lock for PTE tables is enabled compile-time if CONFIG_SPLIT_PTLOCK_CPUS (usually 4) is less or equal to NR_CPUS. -If split lock is disabled, all tables guaded by mm->page_table_lock. +If split lock is disabled, all tables are guarded by mm->page_table_lock. Split page table lock for PMD tables is enabled, if it's enabled for PTE tables and the architecture supports it (see below). diff --git a/Documentation/x86/boot.rst b/Documentation/x86/boot.rst index abb9fc164657..fc844913dece 100644 --- a/Documentation/x86/boot.rst +++ b/Documentation/x86/boot.rst @@ -851,7 +851,7 @@ Protocol: 2.09+ struct setup_data { __u64 next = 0 or ; __u32 type = SETUP_INDIRECT; - __u32 len = sizeof(setup_data); + __u32 len = sizeof(setup_indirect); __u8 data[sizeof(setup_indirect)] = struct setup_indirect { __u32 type = SETUP_INDIRECT | SETUP_E820_EXT; __u32 reserved = 0; diff --git a/MAINTAINERS b/MAINTAINERS index b9d55c84fc12..d92f85ca831d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -261,6 +261,8 @@ ABI/API L: linux-api@vger.kernel.org F: include/linux/syscalls.h F: kernel/sys_ni.c +F: include/uapi/ +F: arch/*/include/uapi/ ABIT UGURU 1,2 HARDWARE MONITOR DRIVER M: Hans de Goede @@ -436,6 +438,15 @@ S: Orphan F: drivers/platform/x86/wmi.c F: include/uapi/linux/wmi.h +ACRN HYPERVISOR SERVICE MODULE +M: Shuo Liu +L: acrn-dev@lists.projectacrn.org (subscribers-only) +S: Supported +W: https://projectacrn.org +F: Documentation/virt/acrn/ +F: drivers/virt/acrn/ +F: include/uapi/linux/acrn.h + AD1889 ALSA SOUND DRIVER L: linux-parisc@vger.kernel.org S: Maintained @@ -1017,7 +1028,7 @@ F: Documentation/devicetree/bindings/mux/adi,adgs1408.txt F: drivers/mux/adgs1408.c ANALOG DEVICES INC ADIN DRIVER -M: Alexandru Ardelean +M: Michael Hennerich L: netdev@vger.kernel.org S: Supported W: http://ez.analog.com/community/linux-device-drivers @@ -1025,7 +1036,7 @@ F: Documentation/devicetree/bindings/net/adi,adin.yaml F: drivers/net/phy/adin.c ANALOG DEVICES INC ADIS DRIVER LIBRARY -M: Alexandru Ardelean +M: Nuno Sa L: linux-iio@vger.kernel.org S: Supported F: drivers/iio/imu/adis.c @@ -2126,10 +2137,13 @@ L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained W: http://linux-chenxing.org/ F: Documentation/devicetree/bindings/arm/mstar/* +F: Documentation/devicetree/bindings/clock/mstar,msc313-mpll.yaml F: Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml F: arch/arm/boot/dts/mstar-* F: arch/arm/mach-mstar/ +F: drivers/clk/mstar/ F: drivers/gpio/gpio-msc313.c +F: include/dt-bindings/clock/mstar-* F: include/dt-bindings/gpio/msc313-gpio.h ARM/NEC MOBILEPRO 900/c MACHINE SUPPORT @@ -2566,7 +2580,7 @@ L: linux-kernel@vger.kernel.org S: Maintained F: drivers/clk/keystone/ -ARM/TEXAS INSTRUMENT KEYSTONE ClOCKSOURCE +ARM/TEXAS INSTRUMENT KEYSTONE CLOCKSOURCE M: Santosh Shilimkar L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) L: linux-kernel@vger.kernel.org @@ -2606,10 +2620,12 @@ S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/iwamatsu/linux-visconti.git F: Documentation/devicetree/bindings/arm/toshiba.yaml F: Documentation/devicetree/bindings/net/toshiba,visconti-dwmac.yaml +F: Documentation/devicetree/bindings/gpio/toshiba,gpio-visconti.yaml F: Documentation/devicetree/bindings/pinctrl/toshiba,tmpv7700-pinctrl.yaml F: Documentation/devicetree/bindings/watchdog/toshiba,visconti-wdt.yaml F: arch/arm64/boot/dts/toshiba/ F: drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c +F: drivers/gpio/gpio-visconti.c F: drivers/pinctrl/visconti/ F: drivers/watchdog/visconti_wdt.c N: visconti @@ -2775,6 +2791,15 @@ F: Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2xxx-scu-ic. F: drivers/irqchip/irq-aspeed-scu-ic.c F: include/dt-bindings/interrupt-controller/aspeed-scu-ic.h +ASPEED SD/MMC DRIVER +M: Andrew Jeffery +L: linux-aspeed@lists.ozlabs.org (moderated for non-subscribers) +L: openbmc@lists.ozlabs.org (moderated for non-subscribers) +L: linux-mmc@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml +F: drivers/mmc/host/sdhci-of-aspeed* + ASPEED VIDEO ENGINE DRIVER M: Eddie James L: linux-media@vger.kernel.org @@ -2814,9 +2839,7 @@ S: Odd fixes W: http://sourceforge.net/projects/xscaleiop F: Documentation/crypto/async-tx-api.rst F: crypto/async_tx/ -F: drivers/dma/ F: include/linux/async_tx.h -F: include/linux/dmaengine.h AT24 EEPROM DRIVER M: Bartosz Golaszewski @@ -2961,7 +2984,7 @@ F: include/uapi/linux/audit.h F: kernel/audit* AUXILIARY DISPLAY DRIVERS -M: Miguel Ojeda Sandonis +M: Miguel Ojeda S: Maintained F: drivers/auxdisplay/ F: include/linux/cfag12864b.h @@ -3719,6 +3742,13 @@ L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/broadcom/tg3.* +BROADCOM VK DRIVER +M: Scott Branden +L: bcm-kernel-feedback-list@broadcom.com +S: Supported +F: drivers/misc/bcm-vk/ +F: include/uapi/linux/misc/bcm_vk.h + BROCADE BFA FC SCSI DRIVER M: Anil Gurumurthy M: Sudarsana Kalluru @@ -3825,6 +3855,29 @@ W: https://github.com/Cascoda/ca8210-linux.git F: Documentation/devicetree/bindings/net/ieee802154/ca8210.txt F: drivers/net/ieee802154/ca8210.c +CANAAN/KENDRYTE K210 SOC FPIOA DRIVER +M: Damien Le Moal +L: linux-riscv@lists.infradead.org +L: linux-gpio@vger.kernel.org (pinctrl driver) +F: Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml +F: drivers/pinctrl/pinctrl-k210.c + +CANAAN/KENDRYTE K210 SOC RESET CONTROLLER DRIVER +M: Damien Le Moal +L: linux-kernel@vger.kernel.org +L: linux-riscv@lists.infradead.org +S: Maintained +F: Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml +F: drivers/reset/reset-k210.c + +CANAAN/KENDRYTE K210 SOC SYSTEM CONTROLLER DRIVER +M: Damien Le Moal +L: linux-riscv@lists.infradead.org +S: Maintained +F: Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml +F: drivers/soc/canaan/ +F: include/soc/canaan/ + CACHEFILES: FS-CACHE BACKEND FOR CACHING ON MOUNTED FILESYSTEMS M: David Howells L: linux-cachefs@redhat.com (moderated for non-subscribers) @@ -4100,13 +4153,13 @@ F: scripts/extract-cert.c F: scripts/sign-file.c CFAG12864B LCD DRIVER -M: Miguel Ojeda Sandonis +M: Miguel Ojeda S: Maintained F: drivers/auxdisplay/cfag12864b.c F: include/linux/cfag12864b.h CFAG12864BFB LCD FRAMEBUFFER DRIVER -M: Miguel Ojeda Sandonis +M: Miguel Ojeda S: Maintained F: drivers/auxdisplay/cfag12864bfb.c F: include/linux/cfag12864b.h @@ -4276,7 +4329,7 @@ S: Supported F: drivers/infiniband/hw/usnic/ CLANG-FORMAT FILE -M: Miguel Ojeda +M: Miguel Ojeda S: Maintained F: .clang-format @@ -4291,8 +4344,6 @@ C: irc://chat.freenode.net/clangbuiltlinux F: Documentation/kbuild/llvm.rst F: include/linux/compiler-clang.h F: scripts/clang-tools/ -F: scripts/clang-version.sh -F: scripts/lld-version.sh K: \b(?i:clang|llvm)\b CLEANCACHE API @@ -4418,10 +4469,21 @@ S: Maintained F: drivers/platform/x86/compal-laptop.c COMPILER ATTRIBUTES -M: Miguel Ojeda +M: Miguel Ojeda S: Maintained F: include/linux/compiler_attributes.h +COMPUTE EXPRESS LINK (CXL) +M: Alison Schofield +M: Vishal Verma +M: Ira Weiny +M: Ben Widawsky +M: Dan Williams +L: linux-cxl@vger.kernel.org +S: Maintained +F: drivers/cxl/ +F: include/uapi/linux/cxl_mem.h + CONEXANT ACCESSRUNNER USB DRIVER L: accessrunner-general@lists.sourceforge.net S: Orphan @@ -4946,17 +5008,17 @@ M: Matthew Garrett M: Pali Rohár L: platform-driver-x86@vger.kernel.org S: Maintained -F: drivers/platform/x86/dell-laptop.c +F: drivers/platform/x86/dell/dell-laptop.c DELL LAPTOP FREEFALL DRIVER M: Pali Rohár S: Maintained -F: drivers/platform/x86/dell-smo8800.c +F: drivers/platform/x86/dell/dell-smo8800.c DELL LAPTOP RBTN DRIVER M: Pali Rohár S: Maintained -F: drivers/platform/x86/dell-rbtn.* +F: drivers/platform/x86/dell/dell-rbtn.* DELL LAPTOP SMM DRIVER M: Pali Rohár @@ -4968,26 +5030,26 @@ DELL REMOTE BIOS UPDATE DRIVER M: Stuart Hayes L: platform-driver-x86@vger.kernel.org S: Maintained -F: drivers/platform/x86/dell_rbu.c +F: drivers/platform/x86/dell/dell_rbu.c DELL SMBIOS DRIVER M: Pali Rohár M: Mario Limonciello L: platform-driver-x86@vger.kernel.org S: Maintained -F: drivers/platform/x86/dell-smbios.* +F: drivers/platform/x86/dell/dell-smbios.* DELL SMBIOS SMM DRIVER M: Mario Limonciello L: platform-driver-x86@vger.kernel.org S: Maintained -F: drivers/platform/x86/dell-smbios-smm.c +F: drivers/platform/x86/dell/dell-smbios-smm.c DELL SMBIOS WMI DRIVER M: Mario Limonciello L: platform-driver-x86@vger.kernel.org S: Maintained -F: drivers/platform/x86/dell-smbios-wmi.c +F: drivers/platform/x86/dell/dell-smbios-wmi.c F: tools/wmi/dell-smbios-example.c DELL SYSTEMS MANAGEMENT BASE DRIVER (dcdbas) @@ -4995,12 +5057,12 @@ M: Stuart Hayes L: platform-driver-x86@vger.kernel.org S: Maintained F: Documentation/driver-api/dcdbas.rst -F: drivers/platform/x86/dcdbas.* +F: drivers/platform/x86/dell/dcdbas.* DELL WMI DESCRIPTOR DRIVER M: Mario Limonciello S: Maintained -F: drivers/platform/x86/dell-wmi-descriptor.c +F: drivers/platform/x86/dell/dell-wmi-descriptor.c DELL WMI SYSMAN DRIVER M: Divya Bharathi @@ -5009,13 +5071,13 @@ M: Prasanth Ksr L: platform-driver-x86@vger.kernel.org S: Maintained F: Documentation/ABI/testing/sysfs-class-firmware-attributes -F: drivers/platform/x86/dell-wmi-sysman/ +F: drivers/platform/x86/dell/dell-wmi-sysman/ DELL WMI NOTIFICATIONS DRIVER M: Matthew Garrett M: Pali Rohár S: Maintained -F: drivers/platform/x86/dell-wmi.c +F: drivers/platform/x86/dell/dell-wmi.c DELTA ST MEDIA DRIVER M: Hugues Fruchet @@ -5257,6 +5319,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git F: Documentation/devicetree/bindings/dma/ F: Documentation/driver-api/dmaengine/ F: drivers/dma/ +F: include/linux/dma/ F: include/linux/dmaengine.h F: include/linux/of_dma.h @@ -5760,6 +5823,7 @@ F: drivers/gpu/drm/vboxvideo/ DRM DRIVER FOR VMWARE VIRTUAL GPU M: "VMware Graphics" M: Roland Scheidegger +M: Zack Rusin L: dri-devel@lists.freedesktop.org S: Supported T: git git://people.freedesktop.org/~sroland/linux @@ -5961,8 +6025,8 @@ F: Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml F: drivers/gpu/drm/stm DRM DRIVERS FOR TI KEYSTONE -M: Jyri Sarha -M: Tomi Valkeinen +M: Jyri Sarha +M: Tomi Valkeinen L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc @@ -5972,15 +6036,15 @@ F: Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml F: drivers/gpu/drm/tidss/ DRM DRIVERS FOR TI LCDC -M: Jyri Sarha -R: Tomi Valkeinen +M: Jyri Sarha +R: Tomi Valkeinen L: dri-devel@lists.freedesktop.org S: Maintained F: Documentation/devicetree/bindings/display/tilcdc/ F: drivers/gpu/drm/tilcdc/ DRM DRIVERS FOR TI OMAP -M: Tomi Valkeinen +M: Tomi Valkeinen L: dri-devel@lists.freedesktop.org S: Maintained F: Documentation/devicetree/bindings/display/ti/ @@ -6925,9 +6989,10 @@ M: Wu Hao R: Tom Rix L: linux-fpga@vger.kernel.org S: Maintained -F: Documentation/ABI/testing/sysfs-bus-dfl +F: Documentation/ABI/testing/sysfs-bus-dfl* F: Documentation/fpga/dfl.rst F: drivers/fpga/dfl* +F: include/linux/dfl.h F: include/uapi/linux/fpga-dfl.h FPGA MANAGER FRAMEWORK @@ -7348,13 +7413,6 @@ M: Kieran Bingham S: Supported F: scripts/gdb/ -GDT SCSI DISK ARRAY CONTROLLER DRIVER -M: Achim Leubner -L: linux-scsi@vger.kernel.org -S: Supported -W: http://www.icp-vortex.com/ -F: drivers/scsi/gdt* - GEMTEK FM RADIO RECEIVER DRIVER M: Hans Verkuil L: linux-media@vger.kernel.org @@ -7900,6 +7958,12 @@ F: drivers/hid/ F: include/linux/hid* F: include/uapi/linux/hid* +HID PLAYSTATION DRIVER +M: Roderick Colenbrander +L: linux-input@vger.kernel.org +S: Supported +F: drivers/hid/hid-playstation.c + HID SENSOR HUB DRIVERS M: Jiri Kosina M: Jonathan Cameron @@ -8131,7 +8195,7 @@ F: net/hsr/ HT16K33 LED CONTROLLER DRIVER M: Robin van der Gracht S: Maintained -F: Documentation/devicetree/bindings/display/ht16k33.txt +F: Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml F: drivers/auxdisplay/ht16k33.c HTCPEN TOUCHSCREEN DRIVER @@ -8849,7 +8913,6 @@ F: drivers/mfd/intel_pmc_bxt.c F: include/linux/mfd/intel_pmc_bxt.h INTEL C600 SERIES SAS CONTROLLER DRIVER -M: Intel SCU Linux support M: Artur Paszkiewicz L: linux-scsi@vger.kernel.org S: Supported @@ -8907,7 +8970,6 @@ L: linux-gpio@vger.kernel.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git F: drivers/gpio/gpio-ich.c -F: drivers/gpio/gpio-intel-mid.c F: drivers/gpio/gpio-merrifield.c F: drivers/gpio/gpio-ml-ioh.c F: drivers/gpio/gpio-pch.c @@ -9031,6 +9093,17 @@ F: drivers/crypto/keembay/keembay-ocs-aes-core.c F: drivers/crypto/keembay/ocs-aes.c F: drivers/crypto/keembay/ocs-aes.h +INTEL KEEM BAY OCS HCU CRYPTO DRIVER +M: Daniele Alessandrelli +M: Declan Murphy +S: Maintained +F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml +F: drivers/crypto/keembay/Kconfig +F: drivers/crypto/keembay/Makefile +F: drivers/crypto/keembay/keembay-ocs-hcu-core.c +F: drivers/crypto/keembay/ocs-hcu.c +F: drivers/crypto/keembay/ocs-hcu.h + INTEL MANAGEMENT ENGINE (mei) M: Tomas Winkler L: linux-kernel@vger.kernel.org @@ -9068,14 +9141,11 @@ M: Andy Shevchenko S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git F: drivers/gpio/gpio-*cove.c -F: drivers/gpio/gpio-msic.c INTEL PMIC MULTIFUNCTION DEVICE DRIVERS M: Andy Shevchenko S: Maintained -F: drivers/mfd/intel_msic.c F: drivers/mfd/intel_soc_pmic* -F: include/linux/mfd/intel_msic.h F: include/linux/mfd/intel_soc_pmic* INTEL PMT DRIVER @@ -9822,6 +9892,18 @@ F: include/linux/keyctl.h F: include/uapi/linux/keyctl.h F: security/keys/ +KFENCE +M: Alexander Potapenko +M: Marco Elver +R: Dmitry Vyukov +L: kasan-dev@googlegroups.com +S: Maintained +F: Documentation/dev-tools/kfence.rst +F: arch/*/include/asm/kfence.h +F: include/linux/kfence.h +F: lib/Kconfig.kfence +F: mm/kfence/ + KFIFO M: Stefani Seibold S: Maintained @@ -9882,7 +9964,7 @@ F: include/linux/kprobes.h F: kernel/kprobes.c KS0108 LCD CONTROLLER DRIVER -M: Miguel Ojeda Sandonis +M: Miguel Ojeda S: Maintained F: Documentation/admin-guide/auxdisplay/ks0108.rst F: drivers/auxdisplay/ks0108.c @@ -11159,6 +11241,15 @@ S: Maintained F: Documentation/devicetree/bindings/i2c/i2c-mt65xx.txt F: drivers/i2c/busses/i2c-mt65xx.c +MEDIATEK IOMMU DRIVER +M: Yong Wu +L: iommu@lists.linux-foundation.org +L: linux-mediatek@lists.infradead.org (moderated for non-subscribers) +S: Supported +F: Documentation/devicetree/bindings/iommu/mediatek* +F: drivers/iommu/mtk_iommu* +F: include/dt-bindings/memory/mt*-port.h + MEDIATEK JPEG DRIVER M: Rick Chang M: Bin Liu @@ -11583,7 +11674,6 @@ F: drivers/dma/at_hdmac.c F: drivers/dma/at_hdmac_regs.h F: drivers/dma/at_xdmac.c F: include/dt-bindings/dma/at91.h -F: include/linux/platform_data/dma-atmel.h MICROCHIP AT91 SERIAL DRIVER M: Richard Genoud @@ -11669,9 +11759,9 @@ F: drivers/video/fbdev/atmel_lcdfb.c F: include/video/atmel_lcdc.h MICROCHIP MCP16502 PMIC DRIVER -M: Andrei Stefanescu +M: Claudiu Beznea L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) -S: Maintained +S: Supported F: Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt F: drivers/regulator/mcp16502.c @@ -11786,12 +11876,31 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git F: drivers/platform/surface/ +MICROSOFT SURFACE HOT-PLUG DRIVER +M: Maximilian Luz +L: platform-driver-x86@vger.kernel.org +S: Maintained +F: drivers/platform/surface/surface_hotplug.c + MICROSOFT SURFACE PRO 3 BUTTON DRIVER M: Chen Yu L: platform-driver-x86@vger.kernel.org S: Supported F: drivers/platform/surface/surfacepro3_button.c +MICROSOFT SURFACE SYSTEM AGGREGATOR SUBSYSTEM +M: Maximilian Luz +S: Maintained +W: https://github.com/linux-surface/surface-aggregator-module +C: irc://chat.freenode.net/##linux-surface +F: Documentation/driver-api/surface_aggregator/ +F: drivers/platform/surface/aggregator/ +F: drivers/platform/surface/surface_acpi_notify.c +F: drivers/platform/surface/surface_aggregator_cdev.c +F: include/linux/surface_acpi_notify.h +F: include/linux/surface_aggregator/ +F: include/uapi/linux/surface_aggregator/ + MICROTEK X6 SCANNER M: Oliver Neukum S: Maintained @@ -12736,6 +12845,13 @@ F: drivers/iio/gyro/fxas21002c_core.c F: drivers/iio/gyro/fxas21002c_i2c.c F: drivers/iio/gyro/fxas21002c_spi.c +NXP i.MX CLOCK DRIVERS +M: Abel Vesa +L: linux-clk@vger.kernel.org +L: linux-imx@nxp.com +S: Maintained +F: drivers/clk/imx/ + NXP i.MX 8MQ DCSS DRIVER M: Laurentiu Palcu R: Lucas Stach @@ -13809,6 +13925,13 @@ S: Supported F: Documentation/devicetree/bindings/pci/mediatek* F: drivers/pci/controller/*mediatek* +PCIE DRIVER FOR MICROCHIP +M: Daire McNamara +L: linux-pci@vger.kernel.org +S: Supported +F: Documentation/devicetree/bindings/pci/microchip* +F: drivers/pci/controller/*microchip* + PCIE DRIVER FOR QUALCOMM MSM M: Stanimir Varbanov L: linux-pci@vger.kernel.org @@ -14620,9 +14743,11 @@ M: Stuart Yoder M: Laurentiu Tudor L: linux-kernel@vger.kernel.org S: Maintained +F: Documentation/ABI/stable/sysfs-bus-fsl-mc F: Documentation/devicetree/bindings/misc/fsl,qoriq-mc.txt F: Documentation/networking/device_drivers/ethernet/freescale/dpaa2/overview.rst F: drivers/bus/fsl-mc/ +F: include/uapi/linux/fsl_mc.h QT1010 MEDIA DRIVER M: Antti Palosaari @@ -15682,7 +15807,6 @@ F: drivers/media/i2c/s5k5baf.c SAMSUNG S5P Security SubSystem (SSS) DRIVER M: Krzysztof Kozlowski M: Vladimir Zapolskiy -M: Kamil Konieczny L: linux-crypto@vger.kernel.org L: linux-samsung-soc@vger.kernel.org S: Maintained @@ -16230,12 +16354,13 @@ S: Maintained F: Documentation/fb/sm712fb.rst F: drivers/video/fbdev/sm712* -SIMPLE FIRMWARE INTERFACE (SFI) -S: Obsolete -W: http://simplefirmware.org/ -F: arch/x86/platform/sfi/ -F: drivers/sfi/ -F: include/linux/sfi*.h +SILVACO I3C DUAL-ROLE MASTER +M: Miquel Raynal +M: Conor Culhane +L: linux-i3c@lists.infradead.org +S: Maintained +F: Documentation/devicetree/bindings/i3c/silvaco,i3c-master.yaml +F: drivers/i3c/master/svc-i3c-master.c SIMPLEFB FB DRIVER M: Hans de Goede @@ -16658,6 +16783,7 @@ R: Pierre-Louis Bossart R: Sanyog Kale L: alsa-devel@alsa-project.org (moderated for non-subscribers) S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git F: Documentation/driver-api/soundwire/ F: drivers/soundwire/ F: include/linux/soundwire/ @@ -17500,6 +17626,14 @@ M: Laxman Dewangan S: Supported F: drivers/spi/spi-tegra* +TEGRA QUAD SPI DRIVER +M: Thierry Reding +M: Jonathan Hunter +M: Sowjanya Komatineni +L: linux-tegra@vger.kernel.org +S: Maintained +F: drivers/spi/spi-tegra210-quad.c + TEGRA VIDEO DRIVER M: Thierry Reding M: Jonathan Hunter @@ -17592,6 +17726,15 @@ F: include/dt-bindings/soc/ti,sci_pm_domain.h F: include/linux/soc/ti/ti_sci_inta_msi.h F: include/linux/soc/ti/ti_sci_protocol.h +TEXAS INSTRUMENTS TPS23861 PoE PSE DRIVER +M: Robert Marko +M: Luka Perkov +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml +F: Documentation/hwmon/tps23861.rst +F: drivers/hwmon/tps23861.c + THANKO'S RAREMONO AM/FM/SW RADIO RECEIVER USB DRIVER M: Hans Verkuil L: linux-media@vger.kernel.org @@ -17645,7 +17788,7 @@ F: drivers/thermal/gov_power_allocator.c F: include/trace/events/thermal_power_allocator.h THINKPAD ACPI EXTRAS DRIVER -M: Henrique de Moraes Holschuh +M: Henrique de Moraes Holschuh L: ibm-acpi-devel@lists.sourceforge.net L: platform-driver-x86@vger.kernel.org S: Maintained diff --git a/Makefile b/Makefile index 95caa81be8fb..f9b54da2fca0 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 5 -PATCHLEVEL = 11 +PATCHLEVEL = 12 SUBLEVEL = 0 -EXTRAVERSION = -NAME = 💕 Valentine's Day Edition 💕 +EXTRAVERSION = -rc1 +NAME = Frozen Wasteland # *DOCUMENTATION* # To see a list of typical targets execute "make help" @@ -96,10 +96,41 @@ endif ifneq ($(findstring s,$(filter-out --%,$(MAKEFLAGS))),) quiet=silent_ + KBUILD_VERBOSE = 0 endif export quiet Q KBUILD_VERBOSE +# Call a source code checker (by default, "sparse") as part of the +# C compilation. +# +# Use 'make C=1' to enable checking of only re-compiled files. +# Use 'make C=2' to enable checking of *all* source files, regardless +# of whether they are re-compiled or not. +# +# See the file "Documentation/dev-tools/sparse.rst" for more details, +# including where to get the "sparse" utility. + +ifeq ("$(origin C)", "command line") + KBUILD_CHECKSRC = $(C) +endif +ifndef KBUILD_CHECKSRC + KBUILD_CHECKSRC = 0 +endif + +export KBUILD_CHECKSRC + +# Use make M=dir or set the environment variable KBUILD_EXTMOD to specify the +# directory of external module to build. Setting M= takes precedence. +ifeq ("$(origin M)", "command line") + KBUILD_EXTMOD := $(M) +endif + +$(if $(word 2, $(KBUILD_EXTMOD)), \ + $(error building multiple external modules is not supported)) + +export KBUILD_EXTMOD + # Kbuild will save output files in the current working directory. # This does not need to match to the root of the kernel source tree. # @@ -145,7 +176,8 @@ else need-sub-make := 1 endif -abs_srctree := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +this-makefile := $(lastword $(MAKEFILE_LIST)) +abs_srctree := $(realpath $(dir $(this-makefile))) ifneq ($(words $(subst :, ,$(abs_srctree))), 1) $(error source directory cannot contain spaces or colons) @@ -160,8 +192,6 @@ MAKEFLAGS += --include-dir=$(abs_srctree) need-sub-make := 1 endif -this-makefile := $(lastword $(MAKEFILE_LIST)) - ifneq ($(filter 3.%,$(MAKE_VERSION)),) # 'MAKEFLAGS += -rR' does not immediately become effective for GNU Make 3.x # We need to invoke sub-make to avoid implicit rules in the top Makefile. @@ -195,36 +225,6 @@ ifeq ($(need-sub-make),) # so that IDEs/editors are able to understand relative filenames. MAKEFLAGS += --no-print-directory -# Call a source code checker (by default, "sparse") as part of the -# C compilation. -# -# Use 'make C=1' to enable checking of only re-compiled files. -# Use 'make C=2' to enable checking of *all* source files, regardless -# of whether they are re-compiled or not. -# -# See the file "Documentation/dev-tools/sparse.rst" for more details, -# including where to get the "sparse" utility. - -ifeq ("$(origin C)", "command line") - KBUILD_CHECKSRC = $(C) -endif -ifndef KBUILD_CHECKSRC - KBUILD_CHECKSRC = 0 -endif - -# Use make M=dir or set the environment variable KBUILD_EXTMOD to specify the -# directory of external module to build. Setting M= takes precedence. -ifeq ("$(origin M)", "command line") - KBUILD_EXTMOD := $(M) -endif - -$(if $(word 2, $(KBUILD_EXTMOD)), \ - $(error building multiple external modules is not supported)) - -export KBUILD_CHECKSRC KBUILD_EXTMOD - -extmod-prefix = $(if $(KBUILD_EXTMOD),$(KBUILD_EXTMOD)/) - ifeq ($(abs_srctree),$(abs_objtree)) # building in the source tree srctree := . @@ -257,7 +257,6 @@ export building_out_of_srctree srctree objtree VPATH # of make so .config is not included in this case either (for *config). version_h := include/generated/uapi/linux/version.h -old_version_h := include/linux/version.h clean-targets := %clean mrproper cleandocs no-dot-config-targets := $(clean-targets) \ @@ -558,7 +557,13 @@ ifdef building_out_of_srctree { echo "# this is build directory, ignore it"; echo "*"; } > .gitignore endif -ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),) +# The expansion should be delayed until arch/$(SRCARCH)/Makefile is included. +# Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile. +# CC_VERSION_TEXT is referenced from Kconfig (so it needs export), +# and from include/config/auto.conf.cmd to detect the compiler upgrade. +CC_VERSION_TEXT = $(shell $(CC) --version 2>/dev/null | head -n 1 | sed 's/\#//g') + +ifneq ($(findstring clang,$(CC_VERSION_TEXT)),) ifneq ($(CROSS_COMPILE),) CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) GCC_TOOLCHAIN_DIR := $(dir $(shell which $(CROSS_COMPILE)elfedit)) @@ -577,12 +582,6 @@ KBUILD_AFLAGS += $(CLANG_FLAGS) export CLANG_FLAGS endif -# The expansion should be delayed until arch/$(SRCARCH)/Makefile is included. -# Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile. -# CC_VERSION_TEXT is referenced from Kconfig (so it needs export), -# and from include/config/auto.conf.cmd to detect the compiler upgrade. -CC_VERSION_TEXT = $(shell $(CC) --version 2>/dev/null | head -n 1) - ifdef config-build # =========================================================================== # *config targets only - make sure prerequisites are updated, and descend @@ -832,8 +831,10 @@ ifneq ($(LLVM_IAS),1) KBUILD_AFLAGS += -Wa,-gdwarf-2 endif -ifdef CONFIG_DEBUG_INFO_DWARF4 -DEBUG_CFLAGS += -gdwarf-4 +ifndef CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT +dwarf-version-$(CONFIG_DEBUG_INFO_DWARF4) := 4 +dwarf-version-$(CONFIG_DEBUG_INFO_DWARF5) := 5 +DEBUG_CFLAGS += -gdwarf-$(dwarf-version-y) endif ifdef CONFIG_DEBUG_INFO_REDUCED @@ -853,12 +854,8 @@ KBUILD_CFLAGS += $(DEBUG_CFLAGS) export DEBUG_CFLAGS ifdef CONFIG_FUNCTION_TRACER -ifdef CONFIG_FTRACE_MCOUNT_RECORD - # gcc 5 supports generating the mcount tables directly - ifeq ($(call cc-option-yn,-mrecord-mcount),y) - CC_FLAGS_FTRACE += -mrecord-mcount - export CC_USING_RECORD_MCOUNT := 1 - endif +ifdef CONFIG_FTRACE_MCOUNT_USE_CC + CC_FLAGS_FTRACE += -mrecord-mcount ifdef CONFIG_HAVE_NOP_MCOUNT ifeq ($(call cc-option-yn, -mnop-mcount),y) CC_FLAGS_FTRACE += -mnop-mcount @@ -866,6 +863,15 @@ ifdef CONFIG_FTRACE_MCOUNT_RECORD endif endif endif +ifdef CONFIG_FTRACE_MCOUNT_USE_OBJTOOL + CC_FLAGS_USING += -DCC_USING_NOP_MCOUNT +endif +ifdef CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT + ifdef CONFIG_HAVE_C_RECORDMCOUNT + BUILD_C_RECORDMCOUNT := y + export BUILD_C_RECORDMCOUNT + endif +endif ifdef CONFIG_HAVE_FENTRY ifeq ($(call cc-option-yn, -mfentry),y) CC_FLAGS_FTRACE += -mfentry @@ -875,12 +881,6 @@ endif export CC_FLAGS_FTRACE KBUILD_CFLAGS += $(CC_FLAGS_FTRACE) $(CC_FLAGS_USING) KBUILD_AFLAGS += $(CC_FLAGS_USING) -ifdef CONFIG_DYNAMIC_FTRACE - ifdef CONFIG_HAVE_C_RECORDMCOUNT - BUILD_C_RECORDMCOUNT := y - export BUILD_C_RECORDMCOUNT - endif -endif endif # We trigger additional mismatches with less inlining @@ -899,6 +899,25 @@ KBUILD_CFLAGS += $(CC_FLAGS_SCS) export CC_FLAGS_SCS endif +ifdef CONFIG_LTO_CLANG +ifdef CONFIG_LTO_CLANG_THIN +CC_FLAGS_LTO := -flto=thin -fsplit-lto-unit +KBUILD_LDFLAGS += --thinlto-cache-dir=$(extmod-prefix).thinlto-cache +else +CC_FLAGS_LTO := -flto +endif +CC_FLAGS_LTO += -fvisibility=hidden + +# Limit inlining across translation units to reduce binary size +KBUILD_LDFLAGS += -mllvm -import-instr-limit=5 +endif + +ifdef CONFIG_LTO +KBUILD_CFLAGS += -fno-lto $(CC_FLAGS_LTO) +KBUILD_AFLAGS += -fno-lto +export CC_FLAGS_LTO +endif + ifdef CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_32B KBUILD_CFLAGS += -falign-functions=32 endif @@ -1054,7 +1073,7 @@ ifdef CONFIG_MODULE_COMPRESS mod_compress_cmd = $(KGZIP) -n -f endif # CONFIG_MODULE_COMPRESS_GZIP ifdef CONFIG_MODULE_COMPRESS_XZ - mod_compress_cmd = $(XZ) -f + mod_compress_cmd = $(XZ) --lzma2=dict=2MiB -f endif # CONFIG_MODULE_COMPRESS_XZ endif # CONFIG_MODULE_COMPRESS export mod_compress_cmd @@ -1105,6 +1124,7 @@ endif # CONFIG_BPF PHONY += prepare0 +extmod-prefix = $(if $(KBUILD_EXTMOD),$(KBUILD_EXTMOD)/) export MODORDER := $(extmod-prefix)modules.order export MODULES_NSDEPS := $(extmod-prefix)modules.nsdeps @@ -1229,6 +1249,10 @@ uapi-asm-generic: PHONY += prepare-objtool prepare-resolve_btfids prepare-objtool: $(objtool_target) ifeq ($(SKIP_STACK_VALIDATION),1) +ifdef CONFIG_FTRACE_MCOUNT_USE_OBJTOOL + @echo "error: Cannot generate __mcount_loc for CONFIG_DYNAMIC_FTRACE=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel" >&2 + @false +endif ifdef CONFIG_UNWINDER_ORC @echo "error: Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel" >&2 @false @@ -1258,14 +1282,24 @@ define filechk_utsrelease.h endef define filechk_version.h - echo \#define LINUX_VERSION_CODE $(shell \ - expr $(VERSION) \* 65536 + 0$(PATCHLEVEL) \* 256 + 0$(SUBLEVEL)); \ - echo '#define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))' + if [ $(SUBLEVEL) -gt 255 ]; then \ + echo \#define LINUX_VERSION_CODE $(shell \ + expr $(VERSION) \* 65536 + $(PATCHLEVEL) \* 256 + 255); \ + else \ + echo \#define LINUX_VERSION_CODE $(shell \ + expr $(VERSION) \* 65536 + $(PATCHLEVEL) \* 256 + $(SUBLEVEL)); \ + fi; \ + echo '#define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + \ + ((c) > 255 ? 255 : (c)))'; \ + echo \#define LINUX_VERSION_MAJOR $(VERSION); \ + echo \#define LINUX_VERSION_PATCHLEVEL $(PATCHLEVEL); \ + echo \#define LINUX_VERSION_SUBLEVEL $(SUBLEVEL) endef +$(version_h): PATCHLEVEL := $(if $(PATCHLEVEL), $(PATCHLEVEL), 0) +$(version_h): SUBLEVEL := $(if $(SUBLEVEL), $(SUBLEVEL), 0) $(version_h): FORCE $(call filechk,version.h) - $(Q)rm -f $(old_version_h) include/generated/utsrelease.h: include/config/kernel.release FORCE $(call filechk,utsrelease.h) @@ -1344,6 +1378,9 @@ ifneq ($(dtstree),) %.dtb: include/config/kernel.release scripts_dtc $(Q)$(MAKE) $(build)=$(dtstree) $(dtstree)/$@ +%.dtbo: include/config/kernel.release scripts_dtc + $(Q)$(MAKE) $(build)=$(dtstree) $(dtstree)/$@ + PHONY += dtbs dtbs_install dtbs_check dtbs: include/config/kernel.release scripts_dtc $(Q)$(MAKE) $(build)=$(dtstree) @@ -1476,7 +1513,7 @@ endif # CONFIG_MODULES # Directories & files removed with 'make clean' CLEAN_FILES += include/ksym vmlinux.symvers \ modules.builtin modules.builtin.modinfo modules.nsdeps \ - compile_commands.json + compile_commands.json .thinlto-cache # Directories & files removed with 'make mrproper' MRPROPER_FILES += include/config include/generated \ @@ -1736,7 +1773,7 @@ PHONY += compile_commands.json clean-dirs := $(KBUILD_EXTMOD) clean: rm-files := $(KBUILD_EXTMOD)/Module.symvers $(KBUILD_EXTMOD)/modules.nsdeps \ - $(KBUILD_EXTMOD)/compile_commands.json + $(KBUILD_EXTMOD)/compile_commands.json $(KBUILD_EXTMOD)/.thinlto-cache PHONY += help help: @@ -1823,7 +1860,7 @@ clean: $(clean-dirs) @find $(if $(KBUILD_EXTMOD), $(KBUILD_EXTMOD), .) $(RCS_FIND_IGNORE) \ \( -name '*.[aios]' -o -name '*.ko' -o -name '.*.cmd' \ -o -name '*.ko.*' \ - -o -name '*.dtb' -o -name '*.dtb.S' -o -name '*.dt.yaml' \ + -o -name '*.dtb' -o -name '*.dtbo' -o -name '*.dtb.S' -o -name '*.dt.yaml' \ -o -name '*.dwo' -o -name '*.lst' \ -o -name '*.su' -o -name '*.mod' \ -o -name '.*.d' -o -name '.*.tmp' -o -name '*.mod.c' \ @@ -1833,7 +1870,8 @@ clean: $(clean-dirs) -o -name '.tmp_*.o.*' \ -o -name '*.c.[012]*.*' \ -o -name '*.ll' \ - -o -name '*.gcno' \) -type f -print | xargs rm -f + -o -name '*.gcno' \ + -o -name '*.*.symversions' \) -type f -print | xargs rm -f # Generate tags for editors # --------------------------------------------------------------------------- diff --git a/arch/Kconfig b/arch/Kconfig index baf67e7f059a..2bb30673d8e6 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -124,8 +124,8 @@ config HAVE_64BIT_ALIGNED_ACCESS accesses are required to be 64 bit aligned in this way even though it is not a 64 bit architecture. - See Documentation/unaligned-memory-access.txt for more - information on the topic of unaligned memory accesses. + See Documentation/core-api/unaligned-memory-access.rst for + more information on the topic of unaligned memory accesses. config HAVE_EFFICIENT_UNALIGNED_ACCESS bool @@ -603,6 +603,96 @@ config SHADOW_CALL_STACK reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the stacks. +config LTO + bool + help + Selected if the kernel will be built using the compiler's LTO feature. + +config LTO_CLANG + bool + select LTO + help + Selected if the kernel will be built using Clang's LTO feature. + +config ARCH_SUPPORTS_LTO_CLANG + bool + help + An architecture should select this option if it supports: + - compiling with Clang, + - compiling inline assembly with Clang's integrated assembler, + - and linking with LLD. + +config ARCH_SUPPORTS_LTO_CLANG_THIN + bool + help + An architecture should select this option if it can support Clang's + ThinLTO mode. + +config HAS_LTO_CLANG + def_bool y + # Clang >= 11: https://github.com/ClangBuiltLinux/linux/issues/510 + depends on CC_IS_CLANG && CLANG_VERSION >= 110000 && LD_IS_LLD + depends on $(success,test $(LLVM) -eq 1) + depends on $(success,test $(LLVM_IAS) -eq 1) + depends on $(success,$(NM) --help | head -n 1 | grep -qi llvm) + depends on $(success,$(AR) --help | head -n 1 | grep -qi llvm) + depends on ARCH_SUPPORTS_LTO_CLANG + depends on !FTRACE_MCOUNT_USE_RECORDMCOUNT + depends on !KASAN + depends on !GCOV_KERNEL + help + The compiler and Kconfig options support building with Clang's + LTO. + +choice + prompt "Link Time Optimization (LTO)" + default LTO_NONE + help + This option enables Link Time Optimization (LTO), which allows the + compiler to optimize binaries globally. + + If unsure, select LTO_NONE. Note that LTO is very resource-intensive + so it's disabled by default. + +config LTO_NONE + bool "None" + help + Build the kernel normally, without Link Time Optimization (LTO). + +config LTO_CLANG_FULL + bool "Clang Full LTO (EXPERIMENTAL)" + depends on HAS_LTO_CLANG + depends on !COMPILE_TEST + select LTO_CLANG + help + This option enables Clang's full Link Time Optimization (LTO), which + allows the compiler to optimize the kernel globally. If you enable + this option, the compiler generates LLVM bitcode instead of ELF + object files, and the actual compilation from bitcode happens at + the LTO link step, which may take several minutes depending on the + kernel configuration. More information can be found from LLVM's + documentation: + + https://llvm.org/docs/LinkTimeOptimization.html + + During link time, this option can use a large amount of RAM, and + may take much longer than the ThinLTO option. + +config LTO_CLANG_THIN + bool "Clang ThinLTO (EXPERIMENTAL)" + depends on HAS_LTO_CLANG && ARCH_SUPPORTS_LTO_CLANG_THIN + select LTO_CLANG + help + This option enables Clang's ThinLTO, which allows for parallel + optimization and faster incremental compiles compared to the + CONFIG_LTO_CLANG_FULL option. More information can be found + from Clang's documentation: + + https://clang.llvm.org/docs/ThinLTO.html + + If unsure, say Y. +endchoice + config HAVE_ARCH_WITHIN_STACK_FRAMES bool help @@ -731,6 +821,12 @@ config HAVE_IRQ_EXIT_ON_IRQ_STACK This spares a stack switch and improves cache usage on softirq processing. +config HAVE_SOFTIRQ_ON_OWN_STACK + bool + help + Architecture provides a function to run __do_softirq() on a + seperate stack. + config PGTABLE_LEVELS int default 2 diff --git a/arch/alpha/configs/defconfig b/arch/alpha/configs/defconfig index 6293675db164..724c4075df40 100644 --- a/arch/alpha/configs/defconfig +++ b/arch/alpha/configs/defconfig @@ -1,4 +1,3 @@ -CONFIG_EXPERIMENTAL=y CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_LOG_BUF_SHIFT=14 diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c index 6c71554206cc..5112ab996394 100644 --- a/arch/alpha/kernel/process.c +++ b/arch/alpha/kernel/process.c @@ -249,7 +249,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, childti->pcb.ksp = (unsigned long) childstack; childti->pcb.flags = 1; /* set FEN, clear everything else */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ memset(childstack, 0, sizeof(struct switch_stack) + sizeof(struct pt_regs)); diff --git a/arch/alpha/kernel/syscalls/Makefile b/arch/alpha/kernel/syscalls/Makefile index 659faefdcb1d..285aaba832d9 100644 --- a/arch/alpha/kernel/syscalls/Makefile +++ b/arch/alpha/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,18 +21,19 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_abi_$(basetarget))' \ '$(systbl_offset_$(basetarget))' -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl index a6617067dbe6..02f0244e005c 100644 --- a/arch/alpha/kernel/syscalls/syscall.tbl +++ b/arch/alpha/kernel/syscalls/syscall.tbl @@ -481,3 +481,4 @@ 549 common faccessat2 sys_faccessat2 550 common process_madvise sys_process_madvise 551 common epoll_pwait2 sys_epoll_pwait2 +552 common mount_setattr sys_mount_setattr diff --git a/arch/arc/kernel/process.c b/arch/arc/kernel/process.c index 37f724ad5e39..d838d0d57696 100644 --- a/arch/arc/kernel/process.c +++ b/arch/arc/kernel/process.c @@ -191,7 +191,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, childksp[0] = 0; /* fp */ childksp[1] = (unsigned long)ret_from_fork; /* blink */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(c_regs, 0, sizeof(struct pt_regs)); c_callee->r13 = kthread_arg; diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 6d0ed2888935..853aab5ab327 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -23,6 +23,7 @@ config ARM select ARCH_HAS_TEARDOWN_DMA_OPS if MMU select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAVE_CUSTOM_GPIO_H + select ARCH_HAVE_NMI_SAFE_CMPXCHG if CPU_V7 || CPU_V7M || CPU_V6K select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_KEEP_MEMBLOCK select ARCH_MIGHT_HAVE_PC_PARPORT @@ -1856,9 +1857,10 @@ config AUTO_ZRELADDR help ZRELADDR is the physical address where the decompressed kernel image will be placed. If AUTO_ZRELADDR is selected, the address - will be determined at run-time by masking the current IP with - 0xf8000000. This assumes the zImage being placed in the first 128MB - from start of memory. + will be determined at run-time, either by masking the current IP + with 0xf8000000, or, if invalid, from the DTB passed in r2. + This assumes the zImage being placed in the first 128MB from + start of memory. config EFI_STUB bool diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug index c36c5d4c6e9c..9e0b5e7f12af 100644 --- a/arch/arm/Kconfig.debug +++ b/arch/arm/Kconfig.debug @@ -1158,10 +1158,9 @@ choice Say Y here if you want kernel low-level debugging support on ST SPEAr13xx based platforms. - config STIH41X_DEBUG_ASC2 + config DEBUG_STIH41X_ASC2 bool "Use StiH415/416 ASC2 UART for low-level debug" depends on ARCH_STI - select DEBUG_STI_UART help Say Y here if you want kernel low-level debugging support on STiH415/416 based platforms like b2000, which has @@ -1169,10 +1168,9 @@ choice If unsure, say N. - config STIH41X_DEBUG_SBC_ASC1 + config DEBUG_STIH41X_SBC_ASC1 bool "Use StiH415/416 SBC ASC1 UART for low-level debug" depends on ARCH_STI - select DEBUG_STI_UART help Say Y here if you want kernel low-level debugging support on STiH415/416 based platforms like b2020. which has @@ -1180,6 +1178,16 @@ choice If unsure, say N. + config DEBUG_STIH418_SBC_ASC0 + bool "Use StiH418 SBC ASC0 UART for low-level debug" + depends on ARCH_STI + help + Say Y here if you want kernel low-level debugging support + on STiH418 based platforms which has default UART wired + up to SBC ASC0. + + If unsure, say N. + config STM32F4_DEBUG_UART bool "Use STM32F4 UART for low-level debug" depends on MACH_STM32F429 || MACH_STM32F469 @@ -1484,10 +1492,6 @@ config DEBUG_TEGRA_UART bool depends on ARCH_TEGRA -config DEBUG_STI_UART - bool - depends on ARCH_STI - config DEBUG_STM32_UART bool depends on ARCH_STM32 @@ -1546,7 +1550,9 @@ config DEBUG_LL_INCLUDE default "debug/renesas-scif.S" if DEBUG_RMOBILE_SCIFA4 default "debug/s3c24xx.S" if DEBUG_S3C24XX_UART || DEBUG_S3C64XX_UART default "debug/s5pv210.S" if DEBUG_S5PV210_UART - default "debug/sti.S" if DEBUG_STI_UART + default "debug/sti.S" if DEBUG_STIH41X_ASC2 + default "debug/sti.S" if DEBUG_STIH41X_SBC_ASC1 + default "debug/sti.S" if DEBUG_STIH418_SBC_ASC0 default "debug/stm32.S" if DEBUG_STM32_UART default "debug/tegra.S" if DEBUG_TEGRA_UART default "debug/ux500.S" if DEBUG_UX500_UART @@ -1579,6 +1585,7 @@ config DEBUG_UART_PHYS default 0x02531000 if DEBUG_KEYSTONE_UART1 default 0x03010fe0 if ARCH_RPC default 0x07000000 if DEBUG_SUN9I_UART0 + default 0x09530000 if DEBUG_STIH418_SBC_ASC0 default 0x10009000 if DEBUG_REALVIEW_STD_PORT || \ DEBUG_VEXPRESS_UART0_CA9 default 0x1010c000 if DEBUG_REALVIEW_PB1176_PORT @@ -1671,7 +1678,9 @@ config DEBUG_UART_PHYS default 0xfc00c000 if DEBUG_AT91_SAMA5D4_USART3 default 0xfcb00000 if DEBUG_HI3620_UART default 0xfd883000 if DEBUG_ALPINE_UART0 + default 0xfe531000 if DEBUG_STIH41X_SBC_ASC1 default 0xfe800000 if ARCH_IOP32X + default 0xfed32000 if DEBUG_STIH41X_ASC2 default 0xff690000 if DEBUG_RK32_UART2 default 0xffc02000 if DEBUG_SOCFPGA_UART0 default 0xffc02100 if DEBUG_SOCFPGA_ARRIA10_UART1 @@ -1699,7 +1708,9 @@ config DEBUG_UART_PHYS DEBUG_S3C64XX_UART || \ DEBUG_BCM63XX_UART || DEBUG_ASM9260_UART || \ DEBUG_DIGICOLOR_UA0 || \ - DEBUG_AT91_UART || DEBUG_STM32_UART + DEBUG_AT91_UART || DEBUG_STM32_UART || \ + DEBUG_STIH41X_ASC2 || DEBUG_STIH41X_SBC_ASC1 || \ + DEBUG_STIH418_SBC_ASC0 config DEBUG_UART_VIRT hex "Virtual base address of debug UART" @@ -1744,6 +1755,7 @@ config DEBUG_UART_VIRT default 0xf8090000 if DEBUG_VEXPRESS_UART0_RS1 default 0xf8ffee00 if DEBUG_AT91_SAM9263_DBGU default 0xf8fff200 if DEBUG_AT91_RM9200_DBGU + default 0xf9530000 if DEBUG_STIH418_SBC_ASC0 default 0xf9e09000 if DEBUG_AM33XXUART1 default 0xfa020000 if DEBUG_OMAP4UART3 || DEBUG_TI81XXUART1 default 0xfa022000 if DEBUG_TI81XXUART2 @@ -1762,7 +1774,9 @@ config DEBUG_UART_VIRT default 0xfb10c000 if DEBUG_REALVIEW_PB1176_PORT default 0xfcfe8600 if DEBUG_BCM63XX_UART default 0xfd000000 if DEBUG_SPEAR3XX || DEBUG_SPEAR13XX + default 0xfd531000 if DEBUG_STIH41X_SBC_ASC1 default 0xfd883000 if DEBUG_ALPINE_UART0 + default 0xfdd32000 if DEBUG_STIH41X_ASC2 default 0xfe010000 if STM32MP1_DEBUG_UART default 0xfe017000 if DEBUG_MMP_UART2 default 0xfe018000 if DEBUG_MMP_UART3 @@ -1803,7 +1817,9 @@ config DEBUG_UART_VIRT DEBUG_S3C64XX_UART || \ DEBUG_BCM63XX_UART || DEBUG_ASM9260_UART || \ DEBUG_DIGICOLOR_UA0 || \ - DEBUG_AT91_UART || DEBUG_STM32_UART + DEBUG_AT91_UART || DEBUG_STM32_UART || \ + DEBUG_STIH41X_ASC2 || DEBUG_STIH41X_SBC_ASC1 || \ + DEBUG_STIH418_SBC_ASC0 config DEBUG_UART_8250_SHIFT int "Register offset shift for the 8250 debug UART" @@ -1837,7 +1853,7 @@ config DEBUG_UNCOMPRESS depends on ARCH_MULTIPLATFORM || PLAT_SAMSUNG || ARM_SINGLE_ARMV7M depends on DEBUG_LL && !DEBUG_OMAP2PLUS_UART && \ (!DEBUG_TEGRA_UART || !ZBOOT_ROM) && \ - !DEBUG_BRCMSTB_UART + !DEBUG_BRCMSTB_UART && !DEBUG_SEMIHOSTING help This option influences the normal decompressor output for multiplatform kernels. Normally, multiplatform kernels disable diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index fb521efcc6c2..fd94e27ba4fa 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -87,10 +87,13 @@ libfdt_objs := fdt_rw.o fdt_ro.o fdt_wip.o fdt.o ifeq ($(CONFIG_ARM_ATAG_DTB_COMPAT),y) OBJS += $(libfdt_objs) atags_to_fdt.o endif +ifeq ($(CONFIG_USE_OF),y) +OBJS += $(libfdt_objs) fdt_check_mem_start.o +endif # -fstack-protector-strong triggers protection checks in this code, # but it is being used too early to link to meaningful stack_chk logic. -$(foreach o, $(libfdt_objs) atags_to_fdt.o, \ +$(foreach o, $(libfdt_objs) atags_to_fdt.o fdt_check_mem_start.o, \ $(eval CFLAGS_$(o) := -I $(srctree)/scripts/dtc/libfdt -fno-stack-protector)) # These were previously generated C files. When you are building the kernel diff --git a/arch/arm/boot/compressed/fdt_check_mem_start.c b/arch/arm/boot/compressed/fdt_check_mem_start.c new file mode 100644 index 000000000000..62450d824c3c --- /dev/null +++ b/arch/arm/boot/compressed/fdt_check_mem_start.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include + +static const void *get_prop(const void *fdt, const char *node_path, + const char *property, int minlen) +{ + const void *prop; + int offset, len; + + offset = fdt_path_offset(fdt, node_path); + if (offset < 0) + return NULL; + + prop = fdt_getprop(fdt, offset, property, &len); + if (!prop || len < minlen) + return NULL; + + return prop; +} + +static uint32_t get_cells(const void *fdt, const char *name) +{ + const fdt32_t *prop = get_prop(fdt, "/", name, sizeof(fdt32_t)); + + if (!prop) { + /* default */ + return 1; + } + + return fdt32_ld(prop); +} + +static uint64_t get_val(const fdt32_t *cells, uint32_t ncells) +{ + uint64_t r; + + r = fdt32_ld(cells); + if (ncells > 1) + r = (r << 32) | fdt32_ld(cells + 1); + + return r; +} + +/* + * Check the start of physical memory + * + * Traditionally, the start address of physical memory is obtained by masking + * the program counter. However, this does require that this address is a + * multiple of 128 MiB, precluding booting Linux on platforms where this + * requirement is not fulfilled. + * Hence validate the calculated address against the memory information in the + * DTB, and, if out-of-range, replace it by the real start address. + * To preserve backwards compatibility (systems reserving a block of memory + * at the start of physical memory, kdump, ...), the traditional method is + * always used if it yields a valid address. + * + * Return value: start address of physical memory to use + */ +uint32_t fdt_check_mem_start(uint32_t mem_start, const void *fdt) +{ + uint32_t addr_cells, size_cells, base; + uint32_t fdt_mem_start = 0xffffffff; + const fdt32_t *reg, *endp; + uint64_t size, end; + const char *type; + int offset, len; + + if (!fdt) + return mem_start; + + if (fdt_magic(fdt) != FDT_MAGIC) + return mem_start; + + /* There may be multiple cells on LPAE platforms */ + addr_cells = get_cells(fdt, "#address-cells"); + size_cells = get_cells(fdt, "#size-cells"); + if (addr_cells > 2 || size_cells > 2) + return mem_start; + + /* Walk all memory nodes and regions */ + for (offset = fdt_next_node(fdt, -1, NULL); offset >= 0; + offset = fdt_next_node(fdt, offset, NULL)) { + type = fdt_getprop(fdt, offset, "device_type", NULL); + if (!type || strcmp(type, "memory")) + continue; + + reg = fdt_getprop(fdt, offset, "linux,usable-memory", &len); + if (!reg) + reg = fdt_getprop(fdt, offset, "reg", &len); + if (!reg) + continue; + + for (endp = reg + (len / sizeof(fdt32_t)); + endp - reg >= addr_cells + size_cells; + reg += addr_cells + size_cells) { + size = get_val(reg + addr_cells, size_cells); + if (!size) + continue; + + if (addr_cells > 1 && fdt32_ld(reg)) { + /* Outside 32-bit address space, skipping */ + continue; + } + + base = fdt32_ld(reg + addr_cells - 1); + end = base + size; + if (mem_start >= base && mem_start < end) { + /* Calculated address is valid, use it */ + return mem_start; + } + + if (base < fdt_mem_start) + fdt_mem_start = base; + } + } + + if (fdt_mem_start == 0xffffffff) { + /* No usable memory found, falling back to default */ + return mem_start; + } + + /* + * The calculated address is not usable. + * Use the lowest usable physical memory address from the DTB instead, + * and make sure this is a multiple of 2 MiB for phys/virt patching. + */ + return round_up(fdt_mem_start, SZ_2M); +} diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S index d9cce7238a36..b1cb1972361b 100644 --- a/arch/arm/boot/compressed/head.S +++ b/arch/arm/boot/compressed/head.S @@ -174,10 +174,7 @@ .macro be32tocpu, val, tmp #ifndef __ARMEB__ /* convert to little endian */ - eor \tmp, \val, \val, ror #16 - bic \tmp, \tmp, #0x00ff0000 - mov \val, \val, ror #8 - eor \val, \val, \tmp, lsr #8 + rev_l \val, \tmp #endif .endm @@ -282,10 +279,40 @@ not_angel: * are already placing their zImage in (eg) the top 64MB * of this range. */ - mov r4, pc - and r4, r4, #0xf8000000 + mov r0, pc + and r0, r0, #0xf8000000 +#ifdef CONFIG_USE_OF + adr r1, LC1 +#ifdef CONFIG_ARM_APPENDED_DTB + /* + * Look for an appended DTB. If found, we cannot use it to + * validate the calculated start of physical memory, as its + * memory nodes may need to be augmented by ATAGS stored at + * an offset from the same start of physical memory. + */ + ldr r2, [r1, #4] @ get &_edata + add r2, r2, r1 @ relocate it + ldr r2, [r2] @ get DTB signature + ldr r3, =OF_DT_MAGIC + cmp r2, r3 @ do we have a DTB there? + beq 1f @ if yes, skip validation +#endif /* CONFIG_ARM_APPENDED_DTB */ + + /* + * Make sure we have some stack before calling C code. + * No GOT fixup has occurred yet, but none of the code we're + * about to call uses any global variables. + */ + ldr sp, [r1] @ get stack location + add sp, sp, r1 @ apply relocation + + /* Validate calculated start against passed DTB */ + mov r1, r8 + bl fdt_check_mem_start +1: +#endif /* CONFIG_USE_OF */ /* Determine final kernel image address. */ - add r4, r4, #TEXT_OFFSET + add r4, r0, #TEXT_OFFSET #else ldr r4, =zreladdr #endif @@ -1164,9 +1191,9 @@ __armv4_mmu_cache_off: __armv7_mmu_cache_off: mrc p15, 0, r0, c1, c0 #ifdef CONFIG_MMU - bic r0, r0, #0x000d + bic r0, r0, #0x0005 #else - bic r0, r0, #0x000c + bic r0, r0, #0x0004 #endif mcr p15, 0, r0, c1, c0 @ turn MMU and cache off mov r0, #0 diff --git a/arch/arm/boot/dts/cros-ec-keyboard.dtsi b/arch/arm/boot/dts/cros-ec-keyboard.dtsi index 165c5bcd510e..55c4744fa7e7 100644 --- a/arch/arm/boot/dts/cros-ec-keyboard.dtsi +++ b/arch/arm/boot/dts/cros-ec-keyboard.dtsi @@ -6,103 +6,18 @@ */ #include +#include &cros_ec { - keyboard-controller { + keyboard_controller: keyboard-controller { compatible = "google,cros-ec-keyb"; keypad,num-rows = <8>; keypad,num-columns = <13>; google,needs-ghost-filter; linux,keymap = < - MATRIX_KEY(0x00, 0x01, KEY_LEFTMETA) - MATRIX_KEY(0x00, 0x02, KEY_F1) - MATRIX_KEY(0x00, 0x03, KEY_B) - MATRIX_KEY(0x00, 0x04, KEY_F10) - MATRIX_KEY(0x00, 0x05, KEY_RO) - MATRIX_KEY(0x00, 0x06, KEY_N) - MATRIX_KEY(0x00, 0x08, KEY_EQUAL) - MATRIX_KEY(0x00, 0x0a, KEY_RIGHTALT) - - MATRIX_KEY(0x01, 0x01, KEY_ESC) - MATRIX_KEY(0x01, 0x02, KEY_F4) - MATRIX_KEY(0x01, 0x03, KEY_G) - MATRIX_KEY(0x01, 0x04, KEY_F7) - MATRIX_KEY(0x01, 0x06, KEY_H) - MATRIX_KEY(0x01, 0x08, KEY_APOSTROPHE) - MATRIX_KEY(0x01, 0x09, KEY_F9) - MATRIX_KEY(0x01, 0x0b, KEY_BACKSPACE) - MATRIX_KEY(0x01, 0x0c, KEY_HENKAN) - - MATRIX_KEY(0x02, 0x00, KEY_LEFTCTRL) - MATRIX_KEY(0x02, 0x01, KEY_TAB) - MATRIX_KEY(0x02, 0x02, KEY_F3) - MATRIX_KEY(0x02, 0x03, KEY_T) - MATRIX_KEY(0x02, 0x04, KEY_F6) - MATRIX_KEY(0x02, 0x05, KEY_RIGHTBRACE) - MATRIX_KEY(0x02, 0x06, KEY_Y) - MATRIX_KEY(0x02, 0x07, KEY_102ND) - MATRIX_KEY(0x02, 0x08, KEY_LEFTBRACE) - MATRIX_KEY(0x02, 0x09, KEY_F8) - MATRIX_KEY(0x02, 0x0a, KEY_YEN) - - MATRIX_KEY(0x03, 0x00, KEY_LEFTMETA) - MATRIX_KEY(0x03, 0x01, KEY_GRAVE) - MATRIX_KEY(0x03, 0x02, KEY_F2) - MATRIX_KEY(0x03, 0x03, KEY_5) - MATRIX_KEY(0x03, 0x04, KEY_F5) - MATRIX_KEY(0x03, 0x06, KEY_6) - MATRIX_KEY(0x03, 0x08, KEY_MINUS) - MATRIX_KEY(0x03, 0x09, KEY_F13) - MATRIX_KEY(0x03, 0x0b, KEY_BACKSLASH) - MATRIX_KEY(0x03, 0x0c, KEY_MUHENKAN) - - MATRIX_KEY(0x04, 0x00, KEY_RIGHTCTRL) - MATRIX_KEY(0x04, 0x01, KEY_A) - MATRIX_KEY(0x04, 0x02, KEY_D) - MATRIX_KEY(0x04, 0x03, KEY_F) - MATRIX_KEY(0x04, 0x04, KEY_S) - MATRIX_KEY(0x04, 0x05, KEY_K) - MATRIX_KEY(0x04, 0x06, KEY_J) - MATRIX_KEY(0x04, 0x08, KEY_SEMICOLON) - MATRIX_KEY(0x04, 0x09, KEY_L) - MATRIX_KEY(0x04, 0x0a, KEY_BACKSLASH) - MATRIX_KEY(0x04, 0x0b, KEY_ENTER) - - MATRIX_KEY(0x05, 0x01, KEY_Z) - MATRIX_KEY(0x05, 0x02, KEY_C) - MATRIX_KEY(0x05, 0x03, KEY_V) - MATRIX_KEY(0x05, 0x04, KEY_X) - MATRIX_KEY(0x05, 0x05, KEY_COMMA) - MATRIX_KEY(0x05, 0x06, KEY_M) - MATRIX_KEY(0x05, 0x07, KEY_LEFTSHIFT) - MATRIX_KEY(0x05, 0x08, KEY_SLASH) - MATRIX_KEY(0x05, 0x09, KEY_DOT) - MATRIX_KEY(0x05, 0x0b, KEY_SPACE) - - MATRIX_KEY(0x06, 0x01, KEY_1) - MATRIX_KEY(0x06, 0x02, KEY_3) - MATRIX_KEY(0x06, 0x03, KEY_4) - MATRIX_KEY(0x06, 0x04, KEY_2) - MATRIX_KEY(0x06, 0x05, KEY_8) - MATRIX_KEY(0x06, 0x06, KEY_7) - MATRIX_KEY(0x06, 0x08, KEY_0) - MATRIX_KEY(0x06, 0x09, KEY_9) - MATRIX_KEY(0x06, 0x0a, KEY_LEFTALT) - MATRIX_KEY(0x06, 0x0b, KEY_DOWN) - MATRIX_KEY(0x06, 0x0c, KEY_RIGHT) - - MATRIX_KEY(0x07, 0x01, KEY_Q) - MATRIX_KEY(0x07, 0x02, KEY_E) - MATRIX_KEY(0x07, 0x03, KEY_R) - MATRIX_KEY(0x07, 0x04, KEY_W) - MATRIX_KEY(0x07, 0x05, KEY_I) - MATRIX_KEY(0x07, 0x06, KEY_U) - MATRIX_KEY(0x07, 0x07, KEY_RIGHTSHIFT) - MATRIX_KEY(0x07, 0x08, KEY_P) - MATRIX_KEY(0x07, 0x09, KEY_O) - MATRIX_KEY(0x07, 0x0b, KEY_UP) - MATRIX_KEY(0x07, 0x0c, KEY_LEFT) + CROS_STD_TOP_ROW_KEYMAP + CROS_STD_MAIN_KEYMAP >; }; }; diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi index 5f1a8bd13880..e025b7c9a357 100644 --- a/arch/arm/boot/dts/omap5.dtsi +++ b/arch/arm/boot/dts/omap5.dtsi @@ -518,6 +518,9 @@ clocks = <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 8>, <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 10>; clock-names = "fck", "sys_clk"; + + #address-cells = <1>; + #size-cells = <0>; }; }; @@ -550,6 +553,9 @@ clocks = <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 8>, <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 10>; clock-names = "fck", "sys_clk"; + + #address-cells = <1>; + #size-cells = <0>; }; }; diff --git a/arch/arm/common/locomo.c b/arch/arm/common/locomo.c index 62f241b09fe3..e45f4e4e06b6 100644 --- a/arch/arm/common/locomo.c +++ b/arch/arm/common/locomo.c @@ -838,11 +838,10 @@ static int locomo_bus_remove(struct device *dev) { struct locomo_dev *ldev = LOCOMO_DEV(dev); struct locomo_driver *drv = LOCOMO_DRV(dev->driver); - int ret = 0; if (drv->remove) - ret = drv->remove(ldev); - return ret; + drv->remove(ldev); + return 0; } struct bus_type locomo_bus_type = { diff --git a/arch/arm/common/sa1111.c b/arch/arm/common/sa1111.c index f89c1ea327a2..ff5e0d04cb89 100644 --- a/arch/arm/common/sa1111.c +++ b/arch/arm/common/sa1111.c @@ -1368,11 +1368,11 @@ static int sa1111_bus_remove(struct device *dev) { struct sa1111_dev *sadev = to_sa1111_device(dev); struct sa1111_driver *drv = SA1111_DRV(dev->driver); - int ret = 0; if (drv->remove) - ret = drv->remove(sadev); - return ret; + drv->remove(sadev); + + return 0; } struct bus_type sa1111_bus_type = { diff --git a/arch/arm/configs/bcm2835_defconfig b/arch/arm/configs/bcm2835_defconfig index 1ef2bc4c7f69..383c632eba7b 100644 --- a/arch/arm/configs/bcm2835_defconfig +++ b/arch/arm/configs/bcm2835_defconfig @@ -176,7 +176,6 @@ CONFIG_BOOT_PRINTK_DELAY=y CONFIG_DYNAMIC_DEBUG=y CONFIG_DEBUG_INFO=y # CONFIG_ENABLE_MUST_CHECK is not set -CONFIG_UNUSED_SYMBOLS=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_LOCKUP_DETECTOR=y CONFIG_SCHED_TRACER=y diff --git a/arch/arm/configs/mxs_defconfig b/arch/arm/configs/mxs_defconfig index a9c6f32a9b1c..ca32446b187f 100644 --- a/arch/arm/configs/mxs_defconfig +++ b/arch/arm/configs/mxs_defconfig @@ -164,7 +164,6 @@ CONFIG_FONTS=y CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y CONFIG_FRAME_WARN=2048 -CONFIG_UNUSED_SYMBOLS=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_SOFTLOCKUP_DETECTOR=y diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig index b26ef4866a35..f250bf1cc022 100644 --- a/arch/arm/configs/omap2plus_defconfig +++ b/arch/arm/configs/omap2plus_defconfig @@ -356,8 +356,8 @@ CONFIG_DRM_OMAP=m CONFIG_OMAP5_DSS_HDMI=y CONFIG_OMAP2_DSS_SDI=y CONFIG_OMAP2_DSS_DSI=y -CONFIG_DRM_OMAP_PANEL_DSI_CM=m CONFIG_DRM_TILCDC=m +CONFIG_DRM_PANEL_DSI_CM=m CONFIG_DRM_PANEL_SIMPLE=m CONFIG_DRM_PANEL_LG_LB035Q02=m CONFIG_DRM_PANEL_NEC_NL8048HL11=m diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index c9bf2df85cb9..2b575792363e 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -62,6 +62,25 @@ config CRYPTO_SHA512_ARM SHA-512 secure hash standard (DFIPS 180-2) implemented using optimized ARM assembler and NEON, when available. +config CRYPTO_BLAKE2S_ARM + tristate "BLAKE2s digest algorithm (ARM)" + select CRYPTO_ARCH_HAVE_LIB_BLAKE2S + help + BLAKE2s digest algorithm optimized with ARM scalar instructions. This + is faster than the generic implementations of BLAKE2s and BLAKE2b, but + slower than the NEON implementation of BLAKE2b. (There is no NEON + implementation of BLAKE2s, since NEON doesn't really help with it.) + +config CRYPTO_BLAKE2B_NEON + tristate "BLAKE2b digest algorithm (ARM NEON)" + depends on KERNEL_MODE_NEON + select CRYPTO_BLAKE2B + help + BLAKE2b digest algorithm optimized with ARM NEON instructions. + On ARM processors that have NEON support but not the ARMv8 + Crypto Extensions, typically this BLAKE2b implementation is + much faster than SHA-2 and slightly faster than SHA-1. + config CRYPTO_AES_ARM tristate "Scalar AES cipher for ARM" select CRYPTO_ALGAPI diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index b745c17d356f..8f26c454ea12 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -9,6 +9,8 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o +obj-$(CONFIG_CRYPTO_BLAKE2S_ARM) += blake2s-arm.o +obj-$(CONFIG_CRYPTO_BLAKE2B_NEON) += blake2b-neon.o obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o @@ -29,6 +31,8 @@ sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o sha256-arm-y := sha256-core.o sha256_glue.o $(sha256-arm-neon-y) sha512-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha512-neon-glue.o sha512-arm-y := sha512-core.o sha512-glue.o $(sha512-arm-neon-y) +blake2s-arm-y := blake2s-core.o blake2s-glue.o +blake2b-neon-y := blake2b-neon-core.o blake2b-neon-glue.o sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index f70af1d0514b..5c6cd3c63cbc 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,8 @@ MODULE_ALIAS_CRYPTO("cbc(aes)-all"); MODULE_ALIAS_CRYPTO("ctr(aes)"); MODULE_ALIAS_CRYPTO("xts(aes)"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); + asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds); asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], diff --git a/arch/arm/crypto/blake2b-neon-core.S b/arch/arm/crypto/blake2b-neon-core.S new file mode 100644 index 000000000000..0406a186377f --- /dev/null +++ b/arch/arm/crypto/blake2b-neon-core.S @@ -0,0 +1,347 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * BLAKE2b digest algorithm, NEON accelerated + * + * Copyright 2020 Google LLC + * + * Author: Eric Biggers + */ + +#include + + .text + .fpu neon + + // The arguments to blake2b_compress_neon() + STATE .req r0 + BLOCK .req r1 + NBLOCKS .req r2 + INC .req r3 + + // Pointers to the rotation tables + ROR24_TABLE .req r4 + ROR16_TABLE .req r5 + + // The original stack pointer + ORIG_SP .req r6 + + // NEON registers which contain the message words of the current block. + // M_0-M_3 are occasionally used for other purposes too. + M_0 .req d16 + M_1 .req d17 + M_2 .req d18 + M_3 .req d19 + M_4 .req d20 + M_5 .req d21 + M_6 .req d22 + M_7 .req d23 + M_8 .req d24 + M_9 .req d25 + M_10 .req d26 + M_11 .req d27 + M_12 .req d28 + M_13 .req d29 + M_14 .req d30 + M_15 .req d31 + + .align 4 + // Tables for computing ror64(x, 24) and ror64(x, 16) using the vtbl.8 + // instruction. This is the most efficient way to implement these + // rotation amounts with NEON. (On Cortex-A53 it's the same speed as + // vshr.u64 + vsli.u64, while on Cortex-A7 it's faster.) +.Lror24_table: + .byte 3, 4, 5, 6, 7, 0, 1, 2 +.Lror16_table: + .byte 2, 3, 4, 5, 6, 7, 0, 1 + // The BLAKE2b initialization vector +.Lblake2b_IV: + .quad 0x6a09e667f3bcc908, 0xbb67ae8584caa73b + .quad 0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1 + .quad 0x510e527fade682d1, 0x9b05688c2b3e6c1f + .quad 0x1f83d9abfb41bd6b, 0x5be0cd19137e2179 + +// Execute one round of BLAKE2b by updating the state matrix v[0..15] in the +// NEON registers q0-q7. The message block is in q8..q15 (M_0-M_15). The stack +// pointer points to a 32-byte aligned buffer containing a copy of q8 and q9 +// (M_0-M_3), so that they can be reloaded if they are used as temporary +// registers. The macro arguments s0-s15 give the order in which the message +// words are used in this round. 'final' is 1 if this is the final round. +.macro _blake2b_round s0, s1, s2, s3, s4, s5, s6, s7, \ + s8, s9, s10, s11, s12, s13, s14, s15, final=0 + + // Mix the columns: + // (v[0], v[4], v[8], v[12]), (v[1], v[5], v[9], v[13]), + // (v[2], v[6], v[10], v[14]), and (v[3], v[7], v[11], v[15]). + + // a += b + m[blake2b_sigma[r][2*i + 0]]; + vadd.u64 q0, q0, q2 + vadd.u64 q1, q1, q3 + vadd.u64 d0, d0, M_\s0 + vadd.u64 d1, d1, M_\s2 + vadd.u64 d2, d2, M_\s4 + vadd.u64 d3, d3, M_\s6 + + // d = ror64(d ^ a, 32); + veor q6, q6, q0 + veor q7, q7, q1 + vrev64.32 q6, q6 + vrev64.32 q7, q7 + + // c += d; + vadd.u64 q4, q4, q6 + vadd.u64 q5, q5, q7 + + // b = ror64(b ^ c, 24); + vld1.8 {M_0}, [ROR24_TABLE, :64] + veor q2, q2, q4 + veor q3, q3, q5 + vtbl.8 d4, {d4}, M_0 + vtbl.8 d5, {d5}, M_0 + vtbl.8 d6, {d6}, M_0 + vtbl.8 d7, {d7}, M_0 + + // a += b + m[blake2b_sigma[r][2*i + 1]]; + // + // M_0 got clobbered above, so we have to reload it if any of the four + // message words this step needs happens to be M_0. Otherwise we don't + // need to reload it here, as it will just get clobbered again below. +.if \s1 == 0 || \s3 == 0 || \s5 == 0 || \s7 == 0 + vld1.8 {M_0}, [sp, :64] +.endif + vadd.u64 q0, q0, q2 + vadd.u64 q1, q1, q3 + vadd.u64 d0, d0, M_\s1 + vadd.u64 d1, d1, M_\s3 + vadd.u64 d2, d2, M_\s5 + vadd.u64 d3, d3, M_\s7 + + // d = ror64(d ^ a, 16); + vld1.8 {M_0}, [ROR16_TABLE, :64] + veor q6, q6, q0 + veor q7, q7, q1 + vtbl.8 d12, {d12}, M_0 + vtbl.8 d13, {d13}, M_0 + vtbl.8 d14, {d14}, M_0 + vtbl.8 d15, {d15}, M_0 + + // c += d; + vadd.u64 q4, q4, q6 + vadd.u64 q5, q5, q7 + + // b = ror64(b ^ c, 63); + // + // This rotation amount isn't a multiple of 8, so it has to be + // implemented using a pair of shifts, which requires temporary + // registers. Use q8-q9 (M_0-M_3) for this, and reload them afterwards. + veor q8, q2, q4 + veor q9, q3, q5 + vshr.u64 q2, q8, #63 + vshr.u64 q3, q9, #63 + vsli.u64 q2, q8, #1 + vsli.u64 q3, q9, #1 + vld1.8 {q8-q9}, [sp, :256] + + // Mix the diagonals: + // (v[0], v[5], v[10], v[15]), (v[1], v[6], v[11], v[12]), + // (v[2], v[7], v[8], v[13]), and (v[3], v[4], v[9], v[14]). + // + // There are two possible ways to do this: use 'vext' instructions to + // shift the rows of the matrix so that the diagonals become columns, + // and undo it afterwards; or just use 64-bit operations on 'd' + // registers instead of 128-bit operations on 'q' registers. We use the + // latter approach, as it performs much better on Cortex-A7. + + // a += b + m[blake2b_sigma[r][2*i + 0]]; + vadd.u64 d0, d0, d5 + vadd.u64 d1, d1, d6 + vadd.u64 d2, d2, d7 + vadd.u64 d3, d3, d4 + vadd.u64 d0, d0, M_\s8 + vadd.u64 d1, d1, M_\s10 + vadd.u64 d2, d2, M_\s12 + vadd.u64 d3, d3, M_\s14 + + // d = ror64(d ^ a, 32); + veor d15, d15, d0 + veor d12, d12, d1 + veor d13, d13, d2 + veor d14, d14, d3 + vrev64.32 d15, d15 + vrev64.32 d12, d12 + vrev64.32 d13, d13 + vrev64.32 d14, d14 + + // c += d; + vadd.u64 d10, d10, d15 + vadd.u64 d11, d11, d12 + vadd.u64 d8, d8, d13 + vadd.u64 d9, d9, d14 + + // b = ror64(b ^ c, 24); + vld1.8 {M_0}, [ROR24_TABLE, :64] + veor d5, d5, d10 + veor d6, d6, d11 + veor d7, d7, d8 + veor d4, d4, d9 + vtbl.8 d5, {d5}, M_0 + vtbl.8 d6, {d6}, M_0 + vtbl.8 d7, {d7}, M_0 + vtbl.8 d4, {d4}, M_0 + + // a += b + m[blake2b_sigma[r][2*i + 1]]; +.if \s9 == 0 || \s11 == 0 || \s13 == 0 || \s15 == 0 + vld1.8 {M_0}, [sp, :64] +.endif + vadd.u64 d0, d0, d5 + vadd.u64 d1, d1, d6 + vadd.u64 d2, d2, d7 + vadd.u64 d3, d3, d4 + vadd.u64 d0, d0, M_\s9 + vadd.u64 d1, d1, M_\s11 + vadd.u64 d2, d2, M_\s13 + vadd.u64 d3, d3, M_\s15 + + // d = ror64(d ^ a, 16); + vld1.8 {M_0}, [ROR16_TABLE, :64] + veor d15, d15, d0 + veor d12, d12, d1 + veor d13, d13, d2 + veor d14, d14, d3 + vtbl.8 d12, {d12}, M_0 + vtbl.8 d13, {d13}, M_0 + vtbl.8 d14, {d14}, M_0 + vtbl.8 d15, {d15}, M_0 + + // c += d; + vadd.u64 d10, d10, d15 + vadd.u64 d11, d11, d12 + vadd.u64 d8, d8, d13 + vadd.u64 d9, d9, d14 + + // b = ror64(b ^ c, 63); + veor d16, d4, d9 + veor d17, d5, d10 + veor d18, d6, d11 + veor d19, d7, d8 + vshr.u64 q2, q8, #63 + vshr.u64 q3, q9, #63 + vsli.u64 q2, q8, #1 + vsli.u64 q3, q9, #1 + // Reloading q8-q9 can be skipped on the final round. +.if ! \final + vld1.8 {q8-q9}, [sp, :256] +.endif +.endm + +// +// void blake2b_compress_neon(struct blake2b_state *state, +// const u8 *block, size_t nblocks, u32 inc); +// +// Only the first three fields of struct blake2b_state are used: +// u64 h[8]; (inout) +// u64 t[2]; (inout) +// u64 f[2]; (in) +// + .align 5 +ENTRY(blake2b_compress_neon) + push {r4-r10} + + // Allocate a 32-byte stack buffer that is 32-byte aligned. + mov ORIG_SP, sp + sub ip, sp, #32 + bic ip, ip, #31 + mov sp, ip + + adr ROR24_TABLE, .Lror24_table + adr ROR16_TABLE, .Lror16_table + + mov ip, STATE + vld1.64 {q0-q1}, [ip]! // Load h[0..3] + vld1.64 {q2-q3}, [ip]! // Load h[4..7] +.Lnext_block: + adr r10, .Lblake2b_IV + vld1.64 {q14-q15}, [ip] // Load t[0..1] and f[0..1] + vld1.64 {q4-q5}, [r10]! // Load IV[0..3] + vmov r7, r8, d28 // Copy t[0] to (r7, r8) + vld1.64 {q6-q7}, [r10] // Load IV[4..7] + adds r7, r7, INC // Increment counter + bcs .Lslow_inc_ctr + vmov.i32 d28[0], r7 + vst1.64 {d28}, [ip] // Update t[0] +.Linc_ctr_done: + + // Load the next message block and finish initializing the state matrix + // 'v'. Fortunately, there are exactly enough NEON registers to fit the + // entire state matrix in q0-q7 and the entire message block in q8-15. + // + // However, _blake2b_round also needs some extra registers for rotates, + // so we have to spill some registers. It's better to spill the message + // registers than the state registers, as the message doesn't change. + // Therefore we store a copy of the first 32 bytes of the message block + // (q8-q9) in an aligned buffer on the stack so that they can be + // reloaded when needed. (We could just reload directly from the + // message buffer, but it's faster to use aligned loads.) + vld1.8 {q8-q9}, [BLOCK]! + veor q6, q6, q14 // v[12..13] = IV[4..5] ^ t[0..1] + vld1.8 {q10-q11}, [BLOCK]! + veor q7, q7, q15 // v[14..15] = IV[6..7] ^ f[0..1] + vld1.8 {q12-q13}, [BLOCK]! + vst1.8 {q8-q9}, [sp, :256] + mov ip, STATE + vld1.8 {q14-q15}, [BLOCK]! + + // Execute the rounds. Each round is provided the order in which it + // needs to use the message words. + _blake2b_round 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + _blake2b_round 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 + _blake2b_round 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 + _blake2b_round 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 + _blake2b_round 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 + _blake2b_round 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 + _blake2b_round 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 + _blake2b_round 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 + _blake2b_round 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 + _blake2b_round 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 + _blake2b_round 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + _blake2b_round 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 \ + final=1 + + // Fold the final state matrix into the hash chaining value: + // + // for (i = 0; i < 8; i++) + // h[i] ^= v[i] ^ v[i + 8]; + // + vld1.64 {q8-q9}, [ip]! // Load old h[0..3] + veor q0, q0, q4 // v[0..1] ^= v[8..9] + veor q1, q1, q5 // v[2..3] ^= v[10..11] + vld1.64 {q10-q11}, [ip] // Load old h[4..7] + veor q2, q2, q6 // v[4..5] ^= v[12..13] + veor q3, q3, q7 // v[6..7] ^= v[14..15] + veor q0, q0, q8 // v[0..1] ^= h[0..1] + veor q1, q1, q9 // v[2..3] ^= h[2..3] + mov ip, STATE + subs NBLOCKS, NBLOCKS, #1 // nblocks-- + vst1.64 {q0-q1}, [ip]! // Store new h[0..3] + veor q2, q2, q10 // v[4..5] ^= h[4..5] + veor q3, q3, q11 // v[6..7] ^= h[6..7] + vst1.64 {q2-q3}, [ip]! // Store new h[4..7] + + // Advance to the next block, if there is one. + bne .Lnext_block // nblocks != 0? + + mov sp, ORIG_SP + pop {r4-r10} + mov pc, lr + +.Lslow_inc_ctr: + // Handle the case where the counter overflowed its low 32 bits, by + // carrying the overflow bit into the full 128-bit counter. + vmov r9, r10, d29 + adcs r8, r8, #0 + adcs r9, r9, #0 + adc r10, r10, #0 + vmov d28, r7, r8 + vmov d29, r9, r10 + vst1.64 {q14}, [ip] // Update t[0] and t[1] + b .Linc_ctr_done +ENDPROC(blake2b_compress_neon) diff --git a/arch/arm/crypto/blake2b-neon-glue.c b/arch/arm/crypto/blake2b-neon-glue.c new file mode 100644 index 000000000000..34d73200e7fa --- /dev/null +++ b/arch/arm/crypto/blake2b-neon-glue.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * BLAKE2b digest algorithm, NEON accelerated + * + * Copyright 2020 Google LLC + */ + +#include +#include +#include + +#include +#include + +#include +#include + +asmlinkage void blake2b_compress_neon(struct blake2b_state *state, + const u8 *block, size_t nblocks, u32 inc); + +static void blake2b_compress_arch(struct blake2b_state *state, + const u8 *block, size_t nblocks, u32 inc) +{ + if (!crypto_simd_usable()) { + blake2b_compress_generic(state, block, nblocks, inc); + return; + } + + do { + const size_t blocks = min_t(size_t, nblocks, + SZ_4K / BLAKE2B_BLOCK_SIZE); + + kernel_neon_begin(); + blake2b_compress_neon(state, block, blocks, inc); + kernel_neon_end(); + + nblocks -= blocks; + block += blocks * BLAKE2B_BLOCK_SIZE; + } while (nblocks); +} + +static int crypto_blake2b_update_neon(struct shash_desc *desc, + const u8 *in, unsigned int inlen) +{ + return crypto_blake2b_update(desc, in, inlen, blake2b_compress_arch); +} + +static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out) +{ + return crypto_blake2b_final(desc, out, blake2b_compress_arch); +} + +#define BLAKE2B_ALG(name, driver_name, digest_size) \ + { \ + .base.cra_name = name, \ + .base.cra_driver_name = driver_name, \ + .base.cra_priority = 200, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_blocksize = BLAKE2B_BLOCK_SIZE, \ + .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), \ + .base.cra_module = THIS_MODULE, \ + .digestsize = digest_size, \ + .setkey = crypto_blake2b_setkey, \ + .init = crypto_blake2b_init, \ + .update = crypto_blake2b_update_neon, \ + .final = crypto_blake2b_final_neon, \ + .descsize = sizeof(struct blake2b_state), \ + } + +static struct shash_alg blake2b_neon_algs[] = { + BLAKE2B_ALG("blake2b-160", "blake2b-160-neon", BLAKE2B_160_HASH_SIZE), + BLAKE2B_ALG("blake2b-256", "blake2b-256-neon", BLAKE2B_256_HASH_SIZE), + BLAKE2B_ALG("blake2b-384", "blake2b-384-neon", BLAKE2B_384_HASH_SIZE), + BLAKE2B_ALG("blake2b-512", "blake2b-512-neon", BLAKE2B_512_HASH_SIZE), +}; + +static int __init blake2b_neon_mod_init(void) +{ + if (!(elf_hwcap & HWCAP_NEON)) + return -ENODEV; + + return crypto_register_shashes(blake2b_neon_algs, + ARRAY_SIZE(blake2b_neon_algs)); +} + +static void __exit blake2b_neon_mod_exit(void) +{ + return crypto_unregister_shashes(blake2b_neon_algs, + ARRAY_SIZE(blake2b_neon_algs)); +} + +module_init(blake2b_neon_mod_init); +module_exit(blake2b_neon_mod_exit); + +MODULE_DESCRIPTION("BLAKE2b digest algorithm, NEON accelerated"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Eric Biggers "); +MODULE_ALIAS_CRYPTO("blake2b-160"); +MODULE_ALIAS_CRYPTO("blake2b-160-neon"); +MODULE_ALIAS_CRYPTO("blake2b-256"); +MODULE_ALIAS_CRYPTO("blake2b-256-neon"); +MODULE_ALIAS_CRYPTO("blake2b-384"); +MODULE_ALIAS_CRYPTO("blake2b-384-neon"); +MODULE_ALIAS_CRYPTO("blake2b-512"); +MODULE_ALIAS_CRYPTO("blake2b-512-neon"); diff --git a/arch/arm/crypto/blake2s-core.S b/arch/arm/crypto/blake2s-core.S new file mode 100644 index 000000000000..bed897e9a181 --- /dev/null +++ b/arch/arm/crypto/blake2s-core.S @@ -0,0 +1,285 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * BLAKE2s digest algorithm, ARM scalar implementation + * + * Copyright 2020 Google LLC + * + * Author: Eric Biggers + */ + +#include + + // Registers used to hold message words temporarily. There aren't + // enough ARM registers to hold the whole message block, so we have to + // load the words on-demand. + M_0 .req r12 + M_1 .req r14 + +// The BLAKE2s initialization vector +.Lblake2s_IV: + .word 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A + .word 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 + +.macro __ldrd a, b, src, offset +#if __LINUX_ARM_ARCH__ >= 6 + ldrd \a, \b, [\src, #\offset] +#else + ldr \a, [\src, #\offset] + ldr \b, [\src, #\offset + 4] +#endif +.endm + +.macro __strd a, b, dst, offset +#if __LINUX_ARM_ARCH__ >= 6 + strd \a, \b, [\dst, #\offset] +#else + str \a, [\dst, #\offset] + str \b, [\dst, #\offset + 4] +#endif +.endm + +// Execute a quarter-round of BLAKE2s by mixing two columns or two diagonals. +// (a0, b0, c0, d0) and (a1, b1, c1, d1) give the registers containing the two +// columns/diagonals. s0-s1 are the word offsets to the message words the first +// column/diagonal needs, and likewise s2-s3 for the second column/diagonal. +// M_0 and M_1 are free to use, and the message block can be found at sp + 32. +// +// Note that to save instructions, the rotations don't happen when the +// pseudocode says they should, but rather they are delayed until the values are +// used. See the comment above _blake2s_round(). +.macro _blake2s_quarterround a0, b0, c0, d0, a1, b1, c1, d1, s0, s1, s2, s3 + + ldr M_0, [sp, #32 + 4 * \s0] + ldr M_1, [sp, #32 + 4 * \s2] + + // a += b + m[blake2s_sigma[r][2*i + 0]]; + add \a0, \a0, \b0, ror #brot + add \a1, \a1, \b1, ror #brot + add \a0, \a0, M_0 + add \a1, \a1, M_1 + + // d = ror32(d ^ a, 16); + eor \d0, \a0, \d0, ror #drot + eor \d1, \a1, \d1, ror #drot + + // c += d; + add \c0, \c0, \d0, ror #16 + add \c1, \c1, \d1, ror #16 + + // b = ror32(b ^ c, 12); + eor \b0, \c0, \b0, ror #brot + eor \b1, \c1, \b1, ror #brot + + ldr M_0, [sp, #32 + 4 * \s1] + ldr M_1, [sp, #32 + 4 * \s3] + + // a += b + m[blake2s_sigma[r][2*i + 1]]; + add \a0, \a0, \b0, ror #12 + add \a1, \a1, \b1, ror #12 + add \a0, \a0, M_0 + add \a1, \a1, M_1 + + // d = ror32(d ^ a, 8); + eor \d0, \a0, \d0, ror#16 + eor \d1, \a1, \d1, ror#16 + + // c += d; + add \c0, \c0, \d0, ror#8 + add \c1, \c1, \d1, ror#8 + + // b = ror32(b ^ c, 7); + eor \b0, \c0, \b0, ror#12 + eor \b1, \c1, \b1, ror#12 +.endm + +// Execute one round of BLAKE2s by updating the state matrix v[0..15]. v[0..9] +// are in r0..r9. The stack pointer points to 8 bytes of scratch space for +// spilling v[8..9], then to v[9..15], then to the message block. r10-r12 and +// r14 are free to use. The macro arguments s0-s15 give the order in which the +// message words are used in this round. +// +// All rotates are performed using the implicit rotate operand accepted by the +// 'add' and 'eor' instructions. This is faster than using explicit rotate +// instructions. To make this work, we allow the values in the second and last +// rows of the BLAKE2s state matrix (rows 'b' and 'd') to temporarily have the +// wrong rotation amount. The rotation amount is then fixed up just in time +// when the values are used. 'brot' is the number of bits the values in row 'b' +// need to be rotated right to arrive at the correct values, and 'drot' +// similarly for row 'd'. (brot, drot) start out as (0, 0) but we make it such +// that they end up as (7, 8) after every round. +.macro _blake2s_round s0, s1, s2, s3, s4, s5, s6, s7, \ + s8, s9, s10, s11, s12, s13, s14, s15 + + // Mix first two columns: + // (v[0], v[4], v[8], v[12]) and (v[1], v[5], v[9], v[13]). + __ldrd r10, r11, sp, 16 // load v[12] and v[13] + _blake2s_quarterround r0, r4, r8, r10, r1, r5, r9, r11, \ + \s0, \s1, \s2, \s3 + __strd r8, r9, sp, 0 + __strd r10, r11, sp, 16 + + // Mix second two columns: + // (v[2], v[6], v[10], v[14]) and (v[3], v[7], v[11], v[15]). + __ldrd r8, r9, sp, 8 // load v[10] and v[11] + __ldrd r10, r11, sp, 24 // load v[14] and v[15] + _blake2s_quarterround r2, r6, r8, r10, r3, r7, r9, r11, \ + \s4, \s5, \s6, \s7 + str r10, [sp, #24] // store v[14] + // v[10], v[11], and v[15] are used below, so no need to store them yet. + + .set brot, 7 + .set drot, 8 + + // Mix first two diagonals: + // (v[0], v[5], v[10], v[15]) and (v[1], v[6], v[11], v[12]). + ldr r10, [sp, #16] // load v[12] + _blake2s_quarterround r0, r5, r8, r11, r1, r6, r9, r10, \ + \s8, \s9, \s10, \s11 + __strd r8, r9, sp, 8 + str r11, [sp, #28] + str r10, [sp, #16] + + // Mix second two diagonals: + // (v[2], v[7], v[8], v[13]) and (v[3], v[4], v[9], v[14]). + __ldrd r8, r9, sp, 0 // load v[8] and v[9] + __ldrd r10, r11, sp, 20 // load v[13] and v[14] + _blake2s_quarterround r2, r7, r8, r10, r3, r4, r9, r11, \ + \s12, \s13, \s14, \s15 + __strd r10, r11, sp, 20 +.endm + +// +// void blake2s_compress_arch(struct blake2s_state *state, +// const u8 *block, size_t nblocks, u32 inc); +// +// Only the first three fields of struct blake2s_state are used: +// u32 h[8]; (inout) +// u32 t[2]; (inout) +// u32 f[2]; (in) +// + .align 5 +ENTRY(blake2s_compress_arch) + push {r0-r2,r4-r11,lr} // keep this an even number + +.Lnext_block: + // r0 is 'state' + // r1 is 'block' + // r3 is 'inc' + + // Load and increment the counter t[0..1]. + __ldrd r10, r11, r0, 32 + adds r10, r10, r3 + adc r11, r11, #0 + __strd r10, r11, r0, 32 + + // _blake2s_round is very short on registers, so copy the message block + // to the stack to save a register during the rounds. This also has the + // advantage that misalignment only needs to be dealt with in one place. + sub sp, sp, #64 + mov r12, sp + tst r1, #3 + bne .Lcopy_block_misaligned + ldmia r1!, {r2-r9} + stmia r12!, {r2-r9} + ldmia r1!, {r2-r9} + stmia r12, {r2-r9} +.Lcopy_block_done: + str r1, [sp, #68] // Update message pointer + + // Calculate v[8..15]. Push v[9..15] onto the stack, and leave space + // for spilling v[8..9]. Leave v[8..9] in r8-r9. + mov r14, r0 // r14 = state + adr r12, .Lblake2s_IV + ldmia r12!, {r8-r9} // load IV[0..1] + __ldrd r0, r1, r14, 40 // load f[0..1] + ldm r12, {r2-r7} // load IV[3..7] + eor r4, r4, r10 // v[12] = IV[4] ^ t[0] + eor r5, r5, r11 // v[13] = IV[5] ^ t[1] + eor r6, r6, r0 // v[14] = IV[6] ^ f[0] + eor r7, r7, r1 // v[15] = IV[7] ^ f[1] + push {r2-r7} // push v[9..15] + sub sp, sp, #8 // leave space for v[8..9] + + // Load h[0..7] == v[0..7]. + ldm r14, {r0-r7} + + // Execute the rounds. Each round is provided the order in which it + // needs to use the message words. + .set brot, 0 + .set drot, 0 + _blake2s_round 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + _blake2s_round 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 + _blake2s_round 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 + _blake2s_round 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 + _blake2s_round 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 + _blake2s_round 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 + _blake2s_round 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 + _blake2s_round 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 + _blake2s_round 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 + _blake2s_round 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 + + // Fold the final state matrix into the hash chaining value: + // + // for (i = 0; i < 8; i++) + // h[i] ^= v[i] ^ v[i + 8]; + // + ldr r14, [sp, #96] // r14 = &h[0] + add sp, sp, #8 // v[8..9] are already loaded. + pop {r10-r11} // load v[10..11] + eor r0, r0, r8 + eor r1, r1, r9 + eor r2, r2, r10 + eor r3, r3, r11 + ldm r14, {r8-r11} // load h[0..3] + eor r0, r0, r8 + eor r1, r1, r9 + eor r2, r2, r10 + eor r3, r3, r11 + stmia r14!, {r0-r3} // store new h[0..3] + ldm r14, {r0-r3} // load old h[4..7] + pop {r8-r11} // load v[12..15] + eor r0, r0, r4, ror #brot + eor r1, r1, r5, ror #brot + eor r2, r2, r6, ror #brot + eor r3, r3, r7, ror #brot + eor r0, r0, r8, ror #drot + eor r1, r1, r9, ror #drot + eor r2, r2, r10, ror #drot + eor r3, r3, r11, ror #drot + add sp, sp, #64 // skip copy of message block + stm r14, {r0-r3} // store new h[4..7] + + // Advance to the next block, if there is one. Note that if there are + // multiple blocks, then 'inc' (the counter increment amount) must be + // 64. So we can simply set it to 64 without re-loading it. + ldm sp, {r0, r1, r2} // load (state, block, nblocks) + mov r3, #64 // set 'inc' + subs r2, r2, #1 // nblocks-- + str r2, [sp, #8] + bne .Lnext_block // nblocks != 0? + + pop {r0-r2,r4-r11,pc} + + // The next message block (pointed to by r1) isn't 4-byte aligned, so it + // can't be loaded using ldmia. Copy it to the stack buffer (pointed to + // by r12) using an alternative method. r2-r9 are free to use. +.Lcopy_block_misaligned: + mov r2, #64 +1: +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + ldr r3, [r1], #4 +#else + ldrb r3, [r1, #0] + ldrb r4, [r1, #1] + ldrb r5, [r1, #2] + ldrb r6, [r1, #3] + add r1, r1, #4 + orr r3, r3, r4, lsl #8 + orr r3, r3, r5, lsl #16 + orr r3, r3, r6, lsl #24 +#endif + subs r2, r2, #4 + str r3, [r12], #4 + bne 1b + b .Lcopy_block_done +ENDPROC(blake2s_compress_arch) diff --git a/arch/arm/crypto/blake2s-glue.c b/arch/arm/crypto/blake2s-glue.c new file mode 100644 index 000000000000..f2cc1e5fc9ec --- /dev/null +++ b/arch/arm/crypto/blake2s-glue.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * BLAKE2s digest algorithm, ARM scalar implementation + * + * Copyright 2020 Google LLC + */ + +#include +#include + +#include + +/* defined in blake2s-core.S */ +EXPORT_SYMBOL(blake2s_compress_arch); + +static int crypto_blake2s_update_arm(struct shash_desc *desc, + const u8 *in, unsigned int inlen) +{ + return crypto_blake2s_update(desc, in, inlen, blake2s_compress_arch); +} + +static int crypto_blake2s_final_arm(struct shash_desc *desc, u8 *out) +{ + return crypto_blake2s_final(desc, out, blake2s_compress_arch); +} + +#define BLAKE2S_ALG(name, driver_name, digest_size) \ + { \ + .base.cra_name = name, \ + .base.cra_driver_name = driver_name, \ + .base.cra_priority = 200, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \ + .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \ + .base.cra_module = THIS_MODULE, \ + .digestsize = digest_size, \ + .setkey = crypto_blake2s_setkey, \ + .init = crypto_blake2s_init, \ + .update = crypto_blake2s_update_arm, \ + .final = crypto_blake2s_final_arm, \ + .descsize = sizeof(struct blake2s_state), \ + } + +static struct shash_alg blake2s_arm_algs[] = { + BLAKE2S_ALG("blake2s-128", "blake2s-128-arm", BLAKE2S_128_HASH_SIZE), + BLAKE2S_ALG("blake2s-160", "blake2s-160-arm", BLAKE2S_160_HASH_SIZE), + BLAKE2S_ALG("blake2s-224", "blake2s-224-arm", BLAKE2S_224_HASH_SIZE), + BLAKE2S_ALG("blake2s-256", "blake2s-256-arm", BLAKE2S_256_HASH_SIZE), +}; + +static int __init blake2s_arm_mod_init(void) +{ + return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? + crypto_register_shashes(blake2s_arm_algs, + ARRAY_SIZE(blake2s_arm_algs)) : 0; +} + +static void __exit blake2s_arm_mod_exit(void) +{ + if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) + crypto_unregister_shashes(blake2s_arm_algs, + ARRAY_SIZE(blake2s_arm_algs)); +} + +module_init(blake2s_arm_mod_init); +module_exit(blake2s_arm_mod_exit); + +MODULE_DESCRIPTION("BLAKE2s digest algorithm, ARM scalar implementation"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Eric Biggers "); +MODULE_ALIAS_CRYPTO("blake2s-128"); +MODULE_ALIAS_CRYPTO("blake2s-128-arm"); +MODULE_ALIAS_CRYPTO("blake2s-160"); +MODULE_ALIAS_CRYPTO("blake2s-160-arm"); +MODULE_ALIAS_CRYPTO("blake2s-224"); +MODULE_ALIAS_CRYPTO("blake2s-224-arm"); +MODULE_ALIAS_CRYPTO("blake2s-256"); +MODULE_ALIAS_CRYPTO("blake2s-256-arm"); diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 6ed30421f697..e2b1fd558bf3 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -578,4 +578,21 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) __adldst_l str, \src, \sym, \tmp, \cond .endm + /* + * rev_l - byte-swap a 32-bit value + * + * @val: source/destination register + * @tmp: scratch register + */ + .macro rev_l, val:req, tmp:req + .if __LINUX_ARM_ARCH__ < 6 + eor \tmp, \val, \val, ror #16 + bic \tmp, \tmp, #0x00ff0000 + mov \val, \val, ror #8 + eor \val, \val, \tmp, lsr #8 + .else + rev \val, \val + .endif + .endm + #endif /* __ASM_ASSEMBLER_H__ */ diff --git a/arch/arm/include/asm/hardware/locomo.h b/arch/arm/include/asm/hardware/locomo.h index f8712e3c29cf..246a3de25931 100644 --- a/arch/arm/include/asm/hardware/locomo.h +++ b/arch/arm/include/asm/hardware/locomo.h @@ -188,7 +188,7 @@ struct locomo_driver { struct device_driver drv; unsigned int devid; int (*probe)(struct locomo_dev *); - int (*remove)(struct locomo_dev *); + void (*remove)(struct locomo_dev *); }; #define LOCOMO_DRV(_d) container_of((_d), struct locomo_driver, drv) diff --git a/arch/arm/include/asm/hardware/sa1111.h b/arch/arm/include/asm/hardware/sa1111.h index d134b9a5ff94..2e70db6f22ea 100644 --- a/arch/arm/include/asm/hardware/sa1111.h +++ b/arch/arm/include/asm/hardware/sa1111.h @@ -403,7 +403,7 @@ struct sa1111_driver { struct device_driver drv; unsigned int devid; int (*probe)(struct sa1111_dev *); - int (*remove)(struct sa1111_dev *); + void (*remove)(struct sa1111_dev *); }; #define SA1111_DRV(_d) container_of((_d), struct sa1111_driver, drv) diff --git a/arch/arm/include/debug/sti.S b/arch/arm/include/debug/sti.S index 72d052511890..dc796ac2ac57 100644 --- a/arch/arm/include/debug/sti.S +++ b/arch/arm/include/debug/sti.S @@ -6,28 +6,6 @@ * Copyright (C) 2013 STMicroelectronics (R&D) Limited. */ -#define STIH41X_COMMS_BASE 0xfed00000 -#define STIH41X_ASC2_BASE (STIH41X_COMMS_BASE+0x32000) - -#define STIH41X_SBC_LPM_BASE 0xfe400000 -#define STIH41X_SBC_COMMS_BASE (STIH41X_SBC_LPM_BASE + 0x100000) -#define STIH41X_SBC_ASC1_BASE (STIH41X_SBC_COMMS_BASE + 0x31000) - - -#define VIRT_ADDRESS(x) (x - 0x1000000) - -#if IS_ENABLED(CONFIG_STIH41X_DEBUG_ASC2) -#define DEBUG_LL_UART_BASE STIH41X_ASC2_BASE -#endif - -#if IS_ENABLED(CONFIG_STIH41X_DEBUG_SBC_ASC1) -#define DEBUG_LL_UART_BASE STIH41X_SBC_ASC1_BASE -#endif - -#ifndef DEBUG_LL_UART_BASE -#error "DEBUG UART is not Configured" -#endif - #define ASC_TX_BUF_OFF 0x04 #define ASC_CTRL_OFF 0x0c #define ASC_STA_OFF 0x14 @@ -37,8 +15,8 @@ .macro addruart, rp, rv, tmp - ldr \rp, =DEBUG_LL_UART_BASE @ physical base - ldr \rv, =VIRT_ADDRESS(DEBUG_LL_UART_BASE) @ virt base + ldr \rp, =CONFIG_DEBUG_UART_PHYS @ physical base + ldr \rv, =CONFIG_DEBUG_UART_VIRT @ virt base .endm .macro senduart,rd,rx diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index ee3aee69e444..5199a2bb4111 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -243,7 +243,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, thread->cpu_domain = get_domain(); #endif - if (likely(!(p->flags & PF_KTHREAD))) { + if (likely(!(p->flags & (PF_KTHREAD | PF_IO_WORKER)))) { *childregs = *current_pt_regs(); childregs->ARM_r0 = 0; if (stack_start) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 5c48eb4fd0e5..74679240a9d8 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -540,12 +540,9 @@ void show_ipi_list(struct seq_file *p, int prec) unsigned int cpu, i; for (i = 0; i < NR_IPI; i++) { - unsigned int irq; - if (!ipi_desc[i]) continue; - irq = irq_desc_get_irq(ipi_desc[i]); seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); for_each_online_cpu(cpu) diff --git a/arch/arm/kernel/sys_oabi-compat.c b/arch/arm/kernel/sys_oabi-compat.c index 0203e545bbc8..075a2e0ed2c1 100644 --- a/arch/arm/kernel/sys_oabi-compat.c +++ b/arch/arm/kernel/sys_oabi-compat.c @@ -248,6 +248,7 @@ struct oabi_epoll_event { __u64 data; } __attribute__ ((packed,aligned(4))); +#ifdef CONFIG_EPOLL asmlinkage long sys_oabi_epoll_ctl(int epfd, int op, int fd, struct oabi_epoll_event __user *event) { @@ -298,6 +299,20 @@ asmlinkage long sys_oabi_epoll_wait(int epfd, kfree(kbuf); return err ? -EFAULT : ret; } +#else +asmlinkage long sys_oabi_epoll_ctl(int epfd, int op, int fd, + struct oabi_epoll_event __user *event) +{ + return -EINVAL; +} + +asmlinkage long sys_oabi_epoll_wait(int epfd, + struct oabi_epoll_event __user *events, + int maxevents, int timeout) +{ + return -EINVAL; +} +#endif struct oabi_sembuf { unsigned short sem_num; diff --git a/arch/arm/mach-imx/common.h b/arch/arm/mach-imx/common.h index 2d76e2c6c99e..2b004cc4f95e 100644 --- a/arch/arm/mach-imx/common.h +++ b/arch/arm/mach-imx/common.h @@ -85,7 +85,6 @@ void imx_anatop_pre_suspend(void); void imx_anatop_post_resume(void); int imx6_set_lpm(enum mxc_cpu_pwr_mode mode); void imx6_set_int_mem_clk_lpm(bool enable); -void imx6sl_set_wait_clk(bool enter); int imx_mmdc_get_ddr_type(void); int imx7ulp_set_lpm(enum ulp_cpu_pwr_mode mode); diff --git a/arch/arm/mach-imx/cpuidle-imx6sl.c b/arch/arm/mach-imx/cpuidle-imx6sl.c index 4521e5352bf6..b86ffbeb28e4 100644 --- a/arch/arm/mach-imx/cpuidle-imx6sl.c +++ b/arch/arm/mach-imx/cpuidle-imx6sl.c @@ -3,6 +3,7 @@ * Copyright (C) 2014 Freescale Semiconductor, Inc. */ +#include #include #include #include diff --git a/arch/arm/mach-imx/pm-imx6.c b/arch/arm/mach-imx/pm-imx6.c index 40c74b4c4d73..9244437cb1b9 100644 --- a/arch/arm/mach-imx/pm-imx6.c +++ b/arch/arm/mach-imx/pm-imx6.c @@ -4,6 +4,7 @@ * Copyright 2011 Linaro Ltd. */ +#include #include #include #include diff --git a/arch/arm/mach-ixp4xx/Kconfig b/arch/arm/mach-ixp4xx/Kconfig index f7211b57b1e7..165c184801e1 100644 --- a/arch/arm/mach-ixp4xx/Kconfig +++ b/arch/arm/mach-ixp4xx/Kconfig @@ -13,7 +13,6 @@ config MACH_IXP4XX_OF select I2C select I2C_IOP3XX select PCI - select TIMER_OF select USE_OF help Say 'Y' here to support Device Tree-based IXP4xx platforms. diff --git a/arch/arm/mach-pxa/mioa701.c b/arch/arm/mach-pxa/mioa701.c index d3af80317f2d..a79f296e81e0 100644 --- a/arch/arm/mach-pxa/mioa701.c +++ b/arch/arm/mach-pxa/mioa701.c @@ -577,7 +577,6 @@ static struct platform_device power_dev = { static struct wm97xx_batt_pdata mioa701_battery_data = { .batt_aux = WM97XX_AUX_ID1, .temp_aux = -1, - .charge_gpio = -1, .min_voltage = 0xc00, .max_voltage = 0xfc0, .batt_tech = POWER_SUPPLY_TECHNOLOGY_LION, diff --git a/arch/arm/mach-pxa/palm27x.c b/arch/arm/mach-pxa/palm27x.c index 0d246a1aebbc..6230381a7ca0 100644 --- a/arch/arm/mach-pxa/palm27x.c +++ b/arch/arm/mach-pxa/palm27x.c @@ -212,7 +212,6 @@ void __init palm27x_irda_init(int pwdn) static struct wm97xx_batt_pdata palm27x_batt_pdata = { .batt_aux = WM97XX_AUX_ID3, .temp_aux = WM97XX_AUX_ID2, - .charge_gpio = -1, .batt_mult = 1000, .batt_div = 414, .temp_mult = 1, diff --git a/arch/arm/mach-pxa/palmte2.c b/arch/arm/mach-pxa/palmte2.c index e3bcf58b4e63..a2b10db4aacc 100644 --- a/arch/arm/mach-pxa/palmte2.c +++ b/arch/arm/mach-pxa/palmte2.c @@ -273,7 +273,6 @@ static struct platform_device power_supply = { static struct wm97xx_batt_pdata palmte2_batt_pdata = { .batt_aux = WM97XX_AUX_ID3, .temp_aux = WM97XX_AUX_ID2, - .charge_gpio = -1, .max_voltage = PALMTE2_BAT_MAX_VOLTAGE, .min_voltage = PALMTE2_BAT_MIN_VOLTAGE, .batt_mult = 1000, diff --git a/arch/arm/mach-pxa/z2.c b/arch/arm/mach-pxa/z2.c index 21fd76bb09cd..8e74fbb0a96e 100644 --- a/arch/arm/mach-pxa/z2.c +++ b/arch/arm/mach-pxa/z2.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -488,7 +487,6 @@ static struct z2_battery_info batt_chip_info = { .batt_I2C_bus = 0, .batt_I2C_addr = 0x55, .batt_I2C_reg = 2, - .charge_gpio = GPIO0_ZIPITZ2_AC_DETECT, .min_voltage = 3475000, .max_voltage = 4190000, .batt_div = 59, @@ -497,9 +495,19 @@ static struct z2_battery_info batt_chip_info = { .batt_name = "Z2", }; +static struct gpiod_lookup_table z2_battery_gpio_table = { + .dev_id = "aer915", + .table = { + GPIO_LOOKUP("gpio-pxa", GPIO0_ZIPITZ2_AC_DETECT, + NULL, GPIO_ACTIVE_HIGH), + { }, + }, +}; + static struct i2c_board_info __initdata z2_i2c_board_info[] = { { I2C_BOARD_INFO("aer915", 0x55), + .dev_name = "aer915", .platform_data = &batt_chip_info, }, { I2C_BOARD_INFO("wm8750", 0x1b), @@ -510,6 +518,7 @@ static struct i2c_board_info __initdata z2_i2c_board_info[] = { static void __init z2_i2c_init(void) { pxa_set_i2c_info(NULL); + gpiod_add_lookup_table(&z2_battery_gpio_table); i2c_register_board_info(0, ARRAY_AND_SIZE(z2_i2c_board_info)); } #else @@ -578,8 +587,13 @@ static struct pxa2xx_spi_chip lms283_chip_info = { .gpio_cs = GPIO88_ZIPITZ2_LCD_CS, }; -static const struct lms283gf05_pdata lms283_pdata = { - .reset_gpio = GPIO19_ZIPITZ2_LCD_RESET, +static struct gpiod_lookup_table lms283_gpio_table = { + .dev_id = "spi2.0", /* SPI bus 2 chip select 0 */ + .table = { + GPIO_LOOKUP("gpio-pxa", GPIO19_ZIPITZ2_LCD_RESET, + "reset", GPIO_ACTIVE_LOW), + { }, + }, }; static struct spi_board_info spi_board_info[] __initdata = { @@ -595,7 +609,6 @@ static struct spi_board_info spi_board_info[] __initdata = { { .modalias = "lms283gf05", .controller_data = &lms283_chip_info, - .platform_data = &lms283_pdata, .max_speed_hz = 400000, .bus_num = 2, .chip_select = 0, @@ -615,6 +628,7 @@ static void __init z2_spi_init(void) { pxa2xx_set_spi_info(1, &pxa_ssp1_master_info); pxa2xx_set_spi_info(2, &pxa_ssp2_master_info); + gpiod_add_lookup_table(&lms283_gpio_table); spi_register_board_info(spi_board_info, ARRAY_SIZE(spi_board_info)); } #else diff --git a/arch/arm/mach-sa1100/collie.c b/arch/arm/mach-sa1100/collie.c index d4e89a02c8c8..14c33ed05318 100644 --- a/arch/arm/mach-sa1100/collie.c +++ b/arch/arm/mach-sa1100/collie.c @@ -224,18 +224,12 @@ static int collie_uart_probe(struct locomo_dev *dev) return 0; } -static int collie_uart_remove(struct locomo_dev *dev) -{ - return 0; -} - static struct locomo_driver collie_uart_driver = { .drv = { .name = "collie_uart", }, .devid = LOCOMO_DEVID_UART, .probe = collie_uart_probe, - .remove = collie_uart_remove, }; static int __init collie_uart_init(void) diff --git a/arch/arm/mach-spear/generic.h b/arch/arm/mach-spear/generic.h index 25b4c5e66e39..8ec2b92dca19 100644 --- a/arch/arm/mach-spear/generic.h +++ b/arch/arm/mach-spear/generic.h @@ -43,16 +43,4 @@ void spear13xx_cpu_die(unsigned int cpu); extern const struct smp_operations spear13xx_smp_ops; -#ifdef CONFIG_MACH_SPEAR1310 -void __init spear1310_clk_init(void __iomem *misc_base, void __iomem *ras_base); -#else -static inline void spear1310_clk_init(void __iomem *misc_base, void __iomem *ras_base) {} -#endif - -#ifdef CONFIG_MACH_SPEAR1340 -void __init spear1340_clk_init(void __iomem *misc_base); -#else -static inline void spear1340_clk_init(void __iomem *misc_base) {} -#endif - #endif /* __MACH_GENERIC_H */ diff --git a/arch/arm/mach-spear/spear13xx.c b/arch/arm/mach-spear/spear13xx.c index 31c43cabf362..74d1ca2a529a 100644 --- a/arch/arm/mach-spear/spear13xx.c +++ b/arch/arm/mach-spear/spear13xx.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include diff --git a/arch/arm/mm/dump.c b/arch/arm/mm/dump.c index c18d23a5e5f1..93ff0097f00b 100644 --- a/arch/arm/mm/dump.c +++ b/arch/arm/mm/dump.c @@ -19,6 +19,10 @@ #include static struct addr_marker address_markers[] = { +#ifdef CONFIG_KASAN + { KASAN_SHADOW_START, "Kasan shadow start"}, + { KASAN_SHADOW_END, "Kasan shadow end"}, +#endif { MODULES_VADDR, "Modules" }, { PAGE_OFFSET, "Kernel Mapping" }, { 0, "vmalloc() Area" }, @@ -429,8 +433,11 @@ static void ptdump_initialize(void) if (pg_level[i].bits[j].nx_bit) pg_level[i].nx_bit = &pg_level[i].bits[j]; } - +#ifdef CONFIG_KASAN + address_markers[4].start_address = VMALLOC_START; +#else address_markers[2].start_address = VMALLOC_START; +#endif } static struct ptdump_info kernel_ptdump_info = { diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index c06ebfbc48c4..a25b660c3017 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/arm/tools/Makefile b/arch/arm/tools/Makefile index 27d8beb7c941..3654f979851b 100644 --- a/arch/arm/tools/Makefile +++ b/arch/arm/tools/Makefile @@ -11,7 +11,7 @@ uapi := $(gen)/uapi/asm syshdr := $(srctree)/$(src)/syscallhdr.sh sysnr := $(srctree)/$(src)/syscallnr.sh systbl := $(srctree)/$(src)/syscalltbl.sh -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl gen-y := $(gen)/calls-oabi.S gen-y += $(gen)/calls-eabi.S diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl index 20e1170e2e0a..dcc1191291a2 100644 --- a/arch/arm/tools/syscall.tbl +++ b/arch/arm/tools/syscall.tbl @@ -455,3 +455,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile index b558bee0e1f6..7c9e395b77f7 100644 --- a/arch/arm/vdso/Makefile +++ b/arch/arm/vdso/Makefile @@ -23,7 +23,6 @@ ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ -T obj-$(CONFIG_VDSO) += vdso.o -extra-$(CONFIG_VDSO) += vdso.lds CPPFLAGS_vdso.lds += -P -C -U$(ARCH) CFLAGS_REMOVE_vdso.o = -pg diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1b8cecdc028e..1f212b47a48a 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -73,6 +73,8 @@ config ARM64 select ARCH_SUPPORTS_DEBUG_PAGEALLOC select ARCH_SUPPORTS_MEMORY_FAILURE select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK + select ARCH_SUPPORTS_LTO_CLANG if CPU_LITTLE_ENDIAN + select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG) select ARCH_SUPPORTS_NUMA_BALANCING @@ -138,6 +140,7 @@ config ARM64 select HAVE_ARCH_KASAN if !(ARM64_16K_PAGES && ARM64_VA_BITS_48) select HAVE_ARCH_KASAN_SW_TAGS if HAVE_ARCH_KASAN select HAVE_ARCH_KASAN_HW_TAGS if (HAVE_ARCH_KASAN && ARM64_MTE) + select HAVE_ARCH_KFENCE select HAVE_ARCH_KGDB select HAVE_ARCH_MMAP_RND_BITS select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT @@ -162,6 +165,8 @@ config ARM64 select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS \ if $(cc-option,-fpatchable-function-entry=2) + select FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY \ + if DYNAMIC_FTRACE_WITH_REGS select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD @@ -994,6 +999,7 @@ config HOTPLUG_CPU # Common NUMA Features config NUMA bool "NUMA Memory Allocation and Scheduler Support" + select GENERIC_ARCH_NUMA select ACPI_NUMA if ACPI select OF_NUMA help @@ -1481,7 +1487,7 @@ config ARM64_PTR_AUTH depends on (CC_HAS_SIGN_RETURN_ADDRESS || CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC # Modern compilers insert a .note.gnu.property section note for PAC # which is only understood by binutils starting with version 2.33.1. - depends on LD_IS_LLD || LD_VERSION >= 233010000 || (CC_IS_GCC && GCC_VERSION < 90100) + depends on LD_IS_LLD || LD_VERSION >= 23301 || (CC_IS_GCC && GCC_VERSION < 90100) depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS) help diff --git a/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts b/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts index 26a4a5a6871e..2f0528d01299 100644 --- a/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts +++ b/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts @@ -232,7 +232,7 @@ &apps_rsc { pm8009-rpmh-regulators { - compatible = "qcom,pm8009-rpmh-regulators"; + compatible = "qcom,pm8009-1-rpmh-regulators"; qcom,pmic-id = "f"; vdd-s1-supply = <&vph_pwr>; @@ -241,6 +241,13 @@ vdd-l5-l6-supply = <&vreg_bob>; vdd-l7-supply = <&vreg_s4a_1p8>; + vreg_s2f_0p95: smps2 { + regulator-name = "vreg_s2f_0p95"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <952000>; + regulator-initial-mode = ; + }; + vreg_l1f_1p1: ldo1 { regulator-name = "vreg_l1f_1p1"; regulator-min-microvolt = <1104000>; diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 43c4280bfc4f..d612f633b771 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -770,7 +770,8 @@ CONFIG_SND_SOC_LPASS_VA_MACRO=m CONFIG_SND_SIMPLE_CARD=m CONFIG_SND_AUDIO_GRAPH_CARD=m CONFIG_HID_MULTITOUCH=m -CONFIG_I2C_HID=m +CONFIG_I2C_HID_ACPI=m +CONFIG_I2C_HID_OF=m CONFIG_USB_CONN_GPIO=m CONFIG_USB=y CONFIG_USB_OTG=y diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 34b8a89197be..17e735931a0c 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -24,6 +24,7 @@ #ifdef USE_V8_CRYPTO_EXTENSIONS #define MODE "ce" #define PRIO 300 +#define STRIDE 5 #define aes_expandkey ce_aes_expandkey #define aes_ecb_encrypt ce_aes_ecb_encrypt #define aes_ecb_decrypt ce_aes_ecb_decrypt @@ -41,6 +42,7 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); #else #define MODE "neon" #define PRIO 200 +#define STRIDE 4 #define aes_ecb_encrypt neon_aes_ecb_encrypt #define aes_ecb_decrypt neon_aes_ecb_decrypt #define aes_cbc_encrypt neon_aes_cbc_encrypt @@ -55,7 +57,7 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); #define aes_mac_update neon_aes_mac_update MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON"); #endif -#if defined(USE_V8_CRYPTO_EXTENSIONS) || !defined(CONFIG_CRYPTO_AES_ARM64_BS) +#if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) MODULE_ALIAS_CRYPTO("ecb(aes)"); MODULE_ALIAS_CRYPTO("cbc(aes)"); MODULE_ALIAS_CRYPTO("ctr(aes)"); @@ -87,7 +89,7 @@ asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], int rounds, int bytes, u8 const iv[]); asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks, u8 ctr[]); + int rounds, int bytes, u8 ctr[], u8 finalbuf[]); asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int bytes, u32 const rk2[], u8 iv[], @@ -103,9 +105,9 @@ asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int blocks, u8 iv[], u32 const rk2[]); -asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds, - int blocks, u8 dg[], int enc_before, - int enc_after); +asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds, + int blocks, u8 dg[], int enc_before, + int enc_after); struct crypto_aes_xts_ctx { struct crypto_aes_ctx key1; @@ -448,34 +450,36 @@ static int ctr_encrypt(struct skcipher_request *req) struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); int err, rounds = 6 + ctx->key_length / 4; struct skcipher_walk walk; - int blocks; err = skcipher_walk_virt(&walk, req, false); - while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { - kernel_neon_begin(); - aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, - ctx->key_enc, rounds, blocks, walk.iv); - kernel_neon_end(); - err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); - } - if (walk.nbytes) { - u8 __aligned(8) tail[AES_BLOCK_SIZE]; + while (walk.nbytes > 0) { + const u8 *src = walk.src.virt.addr; unsigned int nbytes = walk.nbytes; - u8 *tdst = walk.dst.virt.addr; - u8 *tsrc = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; + u8 buf[AES_BLOCK_SIZE]; + unsigned int tail; - /* - * Tell aes_ctr_encrypt() to process a tail block. - */ - blocks = -1; + if (unlikely(nbytes < AES_BLOCK_SIZE)) + src = memcpy(buf, src, nbytes); + else if (nbytes < walk.total) + nbytes &= ~(AES_BLOCK_SIZE - 1); kernel_neon_begin(); - aes_ctr_encrypt(tail, NULL, ctx->key_enc, rounds, - blocks, walk.iv); + aes_ctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes, + walk.iv, buf); kernel_neon_end(); - crypto_xor_cpy(tdst, tsrc, tail, nbytes); - err = skcipher_walk_done(&walk, 0); + + tail = nbytes % (STRIDE * AES_BLOCK_SIZE); + if (tail > 0 && tail < AES_BLOCK_SIZE) + /* + * The final partial block could not be returned using + * an overlapping store, so it was passed via buf[] + * instead. + */ + memcpy(dst + nbytes - tail, buf, tail); + + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); } return err; @@ -650,7 +654,7 @@ static int __maybe_unused xts_decrypt(struct skcipher_request *req) } static struct skcipher_alg aes_algs[] = { { -#if defined(USE_V8_CRYPTO_EXTENSIONS) || !defined(CONFIG_CRYPTO_AES_ARM64_BS) +#if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) .base = { .cra_name = "__ecb(aes)", .cra_driver_name = "__ecb-aes-" MODE, @@ -852,10 +856,17 @@ static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, int rounds = 6 + ctx->key_length / 4; if (crypto_simd_usable()) { - kernel_neon_begin(); - aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before, - enc_after); - kernel_neon_end(); + int rem; + + do { + kernel_neon_begin(); + rem = aes_mac_update(in, ctx->key_enc, rounds, blocks, + dg, enc_before, enc_after); + kernel_neon_end(); + in += (blocks - rem) * AES_BLOCK_SIZE; + blocks = rem; + enc_before = 0; + } while (blocks); } else { if (enc_before) aes_encrypt(ctx, dg, dg); diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index cf618d8f6cec..bbdb54702aa7 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -321,42 +321,76 @@ AES_FUNC_END(aes_cbc_cts_decrypt) /* * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, - * int blocks, u8 ctr[]) + * int bytes, u8 ctr[], u8 finalbuf[]) */ AES_FUNC_START(aes_ctr_encrypt) stp x29, x30, [sp, #-16]! mov x29, sp - enc_prepare w3, x2, x6 + enc_prepare w3, x2, x12 ld1 {vctr.16b}, [x5] - umov x6, vctr.d[1] /* keep swabbed ctr in reg */ - rev x6, x6 - cmn w6, w4 /* 32 bit overflow? */ - bcs .Lctrloop + umov x12, vctr.d[1] /* keep swabbed ctr in reg */ + rev x12, x12 + .LctrloopNx: - subs w4, w4, #MAX_STRIDE - bmi .Lctr1x - add w7, w6, #1 + add w7, w4, #15 + sub w4, w4, #MAX_STRIDE << 4 + lsr w7, w7, #4 + mov w8, #MAX_STRIDE + cmp w7, w8 + csel w7, w7, w8, lt + adds x12, x12, x7 + mov v0.16b, vctr.16b - add w8, w6, #2 mov v1.16b, vctr.16b - add w9, w6, #3 mov v2.16b, vctr.16b - add w9, w6, #3 - rev w7, w7 mov v3.16b, vctr.16b - rev w8, w8 ST5( mov v4.16b, vctr.16b ) - mov v1.s[3], w7 - rev w9, w9 -ST5( add w10, w6, #4 ) - mov v2.s[3], w8 -ST5( rev w10, w10 ) - mov v3.s[3], w9 -ST5( mov v4.s[3], w10 ) - ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */ + bcs 0f + + .subsection 1 + /* apply carry to outgoing counter */ +0: umov x8, vctr.d[0] + rev x8, x8 + add x8, x8, #1 + rev x8, x8 + ins vctr.d[0], x8 + + /* apply carry to N counter blocks for N := x12 */ + adr x16, 1f + sub x16, x16, x12, lsl #3 + br x16 + hint 34 // bti c + mov v0.d[0], vctr.d[0] + hint 34 // bti c + mov v1.d[0], vctr.d[0] + hint 34 // bti c + mov v2.d[0], vctr.d[0] + hint 34 // bti c + mov v3.d[0], vctr.d[0] +ST5( hint 34 ) +ST5( mov v4.d[0], vctr.d[0] ) +1: b 2f + .previous + +2: rev x7, x12 + ins vctr.d[1], x7 + sub x7, x12, #MAX_STRIDE - 1 + sub x8, x12, #MAX_STRIDE - 2 + sub x9, x12, #MAX_STRIDE - 3 + rev x7, x7 + rev x8, x8 + mov v1.d[1], x7 + rev x9, x9 +ST5( sub x10, x12, #MAX_STRIDE - 4 ) + mov v2.d[1], x8 +ST5( rev x10, x10 ) + mov v3.d[1], x9 +ST5( mov v4.d[1], x10 ) + tbnz w4, #31, .Lctrtail + ld1 {v5.16b-v7.16b}, [x1], #48 ST4( bl aes_encrypt_block4x ) ST5( bl aes_encrypt_block5x ) eor v0.16b, v5.16b, v0.16b @@ -368,47 +402,72 @@ ST5( ld1 {v5.16b-v6.16b}, [x1], #32 ) ST5( eor v4.16b, v6.16b, v4.16b ) st1 {v0.16b-v3.16b}, [x0], #64 ST5( st1 {v4.16b}, [x0], #16 ) - add x6, x6, #MAX_STRIDE - rev x7, x6 - ins vctr.d[1], x7 cbz w4, .Lctrout b .LctrloopNx -.Lctr1x: - adds w4, w4, #MAX_STRIDE - beq .Lctrout -.Lctrloop: - mov v0.16b, vctr.16b - encrypt_block v0, w3, x2, x8, w7 - - adds x6, x6, #1 /* increment BE ctr */ - rev x7, x6 - ins vctr.d[1], x7 - bcs .Lctrcarry /* overflow? */ - -.Lctrcarrydone: - subs w4, w4, #1 - bmi .Lctrtailblock /* blocks <0 means tail block */ - ld1 {v3.16b}, [x1], #16 - eor v3.16b, v0.16b, v3.16b - st1 {v3.16b}, [x0], #16 - bne .Lctrloop .Lctrout: st1 {vctr.16b}, [x5] /* return next CTR value */ ldp x29, x30, [sp], #16 ret -.Lctrtailblock: - st1 {v0.16b}, [x0] +.Lctrtail: + /* XOR up to MAX_STRIDE * 16 - 1 bytes of in/output with v0 ... v3/v4 */ + mov x16, #16 + ands x13, x4, #0xf + csel x13, x13, x16, ne + +ST5( cmp w4, #64 - (MAX_STRIDE << 4) ) +ST5( csel x14, x16, xzr, gt ) + cmp w4, #48 - (MAX_STRIDE << 4) + csel x15, x16, xzr, gt + cmp w4, #32 - (MAX_STRIDE << 4) + csel x16, x16, xzr, gt + cmp w4, #16 - (MAX_STRIDE << 4) + ble .Lctrtail1x + + adr_l x12, .Lcts_permute_table + add x12, x12, x13 + +ST5( ld1 {v5.16b}, [x1], x14 ) + ld1 {v6.16b}, [x1], x15 + ld1 {v7.16b}, [x1], x16 + +ST4( bl aes_encrypt_block4x ) +ST5( bl aes_encrypt_block5x ) + + ld1 {v8.16b}, [x1], x13 + ld1 {v9.16b}, [x1] + ld1 {v10.16b}, [x12] + +ST4( eor v6.16b, v6.16b, v0.16b ) +ST4( eor v7.16b, v7.16b, v1.16b ) +ST4( tbl v3.16b, {v3.16b}, v10.16b ) +ST4( eor v8.16b, v8.16b, v2.16b ) +ST4( eor v9.16b, v9.16b, v3.16b ) + +ST5( eor v5.16b, v5.16b, v0.16b ) +ST5( eor v6.16b, v6.16b, v1.16b ) +ST5( tbl v4.16b, {v4.16b}, v10.16b ) +ST5( eor v7.16b, v7.16b, v2.16b ) +ST5( eor v8.16b, v8.16b, v3.16b ) +ST5( eor v9.16b, v9.16b, v4.16b ) + +ST5( st1 {v5.16b}, [x0], x14 ) + st1 {v6.16b}, [x0], x15 + st1 {v7.16b}, [x0], x16 + add x13, x13, x0 + st1 {v9.16b}, [x13] // overlapping stores + st1 {v8.16b}, [x0] b .Lctrout -.Lctrcarry: - umov x7, vctr.d[0] /* load upper word of ctr */ - rev x7, x7 /* ... to handle the carry */ - add x7, x7, #1 - rev x7, x7 - ins vctr.d[0], x7 - b .Lctrcarrydone +.Lctrtail1x: + csel x0, x0, x6, eq // use finalbuf if less than a full block + ld1 {v5.16b}, [x1] +ST5( mov v3.16b, v4.16b ) + encrypt_block v3, w3, x2, x8, w7 + eor v5.16b, v5.16b, v3.16b + st1 {v5.16b}, [x0] + b .Lctrout AES_FUNC_END(aes_ctr_encrypt) @@ -619,61 +678,47 @@ AES_FUNC_END(aes_xts_decrypt) * int blocks, u8 dg[], int enc_before, int enc_after) */ AES_FUNC_START(aes_mac_update) - frame_push 6 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - mov x22, x3 - mov x23, x4 - mov x24, x6 - - ld1 {v0.16b}, [x23] /* get dg */ + ld1 {v0.16b}, [x4] /* get dg */ enc_prepare w2, x1, x7 cbz w5, .Lmacloop4x encrypt_block v0, w2, x1, x7, w8 .Lmacloop4x: - subs w22, w22, #4 + subs w3, w3, #4 bmi .Lmac1x - ld1 {v1.16b-v4.16b}, [x19], #64 /* get next pt block */ + ld1 {v1.16b-v4.16b}, [x0], #64 /* get next pt block */ eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ - encrypt_block v0, w21, x20, x7, w8 + encrypt_block v0, w2, x1, x7, w8 eor v0.16b, v0.16b, v2.16b - encrypt_block v0, w21, x20, x7, w8 + encrypt_block v0, w2, x1, x7, w8 eor v0.16b, v0.16b, v3.16b - encrypt_block v0, w21, x20, x7, w8 + encrypt_block v0, w2, x1, x7, w8 eor v0.16b, v0.16b, v4.16b - cmp w22, wzr - csinv x5, x24, xzr, eq + cmp w3, wzr + csinv x5, x6, xzr, eq cbz w5, .Lmacout - encrypt_block v0, w21, x20, x7, w8 - st1 {v0.16b}, [x23] /* return dg */ - cond_yield_neon .Lmacrestart + encrypt_block v0, w2, x1, x7, w8 + st1 {v0.16b}, [x4] /* return dg */ + cond_yield .Lmacout, x7 b .Lmacloop4x .Lmac1x: - add w22, w22, #4 + add w3, w3, #4 .Lmacloop: - cbz w22, .Lmacout - ld1 {v1.16b}, [x19], #16 /* get next pt block */ + cbz w3, .Lmacout + ld1 {v1.16b}, [x0], #16 /* get next pt block */ eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ - subs w22, w22, #1 - csinv x5, x24, xzr, eq + subs w3, w3, #1 + csinv x5, x6, xzr, eq cbz w5, .Lmacout .Lmacenc: - encrypt_block v0, w21, x20, x7, w8 + encrypt_block v0, w2, x1, x7, w8 b .Lmacloop .Lmacout: - st1 {v0.16b}, [x23] /* return dg */ - frame_pop + st1 {v0.16b}, [x4] /* return dg */ + mov w0, w3 ret - -.Lmacrestart: - ld1 {v0.16b}, [x23] /* get dg */ - enc_prepare w21, x20, x0 - b .Lmacloop4x AES_FUNC_END(aes_mac_update) diff --git a/arch/arm64/crypto/aes-neonbs-core.S b/arch/arm64/crypto/aes-neonbs-core.S index 63a52ad9a75c..a3405b8c344b 100644 --- a/arch/arm64/crypto/aes-neonbs-core.S +++ b/arch/arm64/crypto/aes-neonbs-core.S @@ -613,7 +613,6 @@ SYM_FUNC_END(aesbs_decrypt8) st1 {\o7\().16b}, [x19], #16 cbz x23, 1f - cond_yield_neon b 99b 1: frame_pop @@ -715,7 +714,6 @@ SYM_FUNC_START(aesbs_cbc_decrypt) 1: st1 {v24.16b}, [x24] // store IV cbz x23, 2f - cond_yield_neon b 99b 2: frame_pop @@ -801,7 +799,7 @@ SYM_FUNC_END(__xts_crypt8) mov x23, x4 mov x24, x5 -0: movi v30.2s, #0x1 + movi v30.2s, #0x1 movi v25.2s, #0x87 uzp1 v30.4s, v30.4s, v25.4s ld1 {v25.16b}, [x24] @@ -846,7 +844,6 @@ SYM_FUNC_END(__xts_crypt8) cbz x23, 1f st1 {v25.16b}, [x24] - cond_yield_neon 0b b 99b 1: st1 {v25.16b}, [x24] @@ -889,7 +886,7 @@ SYM_FUNC_START(aesbs_ctr_encrypt) cset x26, ne add x23, x23, x26 // do one extra block if final -98: ldp x7, x8, [x24] + ldp x7, x8, [x24] ld1 {v0.16b}, [x24] CPU_LE( rev x7, x7 ) CPU_LE( rev x8, x8 ) @@ -967,7 +964,6 @@ CPU_LE( rev x8, x8 ) st1 {v0.16b}, [x24] cbz x23, .Lctr_done - cond_yield_neon 98b b 99b .Lctr_done: diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S index 111d9c9abddd..dce6dcebfca1 100644 --- a/arch/arm64/crypto/crct10dif-ce-core.S +++ b/arch/arm64/crypto/crct10dif-ce-core.S @@ -68,10 +68,10 @@ .text .arch armv8-a+crypto - init_crc .req w19 - buf .req x20 - len .req x21 - fold_consts_ptr .req x22 + init_crc .req w0 + buf .req x1 + len .req x2 + fold_consts_ptr .req x3 fold_consts .req v10 @@ -257,12 +257,6 @@ CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 ) .endm .macro crc_t10dif_pmull, p - frame_push 4, 128 - - mov init_crc, w0 - mov buf, x1 - mov len, x2 - __pmull_init_\p // For sizes less than 256 bytes, we can't fold 128 bytes at a time. @@ -317,26 +311,7 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) fold_32_bytes \p, v6, v7 subs len, len, #128 - b.lt .Lfold_128_bytes_loop_done_\@ - - if_will_cond_yield_neon - stp q0, q1, [sp, #.Lframe_local_offset] - stp q2, q3, [sp, #.Lframe_local_offset + 32] - stp q4, q5, [sp, #.Lframe_local_offset + 64] - stp q6, q7, [sp, #.Lframe_local_offset + 96] - do_cond_yield_neon - ldp q0, q1, [sp, #.Lframe_local_offset] - ldp q2, q3, [sp, #.Lframe_local_offset + 32] - ldp q4, q5, [sp, #.Lframe_local_offset + 64] - ldp q6, q7, [sp, #.Lframe_local_offset + 96] - ld1 {fold_consts.2d}, [fold_consts_ptr] - __pmull_init_\p - __pmull_pre_\p fold_consts - endif_yield_neon - - b .Lfold_128_bytes_loop_\@ - -.Lfold_128_bytes_loop_done_\@: + b.ge .Lfold_128_bytes_loop_\@ // Now fold the 112 bytes in v0-v6 into the 16 bytes in v7. @@ -453,7 +428,9 @@ CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) // Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of v0. umov w0, v0.h[0] - frame_pop + .ifc \p, p8 + ldp x29, x30, [sp], #16 + .endif ret .Lless_than_256_bytes_\@: @@ -489,7 +466,9 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) // Assumes len >= 16. // SYM_FUNC_START(crc_t10dif_pmull_p8) - crc_t10dif_pmull p8 + stp x29, x30, [sp, #-16]! + mov x29, sp + crc_t10dif_pmull p8 SYM_FUNC_END(crc_t10dif_pmull_p8) .align 5 diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c index ccc3f6067742..09eb1456aed4 100644 --- a/arch/arm64/crypto/crct10dif-ce-glue.c +++ b/arch/arm64/crypto/crct10dif-ce-glue.c @@ -37,9 +37,18 @@ static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, u16 *crc = shash_desc_ctx(desc); if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { - kernel_neon_begin(); - *crc = crc_t10dif_pmull_p8(*crc, data, length); - kernel_neon_end(); + do { + unsigned int chunk = length; + + if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE) + chunk = SZ_4K; + + kernel_neon_begin(); + *crc = crc_t10dif_pmull_p8(*crc, data, chunk); + kernel_neon_end(); + data += chunk; + length -= chunk; + } while (length); } else { *crc = crc_t10dif_generic(*crc, data, length); } @@ -53,9 +62,18 @@ static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, u16 *crc = shash_desc_ctx(desc); if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { - kernel_neon_begin(); - *crc = crc_t10dif_pmull_p64(*crc, data, length); - kernel_neon_end(); + do { + unsigned int chunk = length; + + if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE) + chunk = SZ_4K; + + kernel_neon_begin(); + *crc = crc_t10dif_pmull_p64(*crc, data, chunk); + kernel_neon_end(); + data += chunk; + length -= chunk; + } while (length); } else { *crc = crc_t10dif_generic(*crc, data, length); } diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S index 92d0d2753e81..8c02bbc2684e 100644 --- a/arch/arm64/crypto/sha1-ce-core.S +++ b/arch/arm64/crypto/sha1-ce-core.S @@ -62,40 +62,34 @@ .endm /* - * void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, - * int blocks) + * int sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, + * int blocks) */ SYM_FUNC_START(sha1_ce_transform) - frame_push 3 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - /* load round constants */ -0: loadrc k0.4s, 0x5a827999, w6 + loadrc k0.4s, 0x5a827999, w6 loadrc k1.4s, 0x6ed9eba1, w6 loadrc k2.4s, 0x8f1bbcdc, w6 loadrc k3.4s, 0xca62c1d6, w6 /* load state */ - ld1 {dgav.4s}, [x19] - ldr dgb, [x19, #16] + ld1 {dgav.4s}, [x0] + ldr dgb, [x0, #16] /* load sha1_ce_state::finalize */ ldr_l w4, sha1_ce_offsetof_finalize, x4 - ldr w4, [x19, x4] + ldr w4, [x0, x4] /* load input */ -1: ld1 {v8.4s-v11.4s}, [x20], #64 - sub w21, w21, #1 +0: ld1 {v8.4s-v11.4s}, [x1], #64 + sub w2, w2, #1 CPU_LE( rev32 v8.16b, v8.16b ) CPU_LE( rev32 v9.16b, v9.16b ) CPU_LE( rev32 v10.16b, v10.16b ) CPU_LE( rev32 v11.16b, v11.16b ) -2: add t0.4s, v8.4s, k0.4s +1: add t0.4s, v8.4s, k0.4s mov dg0v.16b, dgav.16b add_update c, ev, k0, 8, 9, 10, 11, dgb @@ -126,25 +120,18 @@ CPU_LE( rev32 v11.16b, v11.16b ) add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s - cbz w21, 3f - - if_will_cond_yield_neon - st1 {dgav.4s}, [x19] - str dgb, [x19, #16] - do_cond_yield_neon + cbz w2, 2f + cond_yield 3f, x5 b 0b - endif_yield_neon - - b 1b /* * Final block: add padding and total bit count. * Skip if the input size was not a round multiple of the block size, * the padding is handled by the C code in that case. */ -3: cbz x4, 4f +2: cbz x4, 3f ldr_l w4, sha1_ce_offsetof_count, x4 - ldr x4, [x19, x4] + ldr x4, [x0, x4] movi v9.2d, #0 mov x8, #0x80000000 movi v10.2d, #0 @@ -153,11 +140,11 @@ CPU_LE( rev32 v11.16b, v11.16b ) mov x4, #0 mov v11.d[0], xzr mov v11.d[1], x7 - b 2b + b 1b /* store new state */ -4: st1 {dgav.4s}, [x19] - str dgb, [x19, #16] - frame_pop +3: st1 {dgav.4s}, [x0] + str dgb, [x0, #16] + mov w0, w2 ret SYM_FUNC_END(sha1_ce_transform) diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index c93121bcfdeb..71fa4f1122d7 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -19,6 +19,7 @@ MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_CRYPTO("sha1"); struct sha1_ce_state { struct sha1_state sst; @@ -28,14 +29,22 @@ struct sha1_ce_state { extern const u32 sha1_ce_offsetof_count; extern const u32 sha1_ce_offsetof_finalize; -asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, - int blocks); +asmlinkage int sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, + int blocks); static void __sha1_ce_transform(struct sha1_state *sst, u8 const *src, int blocks) { - sha1_ce_transform(container_of(sst, struct sha1_ce_state, sst), src, - blocks); + while (blocks) { + int rem; + + kernel_neon_begin(); + rem = sha1_ce_transform(container_of(sst, struct sha1_ce_state, + sst), src, blocks); + kernel_neon_end(); + src += (blocks - rem) * SHA1_BLOCK_SIZE; + blocks = rem; + } } const u32 sha1_ce_offsetof_count = offsetof(struct sha1_ce_state, sst.count); @@ -50,9 +59,7 @@ static int sha1_ce_update(struct shash_desc *desc, const u8 *data, return crypto_sha1_update(desc, data, len); sctx->finalize = 0; - kernel_neon_begin(); sha1_base_do_update(desc, data, len, __sha1_ce_transform); - kernel_neon_end(); return 0; } @@ -72,11 +79,9 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, */ sctx->finalize = finalize; - kernel_neon_begin(); sha1_base_do_update(desc, data, len, __sha1_ce_transform); if (!finalize) sha1_base_do_finalize(desc, __sha1_ce_transform); - kernel_neon_end(); return sha1_base_finish(desc, out); } @@ -88,9 +93,7 @@ static int sha1_ce_final(struct shash_desc *desc, u8 *out) return crypto_sha1_finup(desc, NULL, 0, out); sctx->finalize = 0; - kernel_neon_begin(); sha1_base_do_finalize(desc, __sha1_ce_transform); - kernel_neon_end(); return sha1_base_finish(desc, out); } diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S index 3f9d0f326987..6cdea7d56059 100644 --- a/arch/arm64/crypto/sha2-ce-core.S +++ b/arch/arm64/crypto/sha2-ce-core.S @@ -76,36 +76,30 @@ */ .text SYM_FUNC_START(sha2_ce_transform) - frame_push 3 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - /* load round constants */ -0: adr_l x8, .Lsha2_rcon + adr_l x8, .Lsha2_rcon ld1 { v0.4s- v3.4s}, [x8], #64 ld1 { v4.4s- v7.4s}, [x8], #64 ld1 { v8.4s-v11.4s}, [x8], #64 ld1 {v12.4s-v15.4s}, [x8] /* load state */ - ld1 {dgav.4s, dgbv.4s}, [x19] + ld1 {dgav.4s, dgbv.4s}, [x0] /* load sha256_ce_state::finalize */ ldr_l w4, sha256_ce_offsetof_finalize, x4 - ldr w4, [x19, x4] + ldr w4, [x0, x4] /* load input */ -1: ld1 {v16.4s-v19.4s}, [x20], #64 - sub w21, w21, #1 +0: ld1 {v16.4s-v19.4s}, [x1], #64 + sub w2, w2, #1 CPU_LE( rev32 v16.16b, v16.16b ) CPU_LE( rev32 v17.16b, v17.16b ) CPU_LE( rev32 v18.16b, v18.16b ) CPU_LE( rev32 v19.16b, v19.16b ) -2: add t0.4s, v16.4s, v0.4s +1: add t0.4s, v16.4s, v0.4s mov dg0v.16b, dgav.16b mov dg1v.16b, dgbv.16b @@ -134,24 +128,18 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgbv.4s, dgbv.4s, dg1v.4s /* handled all input blocks? */ - cbz w21, 3f - - if_will_cond_yield_neon - st1 {dgav.4s, dgbv.4s}, [x19] - do_cond_yield_neon + cbz w2, 2f + cond_yield 3f, x5 b 0b - endif_yield_neon - - b 1b /* * Final block: add padding and total bit count. * Skip if the input size was not a round multiple of the block size, * the padding is handled by the C code in that case. */ -3: cbz x4, 4f +2: cbz x4, 3f ldr_l w4, sha256_ce_offsetof_count, x4 - ldr x4, [x19, x4] + ldr x4, [x0, x4] movi v17.2d, #0 mov x8, #0x80000000 movi v18.2d, #0 @@ -160,10 +148,10 @@ CPU_LE( rev32 v19.16b, v19.16b ) mov x4, #0 mov v19.d[0], xzr mov v19.d[1], x7 - b 2b + b 1b /* store new state */ -4: st1 {dgav.4s, dgbv.4s}, [x19] - frame_pop +3: st1 {dgav.4s, dgbv.4s}, [x0] + mov w0, w2 ret SYM_FUNC_END(sha2_ce_transform) diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 31ba3da5e61b..c57a6119fefc 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -19,6 +19,8 @@ MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_CRYPTO("sha224"); +MODULE_ALIAS_CRYPTO("sha256"); struct sha256_ce_state { struct sha256_state sst; @@ -28,14 +30,22 @@ struct sha256_ce_state { extern const u32 sha256_ce_offsetof_count; extern const u32 sha256_ce_offsetof_finalize; -asmlinkage void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, - int blocks); +asmlinkage int sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, + int blocks); static void __sha2_ce_transform(struct sha256_state *sst, u8 const *src, int blocks) { - sha2_ce_transform(container_of(sst, struct sha256_ce_state, sst), src, - blocks); + while (blocks) { + int rem; + + kernel_neon_begin(); + rem = sha2_ce_transform(container_of(sst, struct sha256_ce_state, + sst), src, blocks); + kernel_neon_end(); + src += (blocks - rem) * SHA256_BLOCK_SIZE; + blocks = rem; + } } const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state, @@ -61,9 +71,7 @@ static int sha256_ce_update(struct shash_desc *desc, const u8 *data, __sha256_block_data_order); sctx->finalize = 0; - kernel_neon_begin(); sha256_base_do_update(desc, data, len, __sha2_ce_transform); - kernel_neon_end(); return 0; } @@ -88,11 +96,9 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, */ sctx->finalize = finalize; - kernel_neon_begin(); sha256_base_do_update(desc, data, len, __sha2_ce_transform); if (!finalize) sha256_base_do_finalize(desc, __sha2_ce_transform); - kernel_neon_end(); return sha256_base_finish(desc, out); } @@ -106,9 +112,7 @@ static int sha256_ce_final(struct shash_desc *desc, u8 *out) } sctx->finalize = 0; - kernel_neon_begin(); sha256_base_do_finalize(desc, __sha2_ce_transform); - kernel_neon_end(); return sha256_base_finish(desc, out); } diff --git a/arch/arm64/crypto/sha3-ce-core.S b/arch/arm64/crypto/sha3-ce-core.S index 1cfb768df350..6f5208414fe3 100644 --- a/arch/arm64/crypto/sha3-ce-core.S +++ b/arch/arm64/crypto/sha3-ce-core.S @@ -37,20 +37,13 @@ .endm /* - * sha3_ce_transform(u64 *st, const u8 *data, int blocks, int dg_size) + * int sha3_ce_transform(u64 *st, const u8 *data, int blocks, int dg_size) */ .text SYM_FUNC_START(sha3_ce_transform) - frame_push 4 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - mov x22, x3 - -0: /* load state */ - add x8, x19, #32 - ld1 { v0.1d- v3.1d}, [x19] + /* load state */ + add x8, x0, #32 + ld1 { v0.1d- v3.1d}, [x0] ld1 { v4.1d- v7.1d}, [x8], #32 ld1 { v8.1d-v11.1d}, [x8], #32 ld1 {v12.1d-v15.1d}, [x8], #32 @@ -58,13 +51,13 @@ SYM_FUNC_START(sha3_ce_transform) ld1 {v20.1d-v23.1d}, [x8], #32 ld1 {v24.1d}, [x8] -1: sub w21, w21, #1 +0: sub w2, w2, #1 mov w8, #24 adr_l x9, .Lsha3_rcon /* load input */ - ld1 {v25.8b-v28.8b}, [x20], #32 - ld1 {v29.8b-v31.8b}, [x20], #24 + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b-v31.8b}, [x1], #24 eor v0.8b, v0.8b, v25.8b eor v1.8b, v1.8b, v26.8b eor v2.8b, v2.8b, v27.8b @@ -73,10 +66,10 @@ SYM_FUNC_START(sha3_ce_transform) eor v5.8b, v5.8b, v30.8b eor v6.8b, v6.8b, v31.8b - tbnz x22, #6, 3f // SHA3-512 + tbnz x3, #6, 2f // SHA3-512 - ld1 {v25.8b-v28.8b}, [x20], #32 - ld1 {v29.8b-v30.8b}, [x20], #16 + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b-v30.8b}, [x1], #16 eor v7.8b, v7.8b, v25.8b eor v8.8b, v8.8b, v26.8b eor v9.8b, v9.8b, v27.8b @@ -84,34 +77,34 @@ SYM_FUNC_START(sha3_ce_transform) eor v11.8b, v11.8b, v29.8b eor v12.8b, v12.8b, v30.8b - tbnz x22, #4, 2f // SHA3-384 or SHA3-224 + tbnz x3, #4, 1f // SHA3-384 or SHA3-224 // SHA3-256 - ld1 {v25.8b-v28.8b}, [x20], #32 + ld1 {v25.8b-v28.8b}, [x1], #32 eor v13.8b, v13.8b, v25.8b eor v14.8b, v14.8b, v26.8b eor v15.8b, v15.8b, v27.8b eor v16.8b, v16.8b, v28.8b - b 4f + b 3f -2: tbz x22, #2, 4f // bit 2 cleared? SHA-384 +1: tbz x3, #2, 3f // bit 2 cleared? SHA-384 // SHA3-224 - ld1 {v25.8b-v28.8b}, [x20], #32 - ld1 {v29.8b}, [x20], #8 + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b}, [x1], #8 eor v13.8b, v13.8b, v25.8b eor v14.8b, v14.8b, v26.8b eor v15.8b, v15.8b, v27.8b eor v16.8b, v16.8b, v28.8b eor v17.8b, v17.8b, v29.8b - b 4f + b 3f // SHA3-512 -3: ld1 {v25.8b-v26.8b}, [x20], #16 +2: ld1 {v25.8b-v26.8b}, [x1], #16 eor v7.8b, v7.8b, v25.8b eor v8.8b, v8.8b, v26.8b -4: sub w8, w8, #1 +3: sub w8, w8, #1 eor3 v29.16b, v4.16b, v9.16b, v14.16b eor3 v26.16b, v1.16b, v6.16b, v11.16b @@ -190,33 +183,19 @@ SYM_FUNC_START(sha3_ce_transform) eor v0.16b, v0.16b, v31.16b - cbnz w8, 4b - cbz w21, 5f - - if_will_cond_yield_neon - add x8, x19, #32 - st1 { v0.1d- v3.1d}, [x19] - st1 { v4.1d- v7.1d}, [x8], #32 - st1 { v8.1d-v11.1d}, [x8], #32 - st1 {v12.1d-v15.1d}, [x8], #32 - st1 {v16.1d-v19.1d}, [x8], #32 - st1 {v20.1d-v23.1d}, [x8], #32 - st1 {v24.1d}, [x8] - do_cond_yield_neon - b 0b - endif_yield_neon - - b 1b + cbnz w8, 3b + cond_yield 3f, x8 + cbnz w2, 0b /* save state */ -5: st1 { v0.1d- v3.1d}, [x19], #32 - st1 { v4.1d- v7.1d}, [x19], #32 - st1 { v8.1d-v11.1d}, [x19], #32 - st1 {v12.1d-v15.1d}, [x19], #32 - st1 {v16.1d-v19.1d}, [x19], #32 - st1 {v20.1d-v23.1d}, [x19], #32 - st1 {v24.1d}, [x19] - frame_pop +3: st1 { v0.1d- v3.1d}, [x0], #32 + st1 { v4.1d- v7.1d}, [x0], #32 + st1 { v8.1d-v11.1d}, [x0], #32 + st1 {v12.1d-v15.1d}, [x0], #32 + st1 {v16.1d-v19.1d}, [x0], #32 + st1 {v20.1d-v23.1d}, [x0], #32 + st1 {v24.1d}, [x0] + mov w0, w2 ret SYM_FUNC_END(sha3_ce_transform) diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c index e5a2936f0886..8c65cecf560a 100644 --- a/arch/arm64/crypto/sha3-ce-glue.c +++ b/arch/arm64/crypto/sha3-ce-glue.c @@ -23,9 +23,13 @@ MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_CRYPTO("sha3-224"); +MODULE_ALIAS_CRYPTO("sha3-256"); +MODULE_ALIAS_CRYPTO("sha3-384"); +MODULE_ALIAS_CRYPTO("sha3-512"); -asmlinkage void sha3_ce_transform(u64 *st, const u8 *data, int blocks, - int md_len); +asmlinkage int sha3_ce_transform(u64 *st, const u8 *data, int blocks, + int md_len); static int sha3_update(struct shash_desc *desc, const u8 *data, unsigned int len) @@ -55,11 +59,15 @@ static int sha3_update(struct shash_desc *desc, const u8 *data, blocks = len / sctx->rsiz; len %= sctx->rsiz; - if (blocks) { + while (blocks) { + int rem; + kernel_neon_begin(); - sha3_ce_transform(sctx->st, data, blocks, digest_size); + rem = sha3_ce_transform(sctx->st, data, blocks, + digest_size); kernel_neon_end(); - data += blocks * sctx->rsiz; + data += (blocks - rem) * sctx->rsiz; + blocks = rem; } } diff --git a/arch/arm64/crypto/sha512-ce-core.S b/arch/arm64/crypto/sha512-ce-core.S index cde606c0323e..d6e7f6c95fa6 100644 --- a/arch/arm64/crypto/sha512-ce-core.S +++ b/arch/arm64/crypto/sha512-ce-core.S @@ -107,23 +107,17 @@ */ .text SYM_FUNC_START(sha512_ce_transform) - frame_push 3 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - /* load state */ -0: ld1 {v8.2d-v11.2d}, [x19] + ld1 {v8.2d-v11.2d}, [x0] /* load first 4 round constants */ adr_l x3, .Lsha512_rcon ld1 {v20.2d-v23.2d}, [x3], #64 /* load input */ -1: ld1 {v12.2d-v15.2d}, [x20], #64 - ld1 {v16.2d-v19.2d}, [x20], #64 - sub w21, w21, #1 +0: ld1 {v12.2d-v15.2d}, [x1], #64 + ld1 {v16.2d-v19.2d}, [x1], #64 + sub w2, w2, #1 CPU_LE( rev64 v12.16b, v12.16b ) CPU_LE( rev64 v13.16b, v13.16b ) @@ -201,19 +195,12 @@ CPU_LE( rev64 v19.16b, v19.16b ) add v10.2d, v10.2d, v2.2d add v11.2d, v11.2d, v3.2d + cond_yield 3f, x4 /* handled all input blocks? */ - cbz w21, 3f - - if_will_cond_yield_neon - st1 {v8.2d-v11.2d}, [x19] - do_cond_yield_neon - b 0b - endif_yield_neon - - b 1b + cbnz w2, 0b /* store new state */ -3: st1 {v8.2d-v11.2d}, [x19] - frame_pop +3: st1 {v8.2d-v11.2d}, [x0] + mov w0, w2 ret SYM_FUNC_END(sha512_ce_transform) diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c index faa83f6cf376..e62a094a9d52 100644 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -23,12 +23,28 @@ MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_CRYPTO("sha384"); +MODULE_ALIAS_CRYPTO("sha512"); -asmlinkage void sha512_ce_transform(struct sha512_state *sst, u8 const *src, - int blocks); +asmlinkage int sha512_ce_transform(struct sha512_state *sst, u8 const *src, + int blocks); asmlinkage void sha512_block_data_order(u64 *digest, u8 const *src, int blocks); +static void __sha512_ce_transform(struct sha512_state *sst, u8 const *src, + int blocks) +{ + while (blocks) { + int rem; + + kernel_neon_begin(); + rem = sha512_ce_transform(sst, src, blocks); + kernel_neon_end(); + src += (blocks - rem) * SHA512_BLOCK_SIZE; + blocks = rem; + } +} + static void __sha512_block_data_order(struct sha512_state *sst, u8 const *src, int blocks) { @@ -38,45 +54,30 @@ static void __sha512_block_data_order(struct sha512_state *sst, u8 const *src, static int sha512_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - if (!crypto_simd_usable()) - return sha512_base_do_update(desc, data, len, - __sha512_block_data_order); - - kernel_neon_begin(); - sha512_base_do_update(desc, data, len, sha512_ce_transform); - kernel_neon_end(); + sha512_block_fn *fn = crypto_simd_usable() ? __sha512_ce_transform + : __sha512_block_data_order; + sha512_base_do_update(desc, data, len, fn); return 0; } static int sha512_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) { - if (len) - sha512_base_do_update(desc, data, len, - __sha512_block_data_order); - sha512_base_do_finalize(desc, __sha512_block_data_order); - return sha512_base_finish(desc, out); - } + sha512_block_fn *fn = crypto_simd_usable() ? __sha512_ce_transform + : __sha512_block_data_order; - kernel_neon_begin(); - sha512_base_do_update(desc, data, len, sha512_ce_transform); - sha512_base_do_finalize(desc, sha512_ce_transform); - kernel_neon_end(); + sha512_base_do_update(desc, data, len, fn); + sha512_base_do_finalize(desc, fn); return sha512_base_finish(desc, out); } static int sha512_ce_final(struct shash_desc *desc, u8 *out) { - if (!crypto_simd_usable()) { - sha512_base_do_finalize(desc, __sha512_block_data_order); - return sha512_base_finish(desc, out); - } + sha512_block_fn *fn = crypto_simd_usable() ? __sha512_ce_transform + : __sha512_block_data_order; - kernel_neon_begin(); - sha512_base_do_finalize(desc, sha512_ce_transform); - kernel_neon_end(); + sha512_base_do_finalize(desc, fn); return sha512_base_finish(desc, out); } diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h index 77cbbe3625f2..a074459f8f2f 100644 --- a/arch/arm64/include/asm/cache.h +++ b/arch/arm64/include/asm/cache.h @@ -6,7 +6,6 @@ #define __ASM_CACHE_H #include -#include #define CTR_L1IP_SHIFT 14 #define CTR_L1IP_MASK 3 diff --git a/arch/arm64/include/asm/kasan.h b/arch/arm64/include/asm/kasan.h index 0aaf9044cd6a..12d5f47f7dbe 100644 --- a/arch/arm64/include/asm/kasan.h +++ b/arch/arm64/include/asm/kasan.h @@ -6,6 +6,7 @@ #include #include +#include #include #define arch_kasan_set_tag(addr, tag) __tag_set(addr, tag) diff --git a/arch/arm64/include/asm/kfence.h b/arch/arm64/include/asm/kfence.h new file mode 100644 index 000000000000..d061176d57ea --- /dev/null +++ b/arch/arm64/include/asm/kfence.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arm64 KFENCE support. + * + * Copyright (C) 2020, Google LLC. + */ + +#ifndef __ASM_KFENCE_H +#define __ASM_KFENCE_H + +#include + +static inline bool arch_kfence_init_pool(void) { return true; } + +static inline bool kfence_protect_page(unsigned long addr, bool protect) +{ + set_memory_valid(addr, 1, !protect); + + return true; +} + +#endif /* __ASM_KFENCE_H */ diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index bc09af26c1b8..c759faf7a1ff 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -244,6 +244,7 @@ static inline const void *__tag_set(const void *addr, u8 tag) #ifdef CONFIG_KASAN_HW_TAGS #define arch_enable_tagging() mte_enable_kernel() +#define arch_set_tagging_report_once(state) mte_set_report_once(state) #define arch_init_tags(max_tag) mte_init_tags(max_tag) #define arch_get_random_tag() mte_get_random_tag() #define arch_get_mem_tag(addr) mte_get_mem_tag(addr) diff --git a/arch/arm64/include/asm/module.lds.h b/arch/arm64/include/asm/module.lds.h index 691f15af788e..810045628c66 100644 --- a/arch/arm64/include/asm/module.lds.h +++ b/arch/arm64/include/asm/module.lds.h @@ -1,7 +1,7 @@ #ifdef CONFIG_ARM64_MODULE_PLTS SECTIONS { - .plt (NOLOAD) : { BYTE(0) } - .init.plt (NOLOAD) : { BYTE(0) } - .text.ftrace_trampoline (NOLOAD) : { BYTE(0) } + .plt 0 (NOLOAD) : { BYTE(0) } + .init.plt 0 (NOLOAD) : { BYTE(0) } + .text.ftrace_trampoline 0 (NOLOAD) : { BYTE(0) } } #endif diff --git a/arch/arm64/include/asm/mte-def.h b/arch/arm64/include/asm/mte-def.h index 2d73a1612f09..cf241b0f0a42 100644 --- a/arch/arm64/include/asm/mte-def.h +++ b/arch/arm64/include/asm/mte-def.h @@ -11,4 +11,6 @@ #define MTE_TAG_SIZE 4 #define MTE_TAG_MASK GENMASK((MTE_TAG_SHIFT + (MTE_TAG_SIZE - 1)), MTE_TAG_SHIFT) +#define __MTE_PREAMBLE ARM64_ASM_PREAMBLE ".arch_extension memtag\n" + #endif /* __ASM_MTE_DEF_H */ diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h index 26349a4b5e2e..7ab500e2ad17 100644 --- a/arch/arm64/include/asm/mte-kasan.h +++ b/arch/arm64/include/asm/mte-kasan.h @@ -11,12 +11,15 @@ #include -/* - * The functions below are meant to be used only for the - * KASAN_HW_TAGS interface defined in asm/memory.h. - */ #ifdef CONFIG_ARM64_MTE +/* + * These functions are meant to be only used from KASAN runtime through + * the arch_*() interface defined in asm/memory.h. + * These functions don't include system_supports_mte() checks, + * as KASAN only calls them when MTE is supported and enabled. + */ + static inline u8 mte_get_ptr_tag(void *ptr) { /* Note: The format of KASAN tags is 0xF */ @@ -25,13 +28,61 @@ static inline u8 mte_get_ptr_tag(void *ptr) return tag; } -u8 mte_get_mem_tag(void *addr); -u8 mte_get_random_tag(void); -void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag); +/* Get allocation tag for the address. */ +static inline u8 mte_get_mem_tag(void *addr) +{ + asm(__MTE_PREAMBLE "ldg %0, [%0]" + : "+r" (addr)); + + return mte_get_ptr_tag(addr); +} + +/* Generate a random tag. */ +static inline u8 mte_get_random_tag(void) +{ + void *addr; + + asm(__MTE_PREAMBLE "irg %0, %0" + : "=r" (addr)); + + return mte_get_ptr_tag(addr); +} + +/* + * Assign allocation tags for a region of memory based on the pointer tag. + * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and + * size must be non-zero and MTE_GRANULE_SIZE aligned. + */ +static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag) +{ + u64 curr, end; + + if (!size) + return; + + curr = (u64)__tag_set(addr, tag); + end = curr + size; + + do { + /* + * 'asm volatile' is required to prevent the compiler to move + * the statement outside of the loop. + */ + asm volatile(__MTE_PREAMBLE "stg %0, [%0]" + : + : "r" (curr) + : "memory"); + + curr += MTE_GRANULE_SIZE; + } while (curr != end); +} void mte_enable_kernel(void); void mte_init_tags(u64 max_tag); +void mte_set_report_once(bool state); +bool mte_report_once(void); + #else /* CONFIG_ARM64_MTE */ static inline u8 mte_get_ptr_tag(void *ptr) @@ -43,13 +94,14 @@ static inline u8 mte_get_mem_tag(void *addr) { return 0xFF; } + static inline u8 mte_get_random_tag(void) { return 0xFF; } -static inline void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag) + +static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag) { - return addr; } static inline void mte_enable_kernel(void) @@ -60,6 +112,15 @@ static inline void mte_init_tags(u64 max_tag) { } +static inline void mte_set_report_once(bool state) +{ +} + +static inline bool mte_report_once(void) +{ + return false; +} + #endif /* CONFIG_ARM64_MTE */ #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index d02aff9f493d..9b557a457f24 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -8,8 +8,6 @@ #include #include -#define __MTE_PREAMBLE ARM64_ASM_PREAMBLE ".arch_extension memtag\n" - #ifndef __ASSEMBLY__ #include diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h index dd870390d639..8c8cf4297cc3 100644 --- a/arch/arm64/include/asm/numa.h +++ b/arch/arm64/include/asm/numa.h @@ -3,52 +3,6 @@ #define __ASM_NUMA_H #include - -#ifdef CONFIG_NUMA - -#define NR_NODE_MEMBLKS (MAX_NUMNODES * 2) - -int __node_distance(int from, int to); -#define node_distance(a, b) __node_distance(a, b) - -extern nodemask_t numa_nodes_parsed __initdata; - -extern bool numa_off; - -/* Mappings between node number and cpus on that node. */ -extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; -void numa_clear_node(unsigned int cpu); - -#ifdef CONFIG_DEBUG_PER_CPU_MAPS -const struct cpumask *cpumask_of_node(int node); -#else -/* Returns a pointer to the cpumask of CPUs on Node 'node'. */ -static inline const struct cpumask *cpumask_of_node(int node) -{ - if (node == NUMA_NO_NODE) - return cpu_all_mask; - - return node_to_cpumask_map[node]; -} -#endif - -void __init arm64_numa_init(void); -int __init numa_add_memblk(int nodeid, u64 start, u64 end); -void __init numa_set_distance(int from, int to, int distance); -void __init numa_free_distance(void); -void __init early_map_cpu_to_node(unsigned int cpu, int nid); -void numa_store_cpu_info(unsigned int cpu); -void numa_add_cpu(unsigned int cpu); -void numa_remove_cpu(unsigned int cpu); - -#else /* CONFIG_NUMA */ - -static inline void numa_store_cpu_info(unsigned int cpu) { } -static inline void numa_add_cpu(unsigned int cpu) { } -static inline void numa_remove_cpu(unsigned int cpu) { } -static inline void arm64_numa_init(void) { } -static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { } - -#endif /* CONFIG_NUMA */ +#include #endif /* __ASM_NUMA_H */ diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index f9fbbb4734e8..dfd4edbfe360 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -191,6 +191,7 @@ #define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6) #define SYS_ZCR_EL1 sys_reg(3, 0, 1, 2, 0) +#define SYS_TRFCR_EL1 sys_reg(3, 0, 1, 2, 1) #define SYS_TTBR0_EL1 sys_reg(3, 0, 2, 0, 0) #define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1) @@ -475,6 +476,7 @@ #define SYS_SCTLR_EL2 sys_reg(3, 4, 1, 0, 0) #define SYS_ZCR_EL2 sys_reg(3, 4, 1, 2, 0) +#define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1) #define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0) #define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1) @@ -833,6 +835,7 @@ #define ID_AA64MMFR2_CNP_SHIFT 0 /* id_aa64dfr0 */ +#define ID_AA64DFR0_TRACE_FILT_SHIFT 40 #define ID_AA64DFR0_DOUBLELOCK_SHIFT 36 #define ID_AA64DFR0_PMSVER_SHIFT 32 #define ID_AA64DFR0_CTX_CMPS_SHIFT 28 @@ -1013,6 +1016,14 @@ /* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */ #define SYS_MPIDR_SAFE_VAL (BIT(31)) +#define TRFCR_ELx_TS_SHIFT 5 +#define TRFCR_ELx_TS_VIRTUAL ((0x1UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_ELx_TS_GUEST_PHYSICAL ((0x2UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_ELx_TS_PHYSICAL ((0x3UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_EL2_CX BIT(3) +#define TRFCR_ELx_ExTRE BIT(1) +#define TRFCR_ELx_E0TRE BIT(0) + #ifdef __ASSEMBLY__ .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 86a9d7b3eabe..949788f5ba40 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -38,7 +38,7 @@ #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5) #define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800) -#define __NR_compat_syscalls 442 +#define __NR_compat_syscalls 443 #endif #define __ARCH_WANT_SYS_CLONE diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index cccfbbefbf95..3d874f624056 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -891,6 +891,8 @@ __SYSCALL(__NR_faccessat2, sys_faccessat2) __SYSCALL(__NR_process_madvise, sys_process_madvise) #define __NR_epoll_pwait2 441 __SYSCALL(__NR_epoll_pwait2, compat_sys_epoll_pwait2) +#define __NR_mount_setattr 442 +__SYSCALL(__NR_mount_setattr, sys_mount_setattr) /* * Please add new compat syscalls above this comment and update diff --git a/arch/arm64/kernel/acpi_numa.c b/arch/arm64/kernel/acpi_numa.c index 7ff800045434..fdfecf0991ce 100644 --- a/arch/arm64/kernel/acpi_numa.c +++ b/arch/arm64/kernel/acpi_numa.c @@ -118,15 +118,3 @@ void __init acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa) node_set(node, numa_nodes_parsed); } -int __init arm64_acpi_numa_init(void) -{ - int ret; - - ret = acpi_numa_init(); - if (ret) { - pr_info("Failed to initialise from firmware\n"); - return ret; - } - - return srat_disabled() ? -EINVAL : 0; -} diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 1e30b5550d2a..66b0e0b66e31 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -837,6 +837,7 @@ SYM_FUNC_START_LOCAL(__primary_switch) tlbi vmalle1 // Remove any stale TLB entries dsb nsh + isb set_sctlr_el1 x19 // re-enable the MMU diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index 678cd2c618ee..5eccbd62fec8 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -75,9 +75,6 @@ SYM_CODE_END(el1_sync) // nVHE? No way! Give me the real thing! SYM_CODE_START_LOCAL(mutate_to_vhe) - // Be prepared to fail - mov_q x0, HVC_STUB_ERR - // Sanity check: MMU *must* be off mrs x1, sctlr_el2 tbnz x1, #0, 1f @@ -96,8 +93,11 @@ SYM_CODE_START_LOCAL(mutate_to_vhe) cmp x1, xzr and x2, x2, x1 csinv x2, x2, xzr, ne - cbz x2, 1f + cbnz x2, 2f +1: mov_q x0, HVC_STUB_ERR + eret +2: // Engage the VHE magic! mov_q x0, HCR_HOST_VHE_FLAGS msr hcr_el2, x0 @@ -131,9 +131,28 @@ SYM_CODE_START_LOCAL(mutate_to_vhe) msr mair_el1, x0 isb + // Hack the exception return to stay at EL2 + mrs x0, spsr_el1 + and x0, x0, #~PSR_MODE_MASK + mov x1, #PSR_MODE_EL2h + orr x0, x0, x1 + msr spsr_el1, x0 + + b enter_vhe +SYM_CODE_END(mutate_to_vhe) + + // At the point where we reach enter_vhe(), we run with + // the MMU off (which is enforced by mutate_to_vhe()). + // We thus need to be in the idmap, or everything will + // explode when enabling the MMU. + + .pushsection .idmap.text, "ax" + +SYM_CODE_START_LOCAL(enter_vhe) // Invalidate TLBs before enabling the MMU tlbi vmalle1 dsb nsh + isb // Enable the EL2 S1 MMU, as set up from EL1 mrs_s x0, SYS_SCTLR_EL12 @@ -143,17 +162,12 @@ SYM_CODE_START_LOCAL(mutate_to_vhe) mov_q x0, INIT_SCTLR_EL1_MMU_OFF msr_s SYS_SCTLR_EL12, x0 - // Hack the exception return to stay at EL2 - mrs x0, spsr_el1 - and x0, x0, #~PSR_MODE_MASK - mov x1, #PSR_MODE_EL2h - orr x0, x0, x1 - msr spsr_el1, x0 - mov x0, xzr -1: eret -SYM_CODE_END(mutate_to_vhe) + eret +SYM_CODE_END(enter_vhe) + + .popsection .macro invalid_vector label SYM_CODE_START_LOCAL(\label) diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c index 03210f644790..0cde47a63beb 100644 --- a/arch/arm64/kernel/machine_kexec_file.c +++ b/arch/arm64/kernel/machine_kexec_file.c @@ -182,8 +182,10 @@ static int create_dtb(struct kimage *image, /* duplicate a device tree blob */ ret = fdt_open_into(initial_boot_params, buf, buf_size); - if (ret) + if (ret) { + vfree(buf); return -EINVAL; + } ret = setup_dtb(image, initrd_load_addr, initrd_len, cmdline, buf); diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 80b62fe49dcf..b3c70a612c7a 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -19,12 +19,13 @@ #include #include #include -#include #include #include u64 gcr_kernel_excl __ro_after_init; +static bool report_fault_once = true; + static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap) { pte_t old_pte = READ_ONCE(*ptep); @@ -86,51 +87,6 @@ int memcmp_pages(struct page *page1, struct page *page2) return ret; } -u8 mte_get_mem_tag(void *addr) -{ - if (!system_supports_mte()) - return 0xFF; - - asm(__MTE_PREAMBLE "ldg %0, [%0]" - : "+r" (addr)); - - return mte_get_ptr_tag(addr); -} - -u8 mte_get_random_tag(void) -{ - void *addr; - - if (!system_supports_mte()) - return 0xFF; - - asm(__MTE_PREAMBLE "irg %0, %0" - : "+r" (addr)); - - return mte_get_ptr_tag(addr); -} - -void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag) -{ - void *ptr = addr; - - if ((!system_supports_mte()) || (size == 0)) - return addr; - - /* Make sure that size is MTE granule aligned. */ - WARN_ON(size & (MTE_GRANULE_SIZE - 1)); - - /* Make sure that the address is MTE granule aligned. */ - WARN_ON((u64)addr & (MTE_GRANULE_SIZE - 1)); - - tag = 0xF0 | tag; - ptr = (void *)__tag_set(ptr, tag); - - mte_assign_mem_tag_range(ptr, size); - - return ptr; -} - void mte_init_tags(u64 max_tag) { static bool gcr_kernel_excl_initialized; @@ -158,6 +114,16 @@ void mte_enable_kernel(void) isb(); } +void mte_set_report_once(bool state) +{ + WRITE_ONCE(report_fault_once, state); +} + +bool mte_report_once(void) +{ + return READ_ONCE(report_fault_once); +} + static void update_sctlr_el1_tcf0(u64 tcf0) { /* ISB required for the kernel uaccess routines */ diff --git a/arch/arm64/kernel/probes/uprobes.c b/arch/arm64/kernel/probes/uprobes.c index a412d8edbcd2..2c247634552b 100644 --- a/arch/arm64/kernel/probes/uprobes.c +++ b/arch/arm64/kernel/probes/uprobes.c @@ -38,7 +38,7 @@ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, /* TODO: Currently we do not support AARCH32 instruction probing */ if (mm->context.flags & MMCF_AARCH32) - return -ENOTSUPP; + return -EOPNOTSUPP; else if (!IS_ALIGNED(addr, AARCH64_INSN_SIZE)) return -EINVAL; diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 4cc1ccc8d6ab..325c83b1a24d 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -398,7 +398,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, ptrauth_thread_init_kernel(p); - if (likely(!(p->flags & PF_KTHREAD))) { + if (likely(!(p->flags & (PF_KTHREAD | PF_IO_WORKER)))) { *childregs = *current_pt_regs(); childregs->regs[0] = 0; diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 3d5c8afca75b..170f42fd6101 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -1797,7 +1797,7 @@ int syscall_trace_enter(struct pt_regs *regs) if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) { tracehook_report_syscall(regs, PTRACE_SYSCALL_ENTER); - if (!in_syscall(regs) || (flags & _TIF_SYSCALL_EMU)) + if (flags & _TIF_SYSCALL_EMU) return NO_SYSCALL; } diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index 0fb42129b469..ad20981dfda4 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -46,7 +46,7 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) /* Terminal record; nothing to unwind */ if (!fp) - return -EINVAL; + return -ENOENT; if (fp & 0xf) return -EINVAL; diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index a67b37a7a47e..d7564891ffe1 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -119,7 +119,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) if (!ret) ret = -EOPNOTSUPP; } else { - __cpu_suspend_exit(); + RCU_NONIDLE(__cpu_suspend_exit()); } unpause_graph_tracing(); diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index 76c0255ecc91..945e6bb326e3 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -29,7 +29,8 @@ ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv \ ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18 ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) $(GCC_PLUGINS_CFLAGS) +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) $(GCC_PLUGINS_CFLAGS) \ + $(CC_FLAGS_LTO) KASAN_SANITIZE := n UBSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index d179056e1af8..5f49df4ffdd8 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -119,7 +119,7 @@ el2_error: .macro invalid_vector label, target = __guest_exit_panic .align 2 -SYM_CODE_START(\label) +SYM_CODE_START_LOCAL(\label) b \target SYM_CODE_END(\label) .endm diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index 9e1a12e10053..351537c12f36 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -149,19 +149,3 @@ SYM_FUNC_START(mte_restore_page_tags) ret SYM_FUNC_END(mte_restore_page_tags) - -/* - * Assign allocation tags for a region of memory based on the pointer tag - * x0 - source pointer - * x1 - size - * - * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and - * size must be non-zero and MTE_GRANULE_SIZE aligned. - */ -SYM_FUNC_START(mte_assign_mem_tag_range) -1: stg x0, [x0] - add x0, x0, #MTE_GRANULE_SIZE - subs x1, x1, #MTE_GRANULE_SIZE - b.gt 1b - ret -SYM_FUNC_END(mte_assign_mem_tag_range) diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 77222d92667a..f188c9092696 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -7,7 +7,6 @@ obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_PTDUMP_CORE) += ptdump.o obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o obj-$(CONFIG_TRANS_TABLE) += trans_pgd.o -obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o KASAN_SANITIZE_physaddr.o += n diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 2e339f0bd958..f37d4e3830b7 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -302,12 +303,24 @@ static void die_kernel_fault(const char *msg, unsigned long addr, static void report_tag_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - bool is_write = ((esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT) != 0; + static bool reported; + bool is_write; + + if (READ_ONCE(reported)) + return; + + /* + * This is used for KASAN tests and assumes that no MTE faults + * happened before running the tests. + */ + if (mte_report_once()) + WRITE_ONCE(reported, true); /* * SAS bits aren't set for all faults reported in EL1, so we can't * find out access size. */ + is_write = !!(esr & ESR_ELx_WNR); kasan_report(addr, 0, is_write, regs->pc); } #else @@ -319,12 +332,8 @@ static inline void report_tag_fault(unsigned long addr, unsigned int esr, static void do_tag_recovery(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - static bool reported; - if (!READ_ONCE(reported)) { - report_tag_fault(addr, esr, regs); - WRITE_ONCE(reported, true); - } + report_tag_fault(addr, esr, regs); /* * Disable MTE Tag Checking on the local CPU for the current EL. @@ -381,6 +390,9 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, } else if (addr < PAGE_SIZE) { msg = "NULL pointer dereference"; } else { + if (kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs)) + return; + msg = "paging request"; } diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 709d98fea90c..0ace5e68efba 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -416,10 +416,10 @@ void __init bootmem_init(void) max_pfn = max_low_pfn = max; min_low_pfn = min; - arm64_numa_init(); + arch_numa_init(); /* - * must be done after arm64_numa_init() which calls numa_init() to + * must be done after arch_numa_init() which calls numa_init() to * initialize node_online_map that gets used in hugetlb_cma_reserve() * while allocating required CMA size across online nodes. */ diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 25af183e4bed..3802cfbdd20d 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1155,7 +1155,7 @@ void vmemmap_free(unsigned long start, unsigned long end, } #endif /* CONFIG_SPARSEMEM_VMEMMAP */ -static inline pud_t * fixmap_pud(unsigned long addr) +static inline pud_t *fixmap_pud(unsigned long addr) { pgd_t *pgdp = pgd_offset_k(addr); p4d_t *p4dp = p4d_offset(pgdp, addr); @@ -1166,7 +1166,7 @@ static inline pud_t * fixmap_pud(unsigned long addr) return pud_offset_kimg(p4dp, addr); } -static inline pmd_t * fixmap_pmd(unsigned long addr) +static inline pmd_t *fixmap_pmd(unsigned long addr) { pud_t *pudp = fixmap_pud(addr); pud_t pud = READ_ONCE(*pudp); @@ -1176,7 +1176,7 @@ static inline pmd_t * fixmap_pmd(unsigned long addr) return pmd_offset_kimg(pudp, addr); } -static inline pte_t * fixmap_pte(unsigned long addr) +static inline pte_t *fixmap_pte(unsigned long addr) { return &bm_pte[pte_index(addr)]; } @@ -1444,16 +1444,19 @@ static void __remove_pgd_mapping(pgd_t *pgdir, unsigned long start, u64 size) free_empty_tables(start, end, PAGE_OFFSET, PAGE_END); } -static bool inside_linear_region(u64 start, u64 size) +struct range arch_get_mappable_range(void) { + struct range mhp_range; + /* * Linear mapping region is the range [PAGE_OFFSET..(PAGE_END - 1)] * accommodating both its ends but excluding PAGE_END. Max physical * range which can be mapped inside this linear mapping range, must * also be derived from its end points. */ - return start >= __pa(_PAGE_OFFSET(vabits_actual)) && - (start + size - 1) <= __pa(PAGE_END - 1); + mhp_range.start = __pa(_PAGE_OFFSET(vabits_actual)); + mhp_range.end = __pa(PAGE_END - 1); + return mhp_range; } int arch_add_memory(int nid, u64 start, u64 size, @@ -1461,12 +1464,14 @@ int arch_add_memory(int nid, u64 start, u64 size, { int ret, flags = 0; - if (!inside_linear_region(start, size)) { - pr_err("[%llx %llx] is outside linear mapping region\n", start, start + size); - return -EINVAL; - } + VM_BUG_ON(!mhp_range_allowed(start, size, true)); - if (rodata_full || debug_pagealloc_enabled()) + /* + * KFENCE requires linear map to be mapped at page granularity, so that + * it is possible to protect/unprotect single pages in the KFENCE pool. + */ + if (rodata_full || debug_pagealloc_enabled() || + IS_ENABLED(CONFIG_KFENCE)) flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index 89dd2fcf38fa..34e91224adc3 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -7,7 +7,7 @@ config CSKY select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_USE_BUILTIN_BSWAP - select ARCH_USE_QUEUED_RWLOCKS if NR_CPUS>2 + select ARCH_USE_QUEUED_RWLOCKS select ARCH_WANT_FRAME_POINTERS if !CPU_CK610 select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT select COMMON_CLK @@ -35,6 +35,9 @@ config CSKY select GENERIC_IRQ_MULTI_HANDLER select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD + select GENERIC_TIME_VSYSCALL + select GENERIC_VDSO_32 + select GENERIC_GETTIMEOFDAY select GX6605S_TIMER if CPU_CK610 select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_AUDITSYSCALL @@ -43,11 +46,14 @@ config CSKY select HAVE_CONTEXT_TRACKING select HAVE_VIRT_CPU_ACCOUNTING_GEN select HAVE_DEBUG_BUGVERBOSE + select HAVE_DEBUG_KMEMLEAK select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_GENERIC_VDSO select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_ERROR_INJECTION + select HAVE_FUTEX_CMPXCHG if FUTEX && SMP select HAVE_FTRACE_MCOUNT_RECORD select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZO @@ -192,6 +198,22 @@ config CPU_CK860 endchoice choice + prompt "PAGE OFFSET" + default PAGE_OFFSET_80000000 + +config PAGE_OFFSET_80000000 + bool "PAGE OFFSET 2G (user:kernel = 2:2)" + +config PAGE_OFFSET_A0000000 + bool "PAGE OFFSET 2.5G (user:kernel = 2.5:1.5)" +endchoice + +config PAGE_OFFSET + hex + default 0x80000000 if PAGE_OFFSET_80000000 + default 0xa0000000 if PAGE_OFFSET_A0000000 +choice + prompt "C-SKY PMU type" depends on PERF_EVENTS depends on CPU_CK807 || CPU_CK810 || CPU_CK860 diff --git a/arch/csky/abiv1/inc/abi/cacheflush.h b/arch/csky/abiv1/inc/abi/cacheflush.h index d3e04208d53c..6cab7afae962 100644 --- a/arch/csky/abiv1/inc/abi/cacheflush.h +++ b/arch/csky/abiv1/inc/abi/cacheflush.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_CACHEFLUSH_H #define __ABI_CSKY_CACHEFLUSH_H diff --git a/arch/csky/abiv1/inc/abi/ckmmu.h b/arch/csky/abiv1/inc/abi/ckmmu.h index ba8eb5870835..416b30c57983 100644 --- a/arch/csky/abiv1/inc/abi/ckmmu.h +++ b/arch/csky/abiv1/inc/abi/ckmmu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CKMMUV1_H #define __ASM_CSKY_CKMMUV1_H @@ -89,13 +88,14 @@ static inline void tlb_invalid_indexed(void) cpwcr("cpcr8", 0x02000000); } -static inline void setup_pgd(unsigned long pgd, bool kernel) +static inline void setup_pgd(pgd_t *pgd, int asid) { - cpwcr("cpcr29", pgd | BIT(0)); + cpwcr("cpcr29", __pa(pgd) | BIT(0)); + write_mmu_entryhi(asid); } -static inline unsigned long get_pgd(void) +static inline pgd_t *get_pgd(void) { - return cprcr("cpcr29") & ~BIT(0); + return __va(cprcr("cpcr29") & ~BIT(0)); } #endif /* __ASM_CSKY_CKMMUV1_H */ diff --git a/arch/csky/abiv1/inc/abi/entry.h b/arch/csky/abiv1/inc/abi/entry.h index 13c23e2c707c..b6a2109b895e 100644 --- a/arch/csky/abiv1/inc/abi/entry.h +++ b/arch/csky/abiv1/inc/abi/entry.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ENTRY_H #define __ASM_CSKY_ENTRY_H diff --git a/arch/csky/abiv1/inc/abi/page.h b/arch/csky/abiv1/inc/abi/page.h index c864519117c7..2d2159933b76 100644 --- a/arch/csky/abiv1/inc/abi/page.h +++ b/arch/csky/abiv1/inc/abi/page.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #include diff --git a/arch/csky/abiv1/inc/abi/pgtable-bits.h b/arch/csky/abiv1/inc/abi/pgtable-bits.h index d605445aad9a..752c8b3f9194 100644 --- a/arch/csky/abiv1/inc/abi/pgtable-bits.h +++ b/arch/csky/abiv1/inc/abi/pgtable-bits.h @@ -1,37 +1,49 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGTABLE_BITS_H #define __ASM_CSKY_PGTABLE_BITS_H /* implemented in software */ -#define _PAGE_ACCESSED (1<<3) -#define PAGE_ACCESSED_BIT (3) - +#define _PAGE_PRESENT (1<<0) #define _PAGE_READ (1<<1) #define _PAGE_WRITE (1<<2) -#define _PAGE_PRESENT (1<<0) - +#define _PAGE_ACCESSED (1<<3) #define _PAGE_MODIFIED (1<<4) -#define PAGE_MODIFIED_BIT (4) /* implemented in hardware */ #define _PAGE_GLOBAL (1<<6) - #define _PAGE_VALID (1<<7) -#define PAGE_VALID_BIT (7) - #define _PAGE_DIRTY (1<<8) -#define PAGE_DIRTY_BIT (8) #define _PAGE_CACHE (3<<9) #define _PAGE_UNCACHE (2<<9) #define _PAGE_SO _PAGE_UNCACHE - #define _CACHE_MASK (7<<9) -#define _CACHE_CACHED (_PAGE_VALID | _PAGE_CACHE) -#define _CACHE_UNCACHED (_PAGE_VALID | _PAGE_UNCACHE) +#define _CACHE_CACHED _PAGE_CACHE +#define _CACHE_UNCACHED _PAGE_UNCACHE + +#define _PAGE_PROT_NONE _PAGE_READ + +/* + * Encode and decode a swap entry + * + * Format of swap PTE: + * bit 0: _PAGE_PRESENT (zero) + * bit 1: _PAGE_READ (zero) + * bit 2 - 5: swap type[0 - 3] + * bit 6: _PAGE_GLOBAL (zero) + * bit 7: _PAGE_VALID (zero) + * bit 8: swap type[4] + * bit 9 - 31: swap offset + */ +#define __swp_type(x) ((((x).val >> 2) & 0xf) | \ + (((x).val >> 4) & 0x10)) +#define __swp_offset(x) ((x).val >> 9) +#define __swp_entry(type, offset) ((swp_entry_t) { \ + ((type & 0xf) << 2) | \ + ((type & 0x10) << 4) | \ + ((offset) << 9)}) #define HAVE_ARCH_UNMAPPED_AREA diff --git a/arch/csky/abiv1/inc/abi/reg_ops.h b/arch/csky/abiv1/inc/abi/reg_ops.h index a153bd3918f7..abd01a243388 100644 --- a/arch/csky/abiv1/inc/abi/reg_ops.h +++ b/arch/csky/abiv1/inc/abi/reg_ops.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_REG_OPS_H #define __ABI_REG_OPS_H diff --git a/arch/csky/abiv1/inc/abi/regdef.h b/arch/csky/abiv1/inc/abi/regdef.h index 104707fbdcc1..7b386fd67070 100644 --- a/arch/csky/abiv1/inc/abi/regdef.h +++ b/arch/csky/abiv1/inc/abi/regdef.h @@ -1,10 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_REGDEF_H #define __ASM_CSKY_REGDEF_H +#ifdef __ASSEMBLY__ #define syscallid r1 +#else +#define syscallid "r1" +#endif + #define regs_syscallid(regs) regs->regs[9] #define regs_fp(regs) regs->regs[2] diff --git a/arch/csky/abiv1/inc/abi/string.h b/arch/csky/abiv1/inc/abi/string.h index 0cd43384f8d2..9d95594b0feb 100644 --- a/arch/csky/abiv1/inc/abi/string.h +++ b/arch/csky/abiv1/inc/abi/string.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_STRING_H #define __ABI_CSKY_STRING_H diff --git a/arch/csky/abiv1/inc/abi/switch_context.h b/arch/csky/abiv1/inc/abi/switch_context.h index 17c82686498e..ec73fd7c9f87 100644 --- a/arch/csky/abiv1/inc/abi/switch_context.h +++ b/arch/csky/abiv1/inc/abi/switch_context.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_PTRACE_H #define __ABI_CSKY_PTRACE_H diff --git a/arch/csky/abiv1/inc/abi/vdso.h b/arch/csky/abiv1/inc/abi/vdso.h index 14352f524f1d..9e6d0a2fdd2b 100644 --- a/arch/csky/abiv1/inc/abi/vdso.h +++ b/arch/csky/abiv1/inc/abi/vdso.h @@ -1,17 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#include +#ifndef __ABI_CSKY_VDSO_H +#define __ABI_CSKY_VDSO_H -static inline int setup_vdso_page(unsigned short *ptr) -{ - int err = 0; +/* movi r1, 127; addi r1, (139 - 127) */ +#define SET_SYSCALL_ID .long 0x20b167f1 - /* movi r1, 127 */ - err |= __put_user(0x67f1, ptr + 0); - /* addi r1, (139 - 127) */ - err |= __put_user(0x20b1, ptr + 1); - /* trap 0 */ - err |= __put_user(0x0008, ptr + 2); - - return err; -} +#endif /* __ABI_CSKY_VDSO_H */ diff --git a/arch/csky/abiv2/cacheflush.c b/arch/csky/abiv2/cacheflush.c index 790f1ebfba44..39c51399dd81 100644 --- a/arch/csky/abiv2/cacheflush.c +++ b/arch/csky/abiv2/cacheflush.c @@ -12,6 +12,9 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, unsigned long addr; struct page *page; + if (!pfn_valid(pte_pfn(*pte))) + return; + page = pfn_to_page(pte_pfn(*pte)); if (page == ZERO_PAGE(0)) return; diff --git a/arch/csky/abiv2/inc/abi/ckmmu.h b/arch/csky/abiv2/inc/abi/ckmmu.h index 73ded7c72482..64215f2380f1 100644 --- a/arch/csky/abiv2/inc/abi/ckmmu.h +++ b/arch/csky/abiv2/inc/abi/ckmmu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CKMMUV2_H #define __ASM_CSKY_CKMMUV2_H @@ -78,8 +77,13 @@ static inline void tlb_read(void) static inline void tlb_invalid_all(void) { #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.alls\n":::"memory"); sync_is(); + asm volatile( + "tlbi.alls \n" + "sync.i \n" + : + : + : "memory"); #else mtcr("cr<8, 15>", 0x04000000); #endif @@ -88,8 +92,13 @@ static inline void tlb_invalid_all(void) static inline void local_tlb_invalid_all(void) { #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.all\n":::"memory"); sync_is(); + asm volatile( + "tlbi.all \n" + "sync.i \n" + : + : + : "memory"); #else tlb_invalid_all(); #endif @@ -100,16 +109,31 @@ static inline void tlb_invalid_indexed(void) mtcr("cr<8, 15>", 0x02000000); } -static inline void setup_pgd(unsigned long pgd, bool kernel) +#define NOP32 ".long 0x4820c400\n" + +static inline void setup_pgd(pgd_t *pgd, int asid) { - if (kernel) - mtcr("cr<28, 15>", pgd | BIT(0)); - else - mtcr("cr<29, 15>", pgd | BIT(0)); +#ifdef CONFIG_CPU_HAS_TLBI + sync_is(); +#else + mb(); +#endif + asm volatile( +#ifdef CONFIG_CPU_HAS_TLBI + "mtcr %1, cr<28, 15> \n" +#endif + "mtcr %1, cr<29, 15> \n" + "mtcr %0, cr< 4, 15> \n" + ".rept 64 \n" + NOP32 + ".endr \n" + : + :"r"(asid), "r"(__pa(pgd) | BIT(0)) + :"memory"); } -static inline unsigned long get_pgd(void) +static inline pgd_t *get_pgd(void) { - return mfcr("cr<29, 15>") & ~BIT(0); + return __va(mfcr("cr<29, 15>") & ~BIT(0)); } #endif /* __ASM_CSKY_CKMMUV2_H */ diff --git a/arch/csky/abiv2/inc/abi/entry.h b/arch/csky/abiv2/inc/abi/entry.h index bedcc6f06bba..cca63e699b58 100644 --- a/arch/csky/abiv2/inc/abi/entry.h +++ b/arch/csky/abiv2/inc/abi/entry.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ENTRY_H #define __ASM_CSKY_ENTRY_H @@ -26,6 +25,9 @@ stw tls, (sp, 0) stw lr, (sp, 4) + RD_MEH lr + WR_MEH lr + mfcr lr, epc movi tls, \epc_inc add lr, tls @@ -231,6 +233,16 @@ mtcr \rx, cr<8, 15> .endm +#ifdef CONFIG_PAGE_OFFSET_80000000 +#define MSA_SET cr<30, 15> +#define MSA_CLR cr<31, 15> +#endif + +#ifdef CONFIG_PAGE_OFFSET_A0000000 +#define MSA_SET cr<31, 15> +#define MSA_CLR cr<30, 15> +#endif + .macro SETUP_MMU /* Init psr and enable ee */ lrw r6, DEFAULT_PSR_VALUE @@ -281,15 +293,15 @@ * 31 - 29 | 28 - 9 | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 * BA Reserved SH WA B SO SEC C D V */ - mfcr r6, cr<30, 15> /* Get MSA0 */ + mfcr r6, MSA_SET /* Get MSA */ 2: lsri r6, 29 lsli r6, 29 addi r6, 0x1ce - mtcr r6, cr<30, 15> /* Set MSA0 */ + mtcr r6, MSA_SET /* Set MSA */ movi r6, 0 - mtcr r6, cr<31, 15> /* Clr MSA1 */ + mtcr r6, MSA_CLR /* Clr MSA */ /* enable MMU */ mfcr r6, cr18 diff --git a/arch/csky/abiv2/inc/abi/fpu.h b/arch/csky/abiv2/inc/abi/fpu.h index 09e2700a3693..aabb79355013 100644 --- a/arch/csky/abiv2/inc/abi/fpu.h +++ b/arch/csky/abiv2/inc/abi/fpu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_FPU_H #define __ASM_CSKY_FPU_H diff --git a/arch/csky/abiv2/inc/abi/page.h b/arch/csky/abiv2/inc/abi/page.h index 0a70cb553dca..cf005f13cd15 100644 --- a/arch/csky/abiv2/inc/abi/page.h +++ b/arch/csky/abiv2/inc/abi/page.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. static inline void clear_user_page(void *addr, unsigned long vaddr, struct page *page) diff --git a/arch/csky/abiv2/inc/abi/pgtable-bits.h b/arch/csky/abiv2/inc/abi/pgtable-bits.h index 137f7932c83b..7e7f389f546f 100644 --- a/arch/csky/abiv2/inc/abi/pgtable-bits.h +++ b/arch/csky/abiv2/inc/abi/pgtable-bits.h @@ -1,37 +1,48 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGTABLE_BITS_H #define __ASM_CSKY_PGTABLE_BITS_H /* implemented in software */ #define _PAGE_ACCESSED (1<<7) -#define PAGE_ACCESSED_BIT (7) - #define _PAGE_READ (1<<8) #define _PAGE_WRITE (1<<9) #define _PAGE_PRESENT (1<<10) - #define _PAGE_MODIFIED (1<<11) -#define PAGE_MODIFIED_BIT (11) /* implemented in hardware */ #define _PAGE_GLOBAL (1<<0) - #define _PAGE_VALID (1<<1) -#define PAGE_VALID_BIT (1) - #define _PAGE_DIRTY (1<<2) -#define PAGE_DIRTY_BIT (2) #define _PAGE_SO (1<<5) #define _PAGE_BUF (1<<6) - #define _PAGE_CACHE (1<<3) - #define _CACHE_MASK _PAGE_CACHE -#define _CACHE_CACHED (_PAGE_VALID | _PAGE_CACHE | _PAGE_BUF) -#define _CACHE_UNCACHED (_PAGE_VALID) +#define _CACHE_CACHED (_PAGE_CACHE | _PAGE_BUF) +#define _CACHE_UNCACHED (0) + +#define _PAGE_PROT_NONE _PAGE_WRITE + +/* + * Encode and decode a swap entry + * + * Format of swap PTE: + * bit 0: _PAGE_GLOBAL (zero) + * bit 1: _PAGE_VALID (zero) + * bit 2 - 6: swap type + * bit 7 - 8: swap offset[0 - 1] + * bit 9: _PAGE_WRITE (zero) + * bit 10: _PAGE_PRESENT (zero) + * bit 11 - 31: swap offset[2 - 22] + */ +#define __swp_type(x) (((x).val >> 2) & 0x1f) +#define __swp_offset(x) ((((x).val >> 7) & 0x3) | \ + (((x).val >> 9) & 0x7ffffc)) +#define __swp_entry(type, offset) ((swp_entry_t) { \ + ((type & 0x1f) << 2) | \ + ((offset & 0x3) << 7) | \ + ((offset & 0x7ffffc) << 9)}) #endif /* __ASM_CSKY_PGTABLE_BITS_H */ diff --git a/arch/csky/abiv2/inc/abi/reg_ops.h b/arch/csky/abiv2/inc/abi/reg_ops.h index ae82c3f26a6b..49ba18a64751 100644 --- a/arch/csky/abiv2/inc/abi/reg_ops.h +++ b/arch/csky/abiv2/inc/abi/reg_ops.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_REG_OPS_H #define __ABI_REG_OPS_H diff --git a/arch/csky/abiv2/inc/abi/regdef.h b/arch/csky/abiv2/inc/abi/regdef.h index d7328bbc1ce7..0933addbc27b 100644 --- a/arch/csky/abiv2/inc/abi/regdef.h +++ b/arch/csky/abiv2/inc/abi/regdef.h @@ -1,10 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_REGDEF_H #define __ASM_CSKY_REGDEF_H +#ifdef __ASSEMBLY__ #define syscallid r7 +#else +#define syscallid "r7" +#endif + #define regs_syscallid(regs) regs->regs[3] #define regs_fp(regs) regs->regs[4] diff --git a/arch/csky/abiv2/inc/abi/switch_context.h b/arch/csky/abiv2/inc/abi/switch_context.h index 73a81245a3b3..5dd5c3f4ee7e 100644 --- a/arch/csky/abiv2/inc/abi/switch_context.h +++ b/arch/csky/abiv2/inc/abi/switch_context.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_PTRACE_H #define __ABI_CSKY_PTRACE_H diff --git a/arch/csky/abiv2/inc/abi/vdso.h b/arch/csky/abiv2/inc/abi/vdso.h index b60d4a070326..40fd10d893ff 100644 --- a/arch/csky/abiv2/inc/abi/vdso.h +++ b/arch/csky/abiv2/inc/abi/vdso.h @@ -3,21 +3,7 @@ #ifndef __ABI_CSKY_VDSO_H #define __ABI_CSKY_VDSO_H -#include +/* movi r7, 173 */ +#define SET_SYSCALL_ID .long 0x008bea07 -static inline int setup_vdso_page(unsigned short *ptr) -{ - int err = 0; - - /* movi r7, 173 */ - err |= __put_user(0xea07, ptr); - err |= __put_user(0x008b, ptr+1); - - /* trap 0 */ - err |= __put_user(0xc000, ptr+2); - err |= __put_user(0x2020, ptr+3); - - return err; -} - -#endif /* __ABI_CSKY_STRING_H */ +#endif /* __ABI_CSKY_VDSO_H */ diff --git a/arch/csky/abiv2/sysdep.h b/arch/csky/abiv2/sysdep.h index bbbedfd34777..61abe9201c50 100644 --- a/arch/csky/abiv2/sysdep.h +++ b/arch/csky/abiv2/sysdep.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __SYSDEP_H #define __SYSDEP_H diff --git a/arch/csky/include/asm/addrspace.h b/arch/csky/include/asm/addrspace.h index d1c2ede692ed..6fc05d44536c 100644 --- a/arch/csky/include/asm/addrspace.h +++ b/arch/csky/include/asm/addrspace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ADDRSPACE_H #define __ASM_CSKY_ADDRSPACE_H diff --git a/arch/csky/include/asm/atomic.h b/arch/csky/include/asm/atomic.h deleted file mode 100644 index e369d73b13e3..000000000000 --- a/arch/csky/include/asm/atomic.h +++ /dev/null @@ -1,212 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef __ASM_CSKY_ATOMIC_H -#define __ASM_CSKY_ATOMIC_H - -#include -#include -#include - -#ifdef CONFIG_CPU_HAS_LDSTEX - -#define __atomic_add_unless __atomic_add_unless -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - unsigned long tmp, ret; - - smp_mb(); - - asm volatile ( - "1: ldex.w %0, (%3) \n" - " mov %1, %0 \n" - " cmpne %0, %4 \n" - " bf 2f \n" - " add %0, %2 \n" - " stex.w %0, (%3) \n" - " bez %0, 1b \n" - "2: \n" - : "=&r" (tmp), "=&r" (ret) - : "r" (a), "r"(&v->counter), "r"(u) - : "memory"); - - if (ret != u) - smp_mb(); - - return ret; -} - -#define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp; \ - \ - asm volatile ( \ - "1: ldex.w %0, (%2) \n" \ - " " #op " %0, %1 \n" \ - " stex.w %0, (%2) \n" \ - " bez %0, 1b \n" \ - : "=&r" (tmp) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ -} - -#define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ -{ \ - unsigned long tmp, ret; \ - \ - smp_mb(); \ - asm volatile ( \ - "1: ldex.w %0, (%3) \n" \ - " " #op " %0, %2 \n" \ - " mov %1, %0 \n" \ - " stex.w %0, (%3) \n" \ - " bez %0, 1b \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - smp_mb(); \ - \ - return ret; \ -} - -#define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp, ret; \ - \ - smp_mb(); \ - asm volatile ( \ - "1: ldex.w %0, (%3) \n" \ - " mov %1, %0 \n" \ - " " #op " %0, %2 \n" \ - " stex.w %0, (%3) \n" \ - " bez %0, 1b \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - smp_mb(); \ - \ - return ret; \ -} - -#else /* CONFIG_CPU_HAS_LDSTEX */ - -#include - -#define __atomic_add_unless __atomic_add_unless -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - unsigned long tmp, ret, flags; - - raw_local_irq_save(flags); - - asm volatile ( - " ldw %0, (%3) \n" - " mov %1, %0 \n" - " cmpne %0, %4 \n" - " bf 2f \n" - " add %0, %2 \n" - " stw %0, (%3) \n" - "2: \n" - : "=&r" (tmp), "=&r" (ret) - : "r" (a), "r"(&v->counter), "r"(u) - : "memory"); - - raw_local_irq_restore(flags); - - return ret; -} - -#define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp, flags; \ - \ - raw_local_irq_save(flags); \ - \ - asm volatile ( \ - " ldw %0, (%2) \n" \ - " " #op " %0, %1 \n" \ - " stw %0, (%2) \n" \ - : "=&r" (tmp) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - \ - raw_local_irq_restore(flags); \ -} - -#define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ -{ \ - unsigned long tmp, ret, flags; \ - \ - raw_local_irq_save(flags); \ - \ - asm volatile ( \ - " ldw %0, (%3) \n" \ - " " #op " %0, %2 \n" \ - " stw %0, (%3) \n" \ - " mov %1, %0 \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - \ - raw_local_irq_restore(flags); \ - \ - return ret; \ -} - -#define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp, ret, flags; \ - \ - raw_local_irq_save(flags); \ - \ - asm volatile ( \ - " ldw %0, (%3) \n" \ - " mov %1, %0 \n" \ - " " #op " %0, %2 \n" \ - " stw %0, (%3) \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - \ - raw_local_irq_restore(flags); \ - \ - return ret; \ -} - -#endif /* CONFIG_CPU_HAS_LDSTEX */ - -#define atomic_add_return atomic_add_return -ATOMIC_OP_RETURN(add, +) -#define atomic_sub_return atomic_sub_return -ATOMIC_OP_RETURN(sub, -) - -#define atomic_fetch_add atomic_fetch_add -ATOMIC_FETCH_OP(add, +) -#define atomic_fetch_sub atomic_fetch_sub -ATOMIC_FETCH_OP(sub, -) -#define atomic_fetch_and atomic_fetch_and -ATOMIC_FETCH_OP(and, &) -#define atomic_fetch_or atomic_fetch_or -ATOMIC_FETCH_OP(or, |) -#define atomic_fetch_xor atomic_fetch_xor -ATOMIC_FETCH_OP(xor, ^) - -#define atomic_and atomic_and -ATOMIC_OP(and, &) -#define atomic_or atomic_or -ATOMIC_OP(or, |) -#define atomic_xor atomic_xor -ATOMIC_OP(xor, ^) - -#undef ATOMIC_FETCH_OP -#undef ATOMIC_OP_RETURN -#undef ATOMIC_OP - -#include - -#endif /* __ASM_CSKY_ATOMIC_H */ diff --git a/arch/csky/include/asm/barrier.h b/arch/csky/include/asm/barrier.h index a430e7fddf35..84fc600c8b45 100644 --- a/arch/csky/include/asm/barrier.h +++ b/arch/csky/include/asm/barrier.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BARRIER_H #define __ASM_CSKY_BARRIER_H @@ -8,6 +7,61 @@ #define nop() asm volatile ("nop\n":::"memory") +#ifdef CONFIG_SMP + +/* + * bar.brwarws: ordering barrier for all load/store instructions + * before/after + * + * |31|30 26|25 21|20 16|15 10|9 5|4 0| + * 1 10000 00000 00000 100001 00001 0 bw br aw ar + * + * b: before + * a: after + * r: read + * w: write + * + * Here are all combinations: + * + * bar.brw + * bar.br + * bar.bw + * bar.arw + * bar.ar + * bar.aw + * bar.brwarw + * bar.brarw + * bar.bwarw + * bar.brwar + * bar.brwaw + * bar.brar + * bar.bwaw + */ +#define __bar_brw() asm volatile (".long 0x842cc000\n":::"memory") +#define __bar_br() asm volatile (".long 0x8424c000\n":::"memory") +#define __bar_bw() asm volatile (".long 0x8428c000\n":::"memory") +#define __bar_arw() asm volatile (".long 0x8423c000\n":::"memory") +#define __bar_ar() asm volatile (".long 0x8421c000\n":::"memory") +#define __bar_aw() asm volatile (".long 0x8422c000\n":::"memory") +#define __bar_brwarw() asm volatile (".long 0x842fc000\n":::"memory") +#define __bar_brarw() asm volatile (".long 0x8427c000\n":::"memory") +#define __bar_bwarw() asm volatile (".long 0x842bc000\n":::"memory") +#define __bar_brwar() asm volatile (".long 0x842dc000\n":::"memory") +#define __bar_brwaw() asm volatile (".long 0x842ec000\n":::"memory") +#define __bar_brar() asm volatile (".long 0x8425c000\n":::"memory") +#define __bar_brar() asm volatile (".long 0x8425c000\n":::"memory") +#define __bar_bwaw() asm volatile (".long 0x842ac000\n":::"memory") + +#define __smp_mb() __bar_brwarw() +#define __smp_rmb() __bar_brar() +#define __smp_wmb() __bar_bwaw() + +#define ACQUIRE_FENCE ".long 0x8427c000\n" +#define __smp_acquire_fence() __bar_brarw() +#define __smp_release_fence() __bar_brwaw() + +#endif /* CONFIG_SMP */ + /* * sync: completion barrier, all sync.xx instructions * guarantee the last response recieved by bus transaction @@ -15,31 +69,14 @@ * sync.s: inherit from sync, but also shareable to other cores * sync.i: inherit from sync, but also flush cpu pipeline * sync.is: the same with sync.i + sync.s - * - * bar.brwarw: ordering barrier for all load/store instructions before it - * bar.brwarws: ordering barrier for all load/store instructions before it - * and shareable to other cores - * bar.brar: ordering barrier for all load instructions before it - * bar.brars: ordering barrier for all load instructions before it - * and shareable to other cores - * bar.bwaw: ordering barrier for all store instructions before it - * bar.bwaws: ordering barrier for all store instructions before it - * and shareable to other cores */ +#define mb() asm volatile ("sync\n":::"memory") #ifdef CONFIG_CPU_HAS_CACHEV2 -#define mb() asm volatile ("sync.s\n":::"memory") - -#ifdef CONFIG_SMP -#define __smp_mb() asm volatile ("bar.brwarws\n":::"memory") -#define __smp_rmb() asm volatile ("bar.brars\n":::"memory") -#define __smp_wmb() asm volatile ("bar.bwaws\n":::"memory") -#endif /* CONFIG_SMP */ - -#define sync_is() asm volatile ("sync.is\n":::"memory") - -#else /* !CONFIG_CPU_HAS_CACHEV2 */ -#define mb() asm volatile ("sync\n":::"memory") +/* + * Using three sync.is to prevent speculative PTW + */ +#define sync_is() asm volatile ("sync.is\nsync.is\nsync.is\n":::"memory") #endif #include diff --git a/arch/csky/include/asm/bitops.h b/arch/csky/include/asm/bitops.h index 43b9838bff63..91818787d860 100644 --- a/arch/csky/include/asm/bitops.h +++ b/arch/csky/include/asm/bitops.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BITOPS_H #define __ASM_CSKY_BITOPS_H diff --git a/arch/csky/include/asm/bug.h b/arch/csky/include/asm/bug.h index 33ebd16b9c78..03f1a5f9184a 100644 --- a/arch/csky/include/asm/bug.h +++ b/arch/csky/include/asm/bug.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BUG_H #define __ASM_CSKY_BUG_H @@ -21,6 +20,8 @@ do { \ struct pt_regs; void die(struct pt_regs *regs, const char *str); +void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr); + void show_regs(struct pt_regs *regs); void show_code(struct pt_regs *regs); diff --git a/arch/csky/include/asm/cacheflush.h b/arch/csky/include/asm/cacheflush.h index f0b8f25429a2..d0f9eafe8988 100644 --- a/arch/csky/include/asm/cacheflush.h +++ b/arch/csky/include/asm/cacheflush.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CACHEFLUSH_H #define __ASM_CSKY_CACHEFLUSH_H diff --git a/arch/csky/include/asm/checksum.h b/arch/csky/include/asm/checksum.h index 7685824291b1..aa12ef4b9080 100644 --- a/arch/csky/include/asm/checksum.h +++ b/arch/csky/include/asm/checksum.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CHECKSUM_H #define __ASM_CSKY_CHECKSUM_H diff --git a/arch/csky/include/asm/clocksource.h b/arch/csky/include/asm/clocksource.h new file mode 100644 index 000000000000..54da0e49efa1 --- /dev/null +++ b/arch/csky/include/asm/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_CLOCKSOURCE_H +#define __ASM_VDSO_CSKY_CLOCKSOURCE_H + +#include + +#endif diff --git a/arch/csky/include/asm/cmpxchg.h b/arch/csky/include/asm/cmpxchg.h index 89224530a0ee..dabc8e46ce7b 100644 --- a/arch/csky/include/asm/cmpxchg.h +++ b/arch/csky/include/asm/cmpxchg.h @@ -3,12 +3,12 @@ #ifndef __ASM_CSKY_CMPXCHG_H #define __ASM_CSKY_CMPXCHG_H -#ifdef CONFIG_CPU_HAS_LDSTEX +#ifdef CONFIG_SMP #include extern void __bad_xchg(void); -#define __xchg(new, ptr, size) \ +#define __xchg_relaxed(new, ptr, size) \ ({ \ __typeof__(ptr) __ptr = (ptr); \ __typeof__(new) __new = (new); \ @@ -16,7 +16,6 @@ extern void __bad_xchg(void); unsigned long tmp; \ switch (size) { \ case 4: \ - smp_mb(); \ asm volatile ( \ "1: ldex.w %0, (%3) \n" \ " mov %1, %2 \n" \ @@ -25,7 +24,6 @@ extern void __bad_xchg(void); : "=&r" (__ret), "=&r" (tmp) \ : "r" (__new), "r"(__ptr) \ :); \ - smp_mb(); \ break; \ default: \ __bad_xchg(); \ @@ -33,9 +31,10 @@ extern void __bad_xchg(void); __ret; \ }) -#define xchg(ptr, x) (__xchg((x), (ptr), sizeof(*(ptr)))) +#define xchg_relaxed(ptr, x) \ + (__xchg_relaxed((x), (ptr), sizeof(*(ptr)))) -#define __cmpxchg(ptr, old, new, size) \ +#define __cmpxchg_relaxed(ptr, old, new, size) \ ({ \ __typeof__(ptr) __ptr = (ptr); \ __typeof__(new) __new = (new); \ @@ -44,7 +43,6 @@ extern void __bad_xchg(void); __typeof__(*(ptr)) __ret; \ switch (size) { \ case 4: \ - smp_mb(); \ asm volatile ( \ "1: ldex.w %0, (%3) \n" \ " cmpne %0, %4 \n" \ @@ -56,7 +54,6 @@ extern void __bad_xchg(void); : "=&r" (__ret), "=&r" (__tmp) \ : "r" (__new), "r"(__ptr), "r"(__old) \ :); \ - smp_mb(); \ break; \ default: \ __bad_xchg(); \ @@ -64,8 +61,18 @@ extern void __bad_xchg(void); __ret; \ }) -#define cmpxchg(ptr, o, n) \ - (__cmpxchg((ptr), (o), (n), sizeof(*(ptr)))) +#define cmpxchg_relaxed(ptr, o, n) \ + (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) + +#define cmpxchg(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) __ret; \ + __smp_release_fence(); \ + __ret = cmpxchg_relaxed(ptr, o, n); \ + __smp_acquire_fence(); \ + __ret; \ +}) + #else #include #endif diff --git a/arch/csky/include/asm/elf.h b/arch/csky/include/asm/elf.h index eb2cc5a673b5..48b83e283ed4 100644 --- a/arch/csky/include/asm/elf.h +++ b/arch/csky/include/asm/elf.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ELF_H #define __ASM_CSKY_ELF_H diff --git a/arch/csky/include/asm/fixmap.h b/arch/csky/include/asm/fixmap.h index 4b589cc20900..49a77cbbe2a9 100644 --- a/arch/csky/include/asm/fixmap.h +++ b/arch/csky/include/asm/fixmap.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_FIXMAP_H #define __ASM_CSKY_FIXMAP_H diff --git a/arch/csky/include/asm/ftrace.h b/arch/csky/include/asm/ftrace.h index fae72b0b1374..9b86341731b6 100644 --- a/arch/csky/include/asm/ftrace.h +++ b/arch/csky/include/asm/ftrace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_FTRACE_H #define __ASM_CSKY_FTRACE_H diff --git a/arch/csky/include/asm/futex.h b/arch/csky/include/asm/futex.h new file mode 100644 index 000000000000..6cfd312723fa --- /dev/null +++ b/arch/csky/include/asm/futex.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_CSKY_FUTEX_H +#define __ASM_CSKY_FUTEX_H + +#ifndef CONFIG_SMP +#include +#else +#include +#include +#include +#include + +#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ +{ \ + u32 tmp; \ + \ + __atomic_pre_full_fence(); \ + \ + __asm__ __volatile__ ( \ + "1: ldex.w %[ov], %[u] \n" \ + " "insn" \n" \ + "2: stex.w %[t], %[u] \n" \ + " bez %[t], 1b \n" \ + " br 4f \n" \ + "3: mov %[r], %[e] \n" \ + "4: \n" \ + " .section __ex_table,\"a\" \n" \ + " .balign 4 \n" \ + " .long 1b, 3b \n" \ + " .long 2b, 3b \n" \ + " .previous \n" \ + : [r] "+r" (ret), [ov] "=&r" (oldval), \ + [u] "+m" (*uaddr), [t] "=&r" (tmp) \ + : [op] "Jr" (oparg), [e] "jr" (-EFAULT) \ + : "memory"); \ + \ + __atomic_post_full_fence(); \ +} + +static inline int +arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) +{ + int oldval = 0, ret = 0; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + switch (op) { + case FUTEX_OP_SET: + __futex_atomic_op("mov %[t], %[ov]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ADD: + __futex_atomic_op("add %[t], %[ov], %[op]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_OR: + __futex_atomic_op("or %[t], %[ov], %[op]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ANDN: + __futex_atomic_op("and %[t], %[ov], %[op]", + ret, oldval, uaddr, ~oparg); + break; + case FUTEX_OP_XOR: + __futex_atomic_op("xor %[t], %[ov], %[op]", + ret, oldval, uaddr, oparg); + break; + default: + ret = -ENOSYS; + } + + if (!ret) + *oval = oldval; + + return ret; +} + + + +static inline int +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) +{ + int ret = 0; + u32 val, tmp; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + __atomic_pre_full_fence(); + + __asm__ __volatile__ ( + "1: ldex.w %[v], %[u] \n" + " cmpne %[v], %[ov] \n" + " bt 4f \n" + " mov %[t], %[nv] \n" + "2: stex.w %[t], %[u] \n" + " bez %[t], 1b \n" + " br 4f \n" + "3: mov %[r], %[e] \n" + "4: \n" + " .section __ex_table,\"a\" \n" + " .balign 4 \n" + " .long 1b, 3b \n" + " .long 2b, 3b \n" + " .previous \n" + : [r] "+r" (ret), [v] "=&r" (val), [u] "+m" (*uaddr), + [t] "=&r" (tmp) + : [ov] "Jr" (oldval), [nv] "Jr" (newval), [e] "Jr" (-EFAULT) + : "memory"); + + __atomic_post_full_fence(); + + *uval = val; + return ret; +} + +#endif /* CONFIG_SMP */ +#endif /* __ASM_CSKY_FUTEX_H */ diff --git a/arch/csky/include/asm/highmem.h b/arch/csky/include/asm/highmem.h index 1f4ed3f4c0d9..1ed810effb3d 100644 --- a/arch/csky/include/asm/highmem.h +++ b/arch/csky/include/asm/highmem.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_HIGHMEM_H #define __ASM_CSKY_HIGHMEM_H diff --git a/arch/csky/include/asm/io.h b/arch/csky/include/asm/io.h index e909587f24c5..f82654053dc0 100644 --- a/arch/csky/include/asm/io.h +++ b/arch/csky/include/asm/io.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_IO_H #define __ASM_CSKY_IO_H diff --git a/arch/csky/include/asm/memory.h b/arch/csky/include/asm/memory.h index a65c6759f537..d12179801ae3 100644 --- a/arch/csky/include/asm/memory.h +++ b/arch/csky/include/asm/memory.h @@ -10,7 +10,7 @@ #define FIXADDR_TOP _AC(0xffffc000, UL) #define PKMAP_BASE _AC(0xff800000, UL) -#define VMALLOC_START _AC(0xc0008000, UL) +#define VMALLOC_START (PAGE_OFFSET + LOWMEM_LIMIT + (PAGE_SIZE * 8)) #define VMALLOC_END (PKMAP_BASE - (PAGE_SIZE * 2)) #ifdef CONFIG_HAVE_TCM diff --git a/arch/csky/include/asm/mmu.h b/arch/csky/include/asm/mmu.h index 26fbb1d15df0..d78321901d06 100644 --- a/arch/csky/include/asm/mmu.h +++ b/arch/csky/include/asm/mmu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_MMU_H #define __ASM_CSKY_MMU_H diff --git a/arch/csky/include/asm/mmu_context.h b/arch/csky/include/asm/mmu_context.h index b227d29393a8..95d99b30792c 100644 --- a/arch/csky/include/asm/mmu_context.h +++ b/arch/csky/include/asm/mmu_context.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_MMU_CONTEXT_H #define __ASM_CSKY_MMU_CONTEXT_H @@ -14,12 +13,6 @@ #include #include -#define TLBMISS_HANDLER_SETUP_PGD(pgd) \ - setup_pgd(__pa(pgd), false) - -#define TLBMISS_HANDLER_SETUP_PGD_KERNEL(pgd) \ - setup_pgd(__pa(pgd), true) - #define ASID_MASK ((1 << CONFIG_CPU_ASID_BITS) - 1) #define cpu_asid(mm) (atomic64_read(&mm->context.asid) & ASID_MASK) @@ -36,8 +29,7 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, if (prev != next) check_and_switch_context(next, cpu); - TLBMISS_HANDLER_SETUP_PGD(next->pgd); - write_mmu_entryhi(next->context.asid.counter); + setup_pgd(next->pgd, next->context.asid.counter); flush_icache_deferred(next); } diff --git a/arch/csky/include/asm/page.h b/arch/csky/include/asm/page.h index 9b98bf31d57c..3b91fc3cf36f 100644 --- a/arch/csky/include/asm/page.h +++ b/arch/csky/include/asm/page.h @@ -24,7 +24,7 @@ * address region. We use them mapping kernel 1GB direct-map address area and * for more than 1GB of memory we use highmem. */ -#define PAGE_OFFSET 0x80000000 +#define PAGE_OFFSET CONFIG_PAGE_OFFSET #define SSEG_SIZE 0x20000000 #define LOWMEM_LIMIT (SSEG_SIZE * 2) diff --git a/arch/csky/include/asm/perf_event.h b/arch/csky/include/asm/perf_event.h index 572093e11001..249905d8a4e8 100644 --- a/arch/csky/include/asm/perf_event.h +++ b/arch/csky/include/asm/perf_event.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PERF_EVENT_H #define __ASM_CSKY_PERF_EVENT_H diff --git a/arch/csky/include/asm/pgalloc.h b/arch/csky/include/asm/pgalloc.h index d58d8146b729..cd211aabbefd 100644 --- a/arch/csky/include/asm/pgalloc.h +++ b/arch/csky/include/asm/pgalloc.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGALLOC_H #define __ASM_CSKY_PGALLOC_H @@ -71,7 +70,7 @@ do { \ } while (0) extern void pagetable_init(void); -extern void pre_mmu_init(void); +extern void mmu_init(unsigned long min_pfn, unsigned long max_pfn); extern void pre_trap_init(void); #endif /* __ASM_CSKY_PGALLOC_H */ diff --git a/arch/csky/include/asm/pgtable.h b/arch/csky/include/asm/pgtable.h index 2002cb7f1053..0d60367b6bfa 100644 --- a/arch/csky/include/asm/pgtable.h +++ b/arch/csky/include/asm/pgtable.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGTABLE_H #define __ASM_CSKY_PGTABLE_H @@ -14,7 +13,7 @@ #define PGDIR_SIZE (1UL << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) -#define USER_PTRS_PER_PGD (0x80000000UL/PGDIR_SIZE) +#define USER_PTRS_PER_PGD (PAGE_OFFSET/PGDIR_SIZE) #define FIRST_USER_ADDRESS 0UL /* @@ -34,23 +33,13 @@ #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) #define pte_clear(mm, addr, ptep) set_pte((ptep), \ - (((unsigned int) addr & PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0))) + (((unsigned int) addr >= PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0))) #define pte_none(pte) (!(pte_val(pte) & ~_PAGE_GLOBAL)) #define pte_present(pte) (pte_val(pte) & _PAGE_PRESENT) #define pte_pfn(x) ((unsigned long)((x).pte_low >> PAGE_SHIFT)) #define pfn_pte(pfn, prot) __pte(((unsigned long long)(pfn) << PAGE_SHIFT) \ | pgprot_val(prot)) -#define __READABLE (_PAGE_READ | _PAGE_VALID | _PAGE_ACCESSED) -#define __WRITEABLE (_PAGE_WRITE | _PAGE_DIRTY | _PAGE_MODIFIED) - -#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_MODIFIED | \ - _CACHE_MASK) - -#define __swp_type(x) (((x).val >> 4) & 0xff) -#define __swp_offset(x) ((x).val >> 12) -#define __swp_entry(type, offset) ((swp_entry_t) {((type) << 4) | \ - ((offset) << 12) }) #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) @@ -59,41 +48,52 @@ pgprot_val(pgprot)) /* - * CSKY can't do page protection for execute, and considers that the same like - * read. Also, write permissions imply read permissions. This is the closest - * we can get by reasonable means.. + * C-SKY only has VALID and DIRTY bit in hardware. So we need to use the + * two bits emulate PRESENT, READ, WRITE, EXEC, MODIFIED, ACCESSED. */ -#define PAGE_NONE __pgprot(_PAGE_PRESENT | _CACHE_CACHED) -#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ +#define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED) + +#define PAGE_NONE __pgprot(_PAGE_PROT_NONE) +#define PAGE_READ __pgprot(_PAGE_BASE | _PAGE_READ | \ _CACHE_CACHED) -#define PAGE_COPY __pgprot(_PAGE_PRESENT | _PAGE_READ | _CACHE_CACHED) -#define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_READ | _CACHE_CACHED) -#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ - _PAGE_GLOBAL | _CACHE_CACHED) -#define PAGE_USERIO __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ +#define PAGE_WRITE __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_WRITE | \ + _CACHE_CACHED) +#define PAGE_SHARED PAGE_WRITE + +#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_VALID | \ + _PAGE_WRITE | _PAGE_DIRTY | _PAGE_MODIFIED | \ + _PAGE_GLOBAL | \ _CACHE_CACHED) -#define _PAGE_IOREMAP \ - (_PAGE_PRESENT | __READABLE | __WRITEABLE | _PAGE_GLOBAL | \ - _CACHE_UNCACHED | _PAGE_SO) +#define _PAGE_IOREMAP (_PAGE_BASE | _PAGE_READ | _PAGE_VALID | \ + _PAGE_WRITE | _PAGE_DIRTY | _PAGE_MODIFIED | \ + _PAGE_GLOBAL | \ + _CACHE_UNCACHED | _PAGE_SO) + +#define _PAGE_CHG_MASK (~(unsigned long) \ + (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ + _CACHE_MASK | _PAGE_GLOBAL)) + +#define MAX_SWAPFILES_CHECK() \ + BUILD_BUG_ON(MAX_SWAPFILES_SHIFT != 5) #define __P000 PAGE_NONE -#define __P001 PAGE_READONLY -#define __P010 PAGE_COPY -#define __P011 PAGE_COPY -#define __P100 PAGE_READONLY -#define __P101 PAGE_READONLY -#define __P110 PAGE_COPY -#define __P111 PAGE_COPY +#define __P001 PAGE_READ +#define __P010 PAGE_READ +#define __P011 PAGE_READ +#define __P100 PAGE_READ +#define __P101 PAGE_READ +#define __P110 PAGE_READ +#define __P111 PAGE_READ #define __S000 PAGE_NONE -#define __S001 PAGE_READONLY -#define __S010 PAGE_SHARED -#define __S011 PAGE_SHARED -#define __S100 PAGE_READONLY -#define __S101 PAGE_READONLY -#define __S110 PAGE_SHARED -#define __S111 PAGE_SHARED +#define __S001 PAGE_READ +#define __S010 PAGE_WRITE +#define __S011 PAGE_WRITE +#define __S100 PAGE_READ +#define __S101 PAGE_READ +#define __S110 PAGE_WRITE +#define __S111 PAGE_WRITE extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) diff --git a/arch/csky/include/asm/processor.h b/arch/csky/include/asm/processor.h index 4800f6563abb..9e933021fe8e 100644 --- a/arch/csky/include/asm/processor.h +++ b/arch/csky/include/asm/processor.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PROCESSOR_H #define __ASM_CSKY_PROCESSOR_H @@ -28,7 +27,7 @@ extern struct cpuinfo_csky cpu_data[]; * for a 64 bit kernel expandable to 8192EB, of which the current CSKY * implementations will "only" be able to use 1TB ... */ -#define TASK_SIZE 0x7fff8000UL +#define TASK_SIZE (PAGE_OFFSET - (PAGE_SIZE * 8)) #ifdef __KERNEL__ #define STACK_TOP TASK_SIZE diff --git a/arch/csky/include/asm/ptrace.h b/arch/csky/include/asm/ptrace.h index 91ceb1b454c9..4202aab6df42 100644 --- a/arch/csky/include/asm/ptrace.h +++ b/arch/csky/include/asm/ptrace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PTRACE_H #define __ASM_CSKY_PTRACE_H diff --git a/arch/csky/include/asm/segment.h b/arch/csky/include/asm/segment.h index 79ede9b1a646..589e8321dc14 100644 --- a/arch/csky/include/asm/segment.h +++ b/arch/csky/include/asm/segment.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SEGMENT_H #define __ASM_CSKY_SEGMENT_H @@ -10,7 +9,7 @@ typedef struct { #define KERNEL_DS ((mm_segment_t) { 0xFFFFFFFF }) -#define USER_DS ((mm_segment_t) { 0x80000000UL }) +#define USER_DS ((mm_segment_t) { PAGE_OFFSET }) #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) #define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) diff --git a/arch/csky/include/asm/shmparam.h b/arch/csky/include/asm/shmparam.h index efafe4c79fed..2fe6cea0dae9 100644 --- a/arch/csky/include/asm/shmparam.h +++ b/arch/csky/include/asm/shmparam.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SHMPARAM_H #define __ASM_CSKY_SHMPARAM_H diff --git a/arch/csky/include/asm/spinlock.h b/arch/csky/include/asm/spinlock.h index 7cf3f2b34cea..69f5aa249c5f 100644 --- a/arch/csky/include/asm/spinlock.h +++ b/arch/csky/include/asm/spinlock.h @@ -6,8 +6,6 @@ #include #include -#ifdef CONFIG_QUEUED_RWLOCKS - /* * Ticket-based spin-locking. */ @@ -88,169 +86,4 @@ static inline int arch_spin_is_contended(arch_spinlock_t *lock) #include -/* See include/linux/spinlock.h */ -#define smp_mb__after_spinlock() smp_mb() - -#else /* CONFIG_QUEUED_RWLOCKS */ - -/* - * Test-and-set spin-locking. - */ -static inline void arch_spin_lock(arch_spinlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 1b \n" - " movi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - smp_mb(); -} - -static inline void arch_spin_unlock(arch_spinlock_t *lock) -{ - smp_mb(); - WRITE_ONCE(lock->lock, 0); -} - -static inline int arch_spin_trylock(arch_spinlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 2f \n" - " movi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - " movi %0, 0 \n" - "2: \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - - if (!tmp) - smp_mb(); - - return !tmp; -} - -#define arch_spin_is_locked(x) (READ_ONCE((x)->lock) != 0) - -/* - * read lock/unlock/trylock - */ -static inline void arch_read_lock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " blz %0, 1b \n" - " addi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - smp_mb(); -} - -static inline void arch_read_unlock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - smp_mb(); - asm volatile ( - "1: ldex.w %0, (%1) \n" - " subi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); -} - -static inline int arch_read_trylock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " blz %0, 2f \n" - " addi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - " movi %0, 0 \n" - "2: \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - - if (!tmp) - smp_mb(); - - return !tmp; -} - -/* - * write lock/unlock/trylock - */ -static inline void arch_write_lock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 1b \n" - " subi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - smp_mb(); -} - -static inline void arch_write_unlock(arch_rwlock_t *lock) -{ - smp_mb(); - WRITE_ONCE(lock->lock, 0); -} - -static inline int arch_write_trylock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 2f \n" - " subi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - " movi %0, 0 \n" - "2: \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - - if (!tmp) - smp_mb(); - - return !tmp; -} - -#endif /* CONFIG_QUEUED_RWLOCKS */ #endif /* __ASM_CSKY_SPINLOCK_H */ diff --git a/arch/csky/include/asm/spinlock_types.h b/arch/csky/include/asm/spinlock_types.h index 88b82438b182..8ff0f6ff3a00 100644 --- a/arch/csky/include/asm/spinlock_types.h +++ b/arch/csky/include/asm/spinlock_types.h @@ -22,16 +22,6 @@ typedef struct { #define __ARCH_SPIN_LOCK_UNLOCKED { { 0 } } -#ifdef CONFIG_QUEUED_RWLOCKS #include -#else /* CONFIG_NR_CPUS > 2 */ - -typedef struct { - u32 lock; -} arch_rwlock_t; - -#define __ARCH_RW_LOCK_UNLOCKED { 0 } - -#endif /* CONFIG_QUEUED_RWLOCKS */ #endif /* __ASM_CSKY_SPINLOCK_TYPES_H */ diff --git a/arch/csky/include/asm/string.h b/arch/csky/include/asm/string.h index 73142de18355..a0d81e9d6b8f 100644 --- a/arch/csky/include/asm/string.h +++ b/arch/csky/include/asm/string.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _CSKY_STRING_MM_H_ #define _CSKY_STRING_MM_H_ diff --git a/arch/csky/include/asm/switch_to.h b/arch/csky/include/asm/switch_to.h index 35a39e88933d..731e466415e2 100644 --- a/arch/csky/include/asm/switch_to.h +++ b/arch/csky/include/asm/switch_to.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SWITCH_TO_H #define __ASM_CSKY_SWITCH_TO_H diff --git a/arch/csky/include/asm/syscalls.h b/arch/csky/include/asm/syscalls.h index 5d48e5e0082e..ea9ce6138b9b 100644 --- a/arch/csky/include/asm/syscalls.h +++ b/arch/csky/include/asm/syscalls.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SYSCALLS_H #define __ASM_CSKY_SYSCALLS_H diff --git a/arch/csky/include/asm/thread_info.h b/arch/csky/include/asm/thread_info.h index 21456a3737c2..8c349a8f904d 100644 --- a/arch/csky/include/asm/thread_info.h +++ b/arch/csky/include/asm/thread_info.h @@ -1,12 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _ASM_CSKY_THREAD_INFO_H #define _ASM_CSKY_THREAD_INFO_H #ifndef __ASSEMBLY__ -#include #include #include #include diff --git a/arch/csky/include/asm/tlb.h b/arch/csky/include/asm/tlb.h index fdff9b8d70c8..3498e65f59f8 100644 --- a/arch/csky/include/asm/tlb.h +++ b/arch/csky/include/asm/tlb.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_TLB_H #define __ASM_CSKY_TLB_H diff --git a/arch/csky/include/asm/tlbflush.h b/arch/csky/include/asm/tlbflush.h index 6845b0667703..407160b4fde7 100644 --- a/arch/csky/include/asm/tlbflush.h +++ b/arch/csky/include/asm/tlbflush.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_TLBFLUSH_H #define __ASM_TLBFLUSH_H diff --git a/arch/csky/include/asm/traps.h b/arch/csky/include/asm/traps.h index 1c081805b962..421a4195e2fe 100644 --- a/arch/csky/include/asm/traps.h +++ b/arch/csky/include/asm/traps.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_TRAPS_H #define __ASM_CSKY_TRAPS_H diff --git a/arch/csky/include/asm/uaccess.h b/arch/csky/include/asm/uaccess.h index 1633ffe5ae15..3dec272e1fa3 100644 --- a/arch/csky/include/asm/uaccess.h +++ b/arch/csky/include/asm/uaccess.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_UACCESS_H #define __ASM_CSKY_UACCESS_H diff --git a/arch/csky/include/asm/unistd.h b/arch/csky/include/asm/unistd.h index da7a18295615..9cf97de9a26d 100644 --- a/arch/csky/include/asm/unistd.h +++ b/arch/csky/include/asm/unistd.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #include diff --git a/arch/csky/include/asm/vdso.h b/arch/csky/include/asm/vdso.h index d963d691f3a1..eb5142f9c564 100644 --- a/arch/csky/include/asm/vdso.h +++ b/arch/csky/include/asm/vdso.h @@ -3,10 +3,25 @@ #ifndef __ASM_CSKY_VDSO_H #define __ASM_CSKY_VDSO_H -#include +#include -struct csky_vdso { - unsigned short rt_signal_retcode[4]; +#ifndef GENERIC_TIME_VSYSCALL +struct vdso_data { }; +#endif + +/* + * The VDSO symbols are mapped into Linux so we can just use regular symbol + * addressing to get their offsets in userspace. The symbols are mapped at an + * offset of 0, but since the linker must support setting weak undefined + * symbols to the absolute address 0 it also happens to support other low + * addresses even when the code model suggests those low addresses would not + * otherwise be availiable. + */ +#define VDSO_SYMBOL(base, name) \ +({ \ + extern const char __vdso_##name[]; \ + (void __user *)((unsigned long)(base) + __vdso_##name); \ +}) #endif /* __ASM_CSKY_VDSO_H */ diff --git a/arch/csky/include/asm/vdso/clocksource.h b/arch/csky/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..dfca7b4724b7 --- /dev/null +++ b/arch/csky/include/asm/vdso/clocksource.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_CLOCKSOURCE_H +#define __ASM_VDSO_CSKY_CLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif /* __ASM_VDSO_CSKY_CLOCKSOURCE_H */ diff --git a/arch/csky/include/asm/vdso/gettimeofday.h b/arch/csky/include/asm/vdso/gettimeofday.h new file mode 100644 index 000000000000..6c4f1446944f --- /dev/null +++ b/arch/csky/include/asm/vdso/gettimeofday.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_GETTIMEOFDAY_H +#define __ASM_VDSO_CSKY_GETTIMEOFDAY_H + +#ifndef __ASSEMBLY__ + +#include +#include +#include +#include + +#define VDSO_HAS_CLOCK_GETRES 1 + +static __always_inline +int gettimeofday_fallback(struct __kernel_old_timeval *_tv, + struct timezone *_tz) +{ + register struct __kernel_old_timeval *tv asm("a0") = _tv; + register struct timezone *tz asm("a1") = _tz; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_gettimeofday; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(tv), "r"(tz), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +long clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_gettime64; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +long clock_gettime32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct old_timespec32 *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_gettime; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_getres_time64; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct old_timespec32 *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_getres; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +uint64_t csky_pmu_read_cc(void); +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, + const struct vdso_data *vd) +{ +#ifdef CONFIG_CSKY_PMU_V1 + return csky_pmu_read_cc(); +#else + return 0; +#endif +} + +static __always_inline const struct vdso_data *__arch_get_vdso_data(void) +{ + return _vdso_data; +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CSKY_GETTIMEOFDAY_H */ diff --git a/arch/csky/include/asm/vdso/processor.h b/arch/csky/include/asm/vdso/processor.h new file mode 100644 index 000000000000..39a6b561d0cc --- /dev/null +++ b/arch/csky/include/asm/vdso/processor.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_VDSO_CSKY_PROCESSOR_H +#define __ASM_VDSO_CSKY_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +#define cpu_relax() barrier() + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CSKY_PROCESSOR_H */ diff --git a/arch/csky/include/asm/vdso/vsyscall.h b/arch/csky/include/asm/vdso/vsyscall.h new file mode 100644 index 000000000000..c276211a7c4d --- /dev/null +++ b/arch/csky/include/asm/vdso/vsyscall.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_VSYSCALL_H +#define __ASM_VDSO_CSKY_VSYSCALL_H + +#ifndef __ASSEMBLY__ + +#include + +extern struct vdso_data *vdso_data; + +static __always_inline struct vdso_data *__csky_get_k_vdso_data(void) +{ + return vdso_data; +} +#define __arch_get_k_vdso_data __csky_get_k_vdso_data + +#include + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CSKY_VSYSCALL_H */ diff --git a/arch/csky/include/uapi/asm/byteorder.h b/arch/csky/include/uapi/asm/byteorder.h index d150cd664873..1aedd513b65a 100644 --- a/arch/csky/include/uapi/asm/byteorder.h +++ b/arch/csky/include/uapi/asm/byteorder.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BYTEORDER_H #define __ASM_CSKY_BYTEORDER_H diff --git a/arch/csky/include/uapi/asm/perf_regs.h b/arch/csky/include/uapi/asm/perf_regs.h index 49d4e147a559..d0a8ac6a1b77 100644 --- a/arch/csky/include/uapi/asm/perf_regs.h +++ b/arch/csky/include/uapi/asm/perf_regs.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _ASM_CSKY_PERF_REGS_H #define _ASM_CSKY_PERF_REGS_H diff --git a/arch/csky/include/uapi/asm/ptrace.h b/arch/csky/include/uapi/asm/ptrace.h index 66b2268e324e..3be9c14334a6 100644 --- a/arch/csky/include/uapi/asm/ptrace.h +++ b/arch/csky/include/uapi/asm/ptrace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _CSKY_PTRACE_H #define _CSKY_PTRACE_H diff --git a/arch/csky/include/uapi/asm/sigcontext.h b/arch/csky/include/uapi/asm/sigcontext.h index 670c020f2cb8..859afb602477 100644 --- a/arch/csky/include/uapi/asm/sigcontext.h +++ b/arch/csky/include/uapi/asm/sigcontext.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SIGCONTEXT_H #define __ASM_CSKY_SIGCONTEXT_H diff --git a/arch/csky/include/uapi/asm/unistd.h b/arch/csky/include/uapi/asm/unistd.h index ba4018929733..7ff6a2466af1 100644 --- a/arch/csky/include/uapi/asm/unistd.h +++ b/arch/csky/include/uapi/asm/unistd.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #define __ARCH_WANT_STAT64 #define __ARCH_WANT_NEW_STAT diff --git a/arch/csky/kernel/Makefile b/arch/csky/kernel/Makefile index 37f37c0e934a..6c0f36010ed0 100644 --- a/arch/csky/kernel/Makefile +++ b/arch/csky/kernel/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only extra-y := head.o vmlinux.lds -obj-y += entry.o atomic.o signal.o traps.o irq.o time.o vdso.o +obj-y += entry.o atomic.o signal.o traps.o irq.o time.o vdso.o vdso/ obj-y += power.o syscall.o syscall_table.o setup.o obj-y += process.o cpu-probe.o ptrace.o stacktrace.o obj-y += probes/ diff --git a/arch/csky/kernel/atomic.S b/arch/csky/kernel/atomic.S index 3821ef9b7567..e73e548f7855 100644 --- a/arch/csky/kernel/atomic.S +++ b/arch/csky/kernel/atomic.S @@ -14,6 +14,10 @@ */ ENTRY(csky_cmpxchg) USPTOKSP + + RD_MEH a3 + WR_MEH a3 + mfcr a3, epc addi a3, TRAP0_SIZE @@ -36,11 +40,11 @@ ENTRY(csky_cmpxchg) 2: sync.is #else -1: +GLOBAL(csky_cmpxchg_ldw) ldw a3, (a2) cmpne a0, a3 bt16 3f -2: +GLOBAL(csky_cmpxchg_stw) stw a1, (a2) 3: #endif @@ -55,19 +59,3 @@ ENTRY(csky_cmpxchg) KSPTOUSP rte END(csky_cmpxchg) - -#ifndef CONFIG_CPU_HAS_LDSTEX -/* - * Called from tlbmodified exception - */ -ENTRY(csky_cmpxchg_fixup) - mfcr a0, epc - lrw a1, 2b - cmpne a1, a0 - bt 1f - subi a1, (2b - 1b) - stw a1, (sp, LSAVE_PC) -1: - rts -END(csky_cmpxchg_fixup) -#endif diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index 5a5cabd076e1..c1bd7a6b4ab6 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -13,10 +13,6 @@ #include #include -#define PTE_INDX_MSK 0xffc -#define PTE_INDX_SHIFT 10 -#define _PGDIR_SHIFT 22 - .macro zero_fp #ifdef CONFIG_STACKTRACE movi r8, 0 @@ -41,108 +37,15 @@ #endif .endm -.macro tlbop_begin name, val0, val1, val2 -ENTRY(csky_\name) - mtcr a3, ss2 - mtcr r6, ss3 - mtcr a2, ss4 - - RD_PGDR r6 - RD_MEH a3 -#ifdef CONFIG_CPU_HAS_TLBI - tlbi.vaas a3 - sync.is - - btsti a3, 31 - bf 1f - RD_PGDR_K r6 -1: -#else - bgeni a2, 31 - WR_MCIR a2 - bgeni a2, 25 - WR_MCIR a2 -#endif - bclri r6, 0 - lrw a2, va_pa_offset - ld.w a2, (a2, 0) - subu r6, a2 - bseti r6, 31 - - mov a2, a3 - lsri a2, _PGDIR_SHIFT - lsli a2, 2 - addu r6, a2 - ldw r6, (r6) - - lrw a2, va_pa_offset - ld.w a2, (a2, 0) - subu r6, a2 - bseti r6, 31 - - lsri a3, PTE_INDX_SHIFT - lrw a2, PTE_INDX_MSK - and a3, a2 - addu r6, a3 - ldw a3, (r6) - - movi a2, (_PAGE_PRESENT | \val0) - and a3, a2 - cmpne a3, a2 - bt \name - - /* First read/write the page, just update the flags */ - ldw a3, (r6) - bgeni a2, PAGE_VALID_BIT - bseti a2, PAGE_ACCESSED_BIT - bseti a2, \val1 - bseti a2, \val2 - or a3, a2 - stw a3, (r6) - - /* Some cpu tlb-hardrefill bypass the cache */ -#ifdef CONFIG_CPU_NEED_TLBSYNC - movi a2, 0x22 - bseti a2, 6 - mtcr r6, cr22 - mtcr a2, cr17 - sync -#endif - - mfcr a3, ss2 - mfcr r6, ss3 - mfcr a2, ss4 - rte -\name: - mfcr a3, ss2 - mfcr r6, ss3 - mfcr a2, ss4 +.text +ENTRY(csky_pagefault) SAVE_ALL 0 -.endm -.macro tlbop_end is_write zero_fp context_tracking - RD_MEH a2 - psrset ee, ie + psrset ee mov a0, sp - movi a1, \is_write jbsr do_page_fault jmpi ret_from_exception -.endm - -.text - -tlbop_begin tlbinvalidl, _PAGE_READ, PAGE_VALID_BIT, PAGE_ACCESSED_BIT -tlbop_end 0 - -tlbop_begin tlbinvalids, _PAGE_WRITE, PAGE_DIRTY_BIT, PAGE_MODIFIED_BIT -tlbop_end 1 - -tlbop_begin tlbmodified, _PAGE_WRITE, PAGE_DIRTY_BIT, PAGE_MODIFIED_BIT -#ifndef CONFIG_CPU_HAS_LDSTEX -jbsr csky_cmpxchg_fixup -#endif -tlbop_end 1 ENTRY(csky_systemcall) SAVE_ALL TRAP0_SIZE @@ -314,6 +217,9 @@ ENTRY(csky_trap) ENTRY(csky_get_tls) USPTOKSP + RD_MEH a0 + WR_MEH a0 + /* increase epc for continue */ mfcr a0, epc addi a0, TRAP0_SIZE diff --git a/arch/csky/kernel/head.S b/arch/csky/kernel/head.S index 17ed9d250480..7e3e4f15b052 100644 --- a/arch/csky/kernel/head.S +++ b/arch/csky/kernel/head.S @@ -21,10 +21,16 @@ END(_start) ENTRY(_start_smp_secondary) SETUP_MMU - /* copy msa1 from CPU0 */ - lrw r6, secondary_msa1 +#ifdef CONFIG_PAGE_OFFSET_80000000 + lrw r6, secondary_msa1 ld.w r6, (r6, 0) mtcr r6, cr<31, 15> +#endif + + lrw r6, secondary_pgd + ld.w r6, (r6, 0) + mtcr r6, cr<28, 15> + mtcr r6, cr<29, 15> /* set stack point */ lrw r6, secondary_stack diff --git a/arch/csky/kernel/perf_event.c b/arch/csky/kernel/perf_event.c index 1a29f1157449..e5f18420ce64 100644 --- a/arch/csky/kernel/perf_event.c +++ b/arch/csky/kernel/perf_event.c @@ -87,7 +87,7 @@ static int csky_pmu_irq; }) /* cycle counter */ -static uint64_t csky_pmu_read_cc(void) +uint64_t csky_pmu_read_cc(void) { uint32_t lo, hi, tmp; uint64_t result; @@ -1319,7 +1319,7 @@ int csky_pmu_device_probe(struct platform_device *pdev, pr_notice("[perf] PMU request irq fail!\n"); } - ret = cpuhp_setup_state(CPUHP_AP_PERF_ONLINE, "AP_PERF_ONLINE", + ret = cpuhp_setup_state(CPUHP_AP_PERF_CSKY_ONLINE, "AP_PERF_ONLINE", csky_pmu_starting_cpu, csky_pmu_dying_cpu); if (ret) { diff --git a/arch/csky/kernel/probes/simulate-insn.c b/arch/csky/kernel/probes/simulate-insn.c index 4e464fed52ec..d6e8d092c9b7 100644 --- a/arch/csky/kernel/probes/simulate-insn.c +++ b/arch/csky/kernel/probes/simulate-insn.c @@ -274,9 +274,9 @@ void __kprobes simulate_bnezad32(u32 opcode, long addr, struct pt_regs *regs) { unsigned long tmp = opcode & 0x1f; - unsigned long val; + long val; - csky_insn_reg_get_val(regs, tmp, &val); + csky_insn_reg_get_val(regs, tmp, (unsigned long *)&val); val -= 1; @@ -286,7 +286,7 @@ simulate_bnezad32(u32 opcode, long addr, struct pt_regs *regs) } else instruction_pointer_set(regs, addr + 4); - csky_insn_reg_set_val(regs, tmp, val); + csky_insn_reg_set_val(regs, tmp, (unsigned long)val); } void __kprobes @@ -297,13 +297,11 @@ simulate_bhsz32(u32 opcode, long addr, struct pt_regs *regs) csky_insn_reg_get_val(regs, tmp, &val); - if (val >= 0) { + if ((long) val >= 0) { instruction_pointer_set(regs, addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); } else instruction_pointer_set(regs, addr + 4); - - csky_insn_reg_set_val(regs, tmp, val); } void __kprobes @@ -314,13 +312,11 @@ simulate_bhz32(u32 opcode, long addr, struct pt_regs *regs) csky_insn_reg_get_val(regs, tmp, &val); - if (val > 0) { + if ((long) val > 0) { instruction_pointer_set(regs, addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); } else instruction_pointer_set(regs, addr + 4); - - csky_insn_reg_set_val(regs, tmp, val); } void __kprobes @@ -331,13 +327,11 @@ simulate_blsz32(u32 opcode, long addr, struct pt_regs *regs) csky_insn_reg_get_val(regs, tmp, &val); - if (val <= 0) { + if ((long) val <= 0) { instruction_pointer_set(regs, addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); } else instruction_pointer_set(regs, addr + 4); - - csky_insn_reg_set_val(regs, tmp, val); } void __kprobes @@ -348,13 +342,11 @@ simulate_blz32(u32 opcode, long addr, struct pt_regs *regs) csky_insn_reg_get_val(regs, tmp, &val); - if (val < 0) { + if ((long) val < 0) { instruction_pointer_set(regs, addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); } else instruction_pointer_set(regs, addr + 4); - - csky_insn_reg_set_val(regs, tmp, val); } void __kprobes diff --git a/arch/csky/kernel/process.c b/arch/csky/kernel/process.c index 69af6bc87e64..3d0ca22cd0e2 100644 --- a/arch/csky/kernel/process.c +++ b/arch/csky/kernel/process.c @@ -49,7 +49,7 @@ int copy_thread(unsigned long clone_flags, /* setup thread.sp for switch_to !!! */ p->thread.sp = (unsigned long)childstack; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); childstack->r15 = (unsigned long) ret_from_kernel_thread; childstack->r10 = kthread_arg; diff --git a/arch/csky/kernel/ptrace.c b/arch/csky/kernel/ptrace.c index d822144906ac..0105ac81b432 100644 --- a/arch/csky/kernel/ptrace.c +++ b/arch/csky/kernel/ptrace.c @@ -22,6 +22,7 @@ #include #include +#include #define CREATE_TRACE_POINTS #include @@ -83,7 +84,7 @@ static int gpr_get(struct task_struct *target, /* Abiv1 regs->tls is fake and we need sync here. */ regs->tls = task_thread_info(target)->tp_value; - return membuf_write(&to, regs, sizeof(regs)); + return membuf_write(&to, regs, sizeof(*regs)); } static int gpr_set(struct task_struct *target, @@ -343,6 +344,124 @@ asmlinkage void syscall_trace_exit(struct pt_regs *regs) trace_sys_exit(regs, syscall_get_return_value(current, regs)); } +#ifdef CONFIG_CPU_CK860 +static void show_iutlb(void) +{ + int entry, i; + unsigned long flags; + unsigned long oldpid; + unsigned long entryhi[16], entrylo0[16], entrylo1[16]; + + oldpid = read_mmu_entryhi(); + + entry = 0x8000; + + local_irq_save(flags); + + for (i = 0; i < 16; i++) { + write_mmu_index(entry); + tlb_read(); + entryhi[i] = read_mmu_entryhi(); + entrylo0[i] = read_mmu_entrylo0(); + entrylo1[i] = read_mmu_entrylo1(); + + entry++; + } + + local_irq_restore(flags); + + write_mmu_entryhi(oldpid); + + printk("\n\n\n"); + for (i = 0; i < 16; i++) + printk("iutlb[%d]: entryhi - 0x%lx; entrylo0 - 0x%lx;" + " entrylo1 - 0x%lx\n", + i, entryhi[i], entrylo0[i], entrylo1[i]); + printk("\n\n\n"); +} + +static void show_dutlb(void) +{ + int entry, i; + unsigned long flags; + unsigned long oldpid; + unsigned long entryhi[16], entrylo0[16], entrylo1[16]; + + oldpid = read_mmu_entryhi(); + + entry = 0x4000; + + local_irq_save(flags); + + for (i = 0; i < 16; i++) { + write_mmu_index(entry); + tlb_read(); + entryhi[i] = read_mmu_entryhi(); + entrylo0[i] = read_mmu_entrylo0(); + entrylo1[i] = read_mmu_entrylo1(); + + entry++; + } + + local_irq_restore(flags); + + write_mmu_entryhi(oldpid); + + printk("\n\n\n"); + for (i = 0; i < 16; i++) + printk("dutlb[%d]: entryhi - 0x%lx; entrylo0 - 0x%lx;" + " entrylo1 - 0x%lx\n", + i, entryhi[i], entrylo0[i], entrylo1[i]); + printk("\n\n\n"); +} + +static unsigned long entryhi[1024], entrylo0[1024], entrylo1[1024]; +static void show_jtlb(void) +{ + int entry; + unsigned long flags; + unsigned long oldpid; + + oldpid = read_mmu_entryhi(); + + entry = 0; + + local_irq_save(flags); + while (entry < 1024) { + write_mmu_index(entry); + tlb_read(); + entryhi[entry] = read_mmu_entryhi(); + entrylo0[entry] = read_mmu_entrylo0(); + entrylo1[entry] = read_mmu_entrylo1(); + + entry++; + } + local_irq_restore(flags); + + write_mmu_entryhi(oldpid); + + printk("\n\n\n"); + + for (entry = 0; entry < 1024; entry++) + printk("jtlb[%x]: entryhi - 0x%lx; entrylo0 - 0x%lx;" + " entrylo1 - 0x%lx\n", + entry, entryhi[entry], entrylo0[entry], entrylo1[entry]); + printk("\n\n\n"); +} + +static void show_tlb(void) +{ + show_iutlb(); + show_dutlb(); + show_jtlb(); +} +#else +static void show_tlb(void) +{ + return; +} +#endif + void show_regs(struct pt_regs *fp) { pr_info("\nCURRENT PROCESS:\n\n"); @@ -363,9 +482,10 @@ void show_regs(struct pt_regs *fp) pr_info("PC: 0x%08lx (%pS)\n", (long)fp->pc, (void *)fp->pc); pr_info("LR: 0x%08lx (%pS)\n", (long)fp->lr, (void *)fp->lr); - pr_info("SP: 0x%08lx\n", (long)fp); - pr_info("orig_a0: 0x%08lx\n", fp->orig_a0); + pr_info("SP: 0x%08lx\n", (long)fp->usp); pr_info("PSR: 0x%08lx\n", (long)fp->sr); + pr_info("orig_a0: 0x%08lx\n", fp->orig_a0); + pr_info("PT_REGS: 0x%08lx\n", (long)fp); pr_info(" a0: 0x%08lx a1: 0x%08lx a2: 0x%08lx a3: 0x%08lx\n", fp->a0, fp->a1, fp->a2, fp->a3); @@ -395,5 +515,7 @@ void show_regs(struct pt_regs *fp) fp->regs[8], fp->regs[9]); #endif + show_tlb(); + return; } diff --git a/arch/csky/kernel/setup.c b/arch/csky/kernel/setup.c index e4cab16056d6..e93bc6f74432 100644 --- a/arch/csky/kernel/setup.c +++ b/arch/csky/kernel/setup.c @@ -45,13 +45,17 @@ static void __init csky_memblock_init(void) if (size >= lowmem_size) { max_low_pfn = min_low_pfn + lowmem_size; +#ifdef CONFIG_PAGE_OFFSET_80000000 write_mmu_msa1(read_mmu_msa0() + SSEG_SIZE); +#endif } else if (size > sseg_size) { max_low_pfn = min_low_pfn + sseg_size; } max_zone_pfn[ZONE_NORMAL] = max_low_pfn; + mmu_init(min_low_pfn, max_low_pfn); + #ifdef CONFIG_HIGHMEM max_zone_pfn[ZONE_HIGHMEM] = max_pfn; @@ -101,16 +105,26 @@ void __init setup_arch(char **cmdline_p) unsigned long va_pa_offset; EXPORT_SYMBOL(va_pa_offset); +static inline unsigned long read_mmu_msa(void) +{ +#ifdef CONFIG_PAGE_OFFSET_80000000 + return read_mmu_msa0(); +#endif + +#ifdef CONFIG_PAGE_OFFSET_A0000000 + return read_mmu_msa1(); +#endif +} + asmlinkage __visible void __init csky_start(unsigned int unused, void *dtb_start) { /* Clean up bss section */ memset(__bss_start, 0, __bss_stop - __bss_start); - va_pa_offset = read_mmu_msa0() & ~(SSEG_SIZE - 1); + va_pa_offset = read_mmu_msa() & ~(SSEG_SIZE - 1); pre_trap_init(); - pre_mmu_init(); if (dtb_start == NULL) early_init_dt_scan(__dtb_start); diff --git a/arch/csky/kernel/signal.c b/arch/csky/kernel/signal.c index 37ea64ed3c12..312f046d452d 100644 --- a/arch/csky/kernel/signal.c +++ b/arch/csky/kernel/signal.c @@ -134,7 +134,6 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { struct rt_sigframe *frame; int err = 0; - struct csky_vdso *vdso = current->mm->context.vdso; frame = get_sigframe(ksig, regs, sizeof(*frame)); if (!access_ok(frame, sizeof(*frame))) @@ -152,7 +151,8 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) return -EFAULT; /* Set up to return from userspace. */ - regs->lr = (unsigned long)(vdso->rt_signal_retcode); + regs->lr = (unsigned long)VDSO_SYMBOL( + current->mm->context.vdso, rt_sigreturn); /* * Set up registers for signal handler. diff --git a/arch/csky/kernel/smp.c b/arch/csky/kernel/smp.c index 041d0de6a1b6..0f9f5eef9338 100644 --- a/arch/csky/kernel/smp.c +++ b/arch/csky/kernel/smp.c @@ -203,8 +203,8 @@ volatile unsigned int secondary_hint; volatile unsigned int secondary_hint2; volatile unsigned int secondary_ccr; volatile unsigned int secondary_stack; - -unsigned long secondary_msa1; +volatile unsigned int secondary_msa1; +volatile unsigned int secondary_pgd; int __cpu_up(unsigned int cpu, struct task_struct *tidle) { @@ -216,6 +216,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) secondary_hint2 = mfcr("cr<21, 1>"); secondary_ccr = mfcr("cr18"); secondary_msa1 = read_mmu_msa1(); + secondary_pgd = mfcr("cr<29, 15>"); /* * Because other CPUs are in reset status, we must flush data @@ -262,8 +263,6 @@ void csky_start_secondary(void) flush_tlb_all(); write_mmu_pagemask(0); - TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); - TLBMISS_HANDLER_SETUP_PGD_KERNEL(swapper_pg_dir); #ifdef CONFIG_CPU_HAS_FPU init_fpu(); diff --git a/arch/csky/kernel/traps.c b/arch/csky/kernel/traps.c index 959a917c989d..e5fbf8653a21 100644 --- a/arch/csky/kernel/traps.c +++ b/arch/csky/kernel/traps.c @@ -39,9 +39,7 @@ asmlinkage void csky_cmpxchg(void); asmlinkage void csky_get_tls(void); asmlinkage void csky_irq(void); -asmlinkage void csky_tlbinvalidl(void); -asmlinkage void csky_tlbinvalids(void); -asmlinkage void csky_tlbmodified(void); +asmlinkage void csky_pagefault(void); /* Defined in head.S */ asmlinkage void _start_smp_secondary(void); @@ -66,9 +64,9 @@ void __init trap_init(void) VEC_INIT(VEC_TRAP3, csky_get_tls); /* setup MMU TLB exception */ - VEC_INIT(VEC_TLBINVALIDL, csky_tlbinvalidl); - VEC_INIT(VEC_TLBINVALIDS, csky_tlbinvalids); - VEC_INIT(VEC_TLBMODIFIED, csky_tlbmodified); + VEC_INIT(VEC_TLBINVALIDL, csky_pagefault); + VEC_INIT(VEC_TLBINVALIDS, csky_pagefault); + VEC_INIT(VEC_TLBMODIFIED, csky_pagefault); #ifdef CONFIG_CPU_HAS_FPU init_fpu(); diff --git a/arch/csky/kernel/vdso.c b/arch/csky/kernel/vdso.c index abc3dbc658d4..16c20d64d165 100644 --- a/arch/csky/kernel/vdso.c +++ b/arch/csky/kernel/vdso.c @@ -1,86 +1,107 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. -#include -#include -#include -#include -#include #include #include -#include -#include -#include +#include +#include +#include +#include +#ifdef GENERIC_TIME_VSYSCALL +#include +#else #include -#include +#endif -static struct page *vdso_page; +extern char vdso_start[], vdso_end[]; -static int __init init_vdso(void) +static unsigned int vdso_pages; +static struct page **vdso_pagelist; + +/* + * The vDSO data page. + */ +static union { + struct vdso_data data; + u8 page[PAGE_SIZE]; +} vdso_data_store __page_aligned_data; +struct vdso_data *vdso_data = &vdso_data_store.data; + +static int __init vdso_init(void) { - struct csky_vdso *vdso; - int err = 0; + unsigned int i; - vdso_page = alloc_page(GFP_KERNEL); - if (!vdso_page) - panic("Cannot allocate vdso"); + vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; + vdso_pagelist = + kcalloc(vdso_pages + 1, sizeof(struct page *), GFP_KERNEL); + if (unlikely(vdso_pagelist == NULL)) { + pr_err("vdso: pagelist allocation failed\n"); + return -ENOMEM; + } - vdso = vmap(&vdso_page, 1, 0, PAGE_KERNEL); - if (!vdso) - panic("Cannot map vdso"); + for (i = 0; i < vdso_pages; i++) { + struct page *pg; - clear_page(vdso); - - err = setup_vdso_page(vdso->rt_signal_retcode); - if (err) - panic("Cannot set signal return code, err: %x.", err); - - dcache_wb_range((unsigned long)vdso, (unsigned long)vdso + 16); - - vunmap(vdso); + pg = virt_to_page(vdso_start + (i << PAGE_SHIFT)); + vdso_pagelist[i] = pg; + } + vdso_pagelist[i] = virt_to_page(vdso_data); return 0; } -subsys_initcall(init_vdso); +arch_initcall(vdso_init); -int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) +int arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp) { - int ret; - unsigned long addr; struct mm_struct *mm = current->mm; + unsigned long vdso_base, vdso_len; + int ret; + + vdso_len = (vdso_pages + 1) << PAGE_SHIFT; mmap_write_lock(mm); - - addr = get_unmapped_area(NULL, STACK_TOP, PAGE_SIZE, 0, 0); - if (IS_ERR_VALUE(addr)) { - ret = addr; - goto up_fail; + vdso_base = get_unmapped_area(NULL, 0, vdso_len, 0, 0); + if (IS_ERR_VALUE(vdso_base)) { + ret = vdso_base; + goto end; } - ret = install_special_mapping( - mm, - addr, - PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, - &vdso_page); - if (ret) - goto up_fail; + /* + * Put vDSO base into mm struct. We need to do this before calling + * install_special_mapping or the perf counter mmap tracking code + * will fail to recognise it as a vDSO (since arch_vma_name fails). + */ + mm->context.vdso = (void *)vdso_base; - mm->context.vdso = (void *)addr; + ret = + install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT, + (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), + vdso_pagelist); -up_fail: + if (unlikely(ret)) { + mm->context.vdso = NULL; + goto end; + } + + vdso_base += (vdso_pages << PAGE_SHIFT); + ret = install_special_mapping(mm, vdso_base, PAGE_SIZE, + (VM_READ | VM_MAYREAD), &vdso_pagelist[vdso_pages]); + + if (unlikely(ret)) + mm->context.vdso = NULL; +end: mmap_write_unlock(mm); return ret; } const char *arch_vma_name(struct vm_area_struct *vma) { - if (vma->vm_mm == NULL) - return NULL; - - if (vma->vm_start == (long)vma->vm_mm->context.vdso) + if (vma->vm_mm && (vma->vm_start == (long)vma->vm_mm->context.vdso)) return "[vdso]"; - else - return NULL; + if (vma->vm_mm && (vma->vm_start == + (long)vma->vm_mm->context.vdso + PAGE_SIZE)) + return "[vdso_data]"; + return NULL; } diff --git a/arch/x86/platform/sfi/Makefile b/arch/csky/kernel/vdso/.gitignore similarity index 58% rename from arch/x86/platform/sfi/Makefile rename to arch/csky/kernel/vdso/.gitignore index 4eba24c2af67..3a19def868ec 100644 --- a/arch/x86/platform/sfi/Makefile +++ b/arch/csky/kernel/vdso/.gitignore @@ -1,2 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_SFI) += sfi.o +vdso.lds +*.tmp +vdso-syms.S diff --git a/arch/csky/kernel/vdso/Makefile b/arch/csky/kernel/vdso/Makefile new file mode 100644 index 000000000000..0b6909f10667 --- /dev/null +++ b/arch/csky/kernel/vdso/Makefile @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: GPL-2.0-only + +# Absolute relocation type $(ARCH_REL_TYPE_ABS) needs to be defined before +# the inclusion of generic Makefile. +ARCH_REL_TYPE_ABS := R_CKCORE_ADDR32|R_CKCORE_JUMP_SLOT +include $(srctree)/lib/vdso/Makefile + +# Symbols present in the vdso +vdso-syms += rt_sigreturn +vdso-syms += vgettimeofday + +# Files to link into the vdso +obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o + +ifneq ($(c-gettimeofday-y),) + CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y) +endif + +ccflags-y := -fno-stack-protector -DBUILD_VDSO32 + +# Build rules +targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds vdso-dummy.o +obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) + +obj-y += vdso.o vdso-syms.o +CPPFLAGS_vdso.lds += -P -C -U$(ARCH) + +# Disable gcov profiling for VDSO code +GCOV_PROFILE := n +KCOV_INSTRUMENT := n + +# Force dependency +$(obj)/vdso.o: $(obj)/vdso.so + +SYSCFLAGS_vdso.so.dbg = $(c_flags) +$(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) FORCE + $(call if_changed,vdsold) +SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ + -Wl,--build-id=sha1 -Wl,--hash-style=both + +$(obj)/vdso-syms.S: $(obj)/vdso.so FORCE + $(call if_changed,so2s) + +# strip rule for the .so file +$(obj)/%.so: OBJCOPYFLAGS := -S +$(obj)/%.so: $(obj)/%.so.dbg FORCE + $(call if_changed,objcopy) + +# actual build commands +# The DSO images are built using a special linker script +# Make sure only to export the intended __vdso_xxx symbol offsets. +quiet_cmd_vdsold = VDSOLD $@ + cmd_vdsold = $(CC) $(KBUILD_CFLAGS) $(call cc-option, -no-pie) -nostdlib -nostartfiles $(SYSCFLAGS_$(@F)) \ + -Wl,-T,$(filter-out FORCE,$^) -o $@.tmp && \ + $(CROSS_COMPILE)objcopy \ + $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ + rm $@.tmp + +# Extracts symbol offsets from the VDSO, converting them into an assembly file +# that contains the same symbols at the same offsets. +quiet_cmd_so2s = SO2S $@ + cmd_so2s = $(NM) -D $< | $(srctree)/$(src)/so2s.sh > $@ + +# install commands for the unstripped file +quiet_cmd_vdso_install = INSTALL $@ + cmd_vdso_install = cp $(obj)/$@.dbg $(MODLIB)/vdso/$@ + +vdso.so: $(obj)/vdso.so.dbg + @mkdir -p $(MODLIB)/vdso + $(call cmd,vdso_install) + +vdso_install: vdso.so diff --git a/arch/csky/kernel/vdso/note.S b/arch/csky/kernel/vdso/note.S new file mode 100644 index 000000000000..2a956c942211 --- /dev/null +++ b/arch/csky/kernel/vdso/note.S @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. + * Here we can supply some information useful to userland. + */ + +#include +#include + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/csky/kernel/vdso/rt_sigreturn.S b/arch/csky/kernel/vdso/rt_sigreturn.S new file mode 100644 index 000000000000..0a6bd1216118 --- /dev/null +++ b/arch/csky/kernel/vdso/rt_sigreturn.S @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include +#include +#include + + .text +ENTRY(__vdso_rt_sigreturn) + .cfi_startproc + .cfi_signal_frame + SET_SYSCALL_ID + trap 0 + .cfi_endproc +ENDPROC(__vdso_rt_sigreturn) diff --git a/arch/csky/kernel/vdso/so2s.sh b/arch/csky/kernel/vdso/so2s.sh new file mode 100755 index 000000000000..69da3d529c6d --- /dev/null +++ b/arch/csky/kernel/vdso/so2s.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ + +sed 's!\([0-9a-f]*\) T \([a-z0-9_]*\)\(@@LINUX_5.10\)*!.global \2\n.set \2,0x\1!' \ +| grep '^\.' diff --git a/arch/csky/kernel/vdso/vdso.S b/arch/csky/kernel/vdso/vdso.S new file mode 100644 index 000000000000..5162ca069494 --- /dev/null +++ b/arch/csky/kernel/vdso/vdso.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include +#include +#include + + __PAGE_ALIGNED_DATA + + .globl vdso_start, vdso_end + .balign PAGE_SIZE +vdso_start: + .incbin "arch/csky/kernel/vdso/vdso.so" + .balign PAGE_SIZE +vdso_end: + + .previous diff --git a/arch/csky/kernel/vdso/vdso.lds.S b/arch/csky/kernel/vdso/vdso.lds.S new file mode 100644 index 000000000000..590a6c79fff7 --- /dev/null +++ b/arch/csky/kernel/vdso/vdso.lds.S @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include + +OUTPUT_ARCH(csky) + +SECTIONS +{ + PROVIDE(_vdso_data = . + PAGE_SIZE); + . = SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + .dynamic : { *(.dynamic) } :text :dynamic + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + + .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) } + + . = 0x800; + .text : { *(.text .text.*) } :text + + .data : { + *(.got.plt) *(.got) + *(.data .data.* .gnu.linkonce.d.*) + *(.dynbss) + *(.bss .bss.* .gnu.linkonce.b.*) + } +} + +PHDRS +{ + text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + +VERSION +{ + LINUX_5.10 { + global: + __vdso_rt_sigreturn; + __vdso_clock_gettime; + __vdso_clock_gettime64; + __vdso_gettimeofday; + __vdso_clock_getres; + local: *; + }; +} diff --git a/arch/csky/kernel/vdso/vgettimeofday.c b/arch/csky/kernel/vdso/vgettimeofday.c new file mode 100644 index 000000000000..da491832c098 --- /dev/null +++ b/arch/csky/kernel/vdso/vgettimeofday.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +int __vdso_clock_gettime(clockid_t clock, + struct old_timespec32 *ts) +{ + return __cvdso_clock_gettime32(clock, ts); +} + +int __vdso_clock_gettime64(clockid_t clock, + struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, + struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +int __vdso_clock_getres(clockid_t clock_id, + struct old_timespec32 *res) +{ + return __cvdso_clock_getres_time32(clock_id, res); +} diff --git a/arch/csky/kernel/vmlinux.lds.S b/arch/csky/kernel/vmlinux.lds.S index f03033e17c29..e8b1a4a49798 100644 --- a/arch/csky/kernel/vmlinux.lds.S +++ b/arch/csky/kernel/vmlinux.lds.S @@ -33,6 +33,7 @@ SECTIONS .text : AT(ADDR(.text) - LOAD_OFFSET) { _text = .; + VBR_BASE IRQENTRY_TEXT SOFTIRQENTRY_TEXT TEXT_TEXT @@ -104,7 +105,6 @@ SECTIONS EXCEPTION_TABLE(L1_CACHE_BYTES) BSS_SECTION(L1_CACHE_BYTES, PAGE_SIZE, L1_CACHE_BYTES) - VBR_BASE _end = . ; STABS_DEBUG diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c index 081b178b41b1..1482de56f4f7 100644 --- a/arch/csky/mm/fault.c +++ b/arch/csky/mm/fault.c @@ -1,29 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include #include - -#include -#include -#include -#include +#include +#include int fixup_exception(struct pt_regs *regs) { @@ -39,148 +20,40 @@ int fixup_exception(struct pt_regs *regs) return 0; } -/* - * This routine handles page faults. It determines the address, - * and the problem, and then passes it off to one of the appropriate - * routines. - */ -asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write, - unsigned long mmu_meh) +static inline bool is_write(struct pt_regs *regs) { - struct vm_area_struct *vma = NULL; - struct task_struct *tsk = current; - struct mm_struct *mm = tsk->mm; - int si_code; - int fault; - unsigned long address = mmu_meh & PAGE_MASK; - - if (kprobe_page_fault(regs, tsk->thread.trap_no)) - return; - - si_code = SEGV_MAPERR; - -#ifndef CONFIG_CPU_HAS_TLBI - /* - * We fault-in kernel-space virtual memory on-demand. The - * 'reference' page table is init_mm.pgd. - * - * NOTE! We MUST NOT take any locks for this case. We may - * be in an interrupt or a critical region, and should - * only copy the information from the master page table, - * nothing more. - */ - if (unlikely(address >= VMALLOC_START) && - unlikely(address <= VMALLOC_END)) { - /* - * Synchronize this task's top level page-table - * with the 'reference' page table. - * - * Do _not_ use "tsk" here. We might be inside - * an interrupt in the middle of a task switch.. - */ - int offset = pgd_index(address); - pgd_t *pgd, *pgd_k; - pud_t *pud, *pud_k; - pmd_t *pmd, *pmd_k; - pte_t *pte_k; - - unsigned long pgd_base; - - pgd_base = (unsigned long)__va(get_pgd()); - pgd = (pgd_t *)pgd_base + offset; - pgd_k = init_mm.pgd + offset; - - if (!pgd_present(*pgd_k)) - goto no_context; - set_pgd(pgd, *pgd_k); - - pud = (pud_t *)pgd; - pud_k = (pud_t *)pgd_k; - if (!pud_present(*pud_k)) - goto no_context; - - pmd = pmd_offset(pud, address); - pmd_k = pmd_offset(pud_k, address); - if (!pmd_present(*pmd_k)) - goto no_context; - set_pmd(pmd, *pmd_k); - - pte_k = pte_offset_kernel(pmd_k, address); - if (!pte_present(*pte_k)) - goto no_context; - return; + switch (trap_no(regs)) { + case VEC_TLBINVALIDS: + return true; + case VEC_TLBMODIFIED: + return true; } + + return false; +} + +#ifdef CONFIG_CPU_HAS_LDSTEX +static inline void csky_cmpxchg_fixup(struct pt_regs *regs) +{ + return; +} +#else +extern unsigned long csky_cmpxchg_ldw; +extern unsigned long csky_cmpxchg_stw; +static inline void csky_cmpxchg_fixup(struct pt_regs *regs) +{ + if (trap_no(regs) != VEC_TLBMODIFIED) + return; + + if (instruction_pointer(regs) == csky_cmpxchg_stw) + instruction_pointer_set(regs, csky_cmpxchg_ldw); + return; +} #endif - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); - /* - * If we're in an interrupt or have no user - * context, we must not take the fault.. - */ - if (in_atomic() || !mm) - goto bad_area_nosemaphore; - - mmap_read_lock(mm); - vma = find_vma(mm, address); - if (!vma) - goto bad_area; - if (vma->vm_start <= address) - goto good_area; - if (!(vma->vm_flags & VM_GROWSDOWN)) - goto bad_area; - if (expand_stack(vma, address)) - goto bad_area; - /* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. - */ -good_area: - si_code = SEGV_ACCERR; - - if (write) { - if (!(vma->vm_flags & VM_WRITE)) - goto bad_area; - } else { - if (unlikely(!vma_is_accessible(vma))) - goto bad_area; - } - - /* - * If for any reason at all we couldn't handle the fault, - * make sure we exit gracefully rather than endlessly redo - * the fault. - */ - fault = handle_mm_fault(vma, address, write ? FAULT_FLAG_WRITE : 0, - regs); - if (unlikely(fault & VM_FAULT_ERROR)) { - if (fault & VM_FAULT_OOM) - goto out_of_memory; - else if (fault & VM_FAULT_SIGBUS) - goto do_sigbus; - else if (fault & VM_FAULT_SIGSEGV) - goto bad_area; - BUG(); - } - mmap_read_unlock(mm); - return; - - /* - * Something tried to access memory that isn't in our memory map.. - * Fix it, but check if it's kernel or user first.. - */ -bad_area: - mmap_read_unlock(mm); - -bad_area_nosemaphore: - /* User mode accesses just cause a SIGSEGV */ - if (user_mode(regs)) { - tsk->thread.trap_no = trap_no(regs); - force_sig_fault(SIGSEGV, si_code, (void __user *)address); - return; - } - -no_context: - tsk->thread.trap_no = trap_no(regs); +static inline void no_context(struct pt_regs *regs, unsigned long addr) +{ + current->thread.trap_no = trap_no(regs); /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) @@ -192,27 +65,242 @@ no_context: */ bust_spinlocks(1); pr_alert("Unable to handle kernel paging request at virtual " - "address 0x%08lx, pc: 0x%08lx\n", address, regs->pc); + "addr 0x%08lx, pc: 0x%08lx\n", addr, regs->pc); die(regs, "Oops"); + do_exit(SIGKILL); +} -out_of_memory: - tsk->thread.trap_no = trap_no(regs); +static inline void mm_fault_error(struct pt_regs *regs, unsigned long addr, vm_fault_t fault) +{ + current->thread.trap_no = trap_no(regs); + + if (fault & VM_FAULT_OOM) { + /* + * We ran out of memory, call the OOM killer, and return the userspace + * (which will retry the fault, or kill us if we got oom-killed). + */ + if (!user_mode(regs)) { + no_context(regs, addr); + return; + } + pagefault_out_of_memory(); + return; + } else if (fault & VM_FAULT_SIGBUS) { + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) { + no_context(regs, addr); + return; + } + do_trap(regs, SIGBUS, BUS_ADRERR, addr); + return; + } + BUG(); +} + +static inline void bad_area(struct pt_regs *regs, struct mm_struct *mm, int code, unsigned long addr) +{ + /* + * Something tried to access memory that isn't in our memory map. + * Fix it, but check if it's kernel or user first. + */ + mmap_read_unlock(mm); + /* User mode accesses just cause a SIGSEGV */ + if (user_mode(regs)) { + do_trap(regs, SIGSEGV, code, addr); + return; + } + + no_context(regs, addr); +} + +static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long addr) +{ + pgd_t *pgd, *pgd_k; + pud_t *pud, *pud_k; + pmd_t *pmd, *pmd_k; + pte_t *pte_k; + int offset; + + /* User mode accesses just cause a SIGSEGV */ + if (user_mode(regs)) { + do_trap(regs, SIGSEGV, code, addr); + return; + } /* - * We ran out of memory, call the OOM killer, and return the userspace - * (which will retry the fault, or kill us if we got oom-killed). + * Synchronize this task's top level page-table + * with the 'reference' page table. + * + * Do _not_ use "tsk" here. We might be inside + * an interrupt in the middle of a task switch.. */ - pagefault_out_of_memory(); - return; + offset = pgd_index(addr); -do_sigbus: - tsk->thread.trap_no = trap_no(regs); + pgd = get_pgd() + offset; + pgd_k = init_mm.pgd + offset; + + if (!pgd_present(*pgd_k)) { + no_context(regs, addr); + return; + } + set_pgd(pgd, *pgd_k); + + pud = (pud_t *)pgd; + pud_k = (pud_t *)pgd_k; + if (!pud_present(*pud_k)) { + no_context(regs, addr); + return; + } + + pmd = pmd_offset(pud, addr); + pmd_k = pmd_offset(pud_k, addr); + if (!pmd_present(*pmd_k)) { + no_context(regs, addr); + return; + } + set_pmd(pmd, *pmd_k); + + pte_k = pte_offset_kernel(pmd_k, addr); + if (!pte_present(*pte_k)) { + no_context(regs, addr); + return; + } + + flush_tlb_one(addr); +} + +static inline bool access_error(struct pt_regs *regs, struct vm_area_struct *vma) +{ + if (is_write(regs)) { + if (!(vma->vm_flags & VM_WRITE)) + return true; + } else { + if (unlikely(!vma_is_accessible(vma))) + return true; + } + return false; +} + +/* + * This routine handles page faults. It determines the address and the + * problem, and then passes it off to one of the appropriate routines. + */ +asmlinkage void do_page_fault(struct pt_regs *regs) +{ + struct task_struct *tsk; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long addr = read_mmu_entryhi() & PAGE_MASK; + unsigned int flags = FAULT_FLAG_DEFAULT; + int code = SEGV_MAPERR; + vm_fault_t fault; + + tsk = current; + mm = tsk->mm; + + csky_cmpxchg_fixup(regs); + + if (kprobe_page_fault(regs, tsk->thread.trap_no)) + return; + + /* + * Fault-in kernel-space virtual memory on-demand. + * The 'reference' page table is init_mm.pgd. + * + * NOTE! We MUST NOT take any locks for this case. We may + * be in an interrupt or a critical region, and should + * only copy the information from the master page table, + * nothing more. + */ + if (unlikely((addr >= VMALLOC_START) && (addr <= VMALLOC_END))) { + vmalloc_fault(regs, code, addr); + return; + } + + /* Enable interrupts if they were enabled in the parent context. */ + if (likely(regs->sr & BIT(6))) + local_irq_enable(); + + /* + * If we're in an interrupt, have no user context, or are running + * in an atomic region, then we must not take the fault. + */ + if (unlikely(faulthandler_disabled() || !mm)) { + no_context(regs, addr); + return; + } + + if (user_mode(regs)) + flags |= FAULT_FLAG_USER; + + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); + + if (is_write(regs)) + flags |= FAULT_FLAG_WRITE; +retry: + mmap_read_lock(mm); + vma = find_vma(mm, addr); + if (unlikely(!vma)) { + bad_area(regs, mm, code, addr); + return; + } + if (likely(vma->vm_start <= addr)) + goto good_area; + if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) { + bad_area(regs, mm, code, addr); + return; + } + if (unlikely(expand_stack(vma, addr))) { + bad_area(regs, mm, code, addr); + return; + } + + /* + * Ok, we have a good vm_area for this memory access, so + * we can handle it. + */ +good_area: + code = SEGV_ACCERR; + + if (unlikely(access_error(regs, vma))) { + bad_area(regs, mm, code, addr); + return; + } + + /* + * If for any reason at all we could not handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ + fault = handle_mm_fault(vma, addr, flags, regs); + + /* + * If we need to retry but a fatal signal is pending, handle the + * signal first. We do not need to release the mmap_lock because it + * would already be released in __lock_page_or_retry in mm/filemap.c. + */ + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + no_context(regs, addr); + return; + } + + if (unlikely((fault & VM_FAULT_RETRY) && (flags & FAULT_FLAG_ALLOW_RETRY))) { + flags |= FAULT_FLAG_TRIED; + + /* + * No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + goto retry; + } mmap_read_unlock(mm); - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - goto no_context; - - force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); + if (unlikely(fault & VM_FAULT_ERROR)) { + mm_fault_error(regs, addr, fault); + return; + } + return; } diff --git a/arch/csky/mm/init.c b/arch/csky/mm/init.c index af627128314f..894050a8ce09 100644 --- a/arch/csky/mm/init.c +++ b/arch/csky/mm/init.c @@ -28,9 +28,15 @@ #include #include #include +#include + +#define PTRS_KERN_TABLE \ + ((PTRS_PER_PGD - USER_PTRS_PER_PGD) * PTRS_PER_PTE) pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; +pte_t kernel_pte_tables[PTRS_KERN_TABLE] __page_aligned_bss; + EXPORT_SYMBOL(invalid_pte_table); unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; @@ -80,9 +86,9 @@ void __init mem_init(void) #ifdef CONFIG_HIGHMEM unsigned long tmp; - max_mapnr = highend_pfn; + set_max_mapnr(highend_pfn - ARCH_PFN_OFFSET); #else - max_mapnr = max_low_pfn; + set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); #endif high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); @@ -104,24 +110,9 @@ void __init mem_init(void) mem_init_print_info(NULL); } -extern char __init_begin[], __init_end[]; - void free_initmem(void) { - unsigned long addr; - - addr = (unsigned long) &__init_begin; - - while (addr < (unsigned long) &__init_end) { - ClearPageReserved(virt_to_page(addr)); - init_page_count(virt_to_page(addr)); - free_page(addr); - totalram_pages_inc(); - addr += PAGE_SIZE; - } - - pr_info("Freeing unused kernel memory: %dk freed\n", - ((unsigned int)&__init_end - (unsigned int)&__init_begin) >> 10); + free_initmem_default(-1); } void pgd_init(unsigned long *p) @@ -130,20 +121,35 @@ void pgd_init(unsigned long *p) for (i = 0; i < PTRS_PER_PGD; i++) p[i] = __pa(invalid_pte_table); + + flush_tlb_all(); + local_icache_inv_all(NULL); } -void __init pre_mmu_init(void) +void __init mmu_init(unsigned long min_pfn, unsigned long max_pfn) { - /* - * Setup page-table and enable TLB-hardrefill - */ + int i; + + for (i = 0; i < USER_PTRS_PER_PGD; i++) + swapper_pg_dir[i].pgd = __pa(invalid_pte_table); + + for (i = USER_PTRS_PER_PGD; i < PTRS_PER_PGD; i++) + swapper_pg_dir[i].pgd = + __pa(kernel_pte_tables + (PTRS_PER_PTE * (i - USER_PTRS_PER_PGD))); + + for (i = 0; i < PTRS_KERN_TABLE; i++) + set_pte(&kernel_pte_tables[i], __pte(_PAGE_GLOBAL)); + + for (i = min_pfn; i < max_pfn; i++) + set_pte(&kernel_pte_tables[i - PFN_DOWN(va_pa_offset)], pfn_pte(i, PAGE_KERNEL)); + flush_tlb_all(); - pgd_init((unsigned long *)swapper_pg_dir); - TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); - TLBMISS_HANDLER_SETUP_PGD_KERNEL(swapper_pg_dir); + local_icache_inv_all(NULL); /* Setup page mask to 4k */ write_mmu_pagemask(0); + + setup_pgd(swapper_pg_dir, 0); } void __init fixrange_init(unsigned long start, unsigned long end, diff --git a/arch/csky/mm/tlb.c b/arch/csky/mm/tlb.c index ed1512381112..9234c5e5ceaf 100644 --- a/arch/csky/mm/tlb.c +++ b/arch/csky/mm/tlb.c @@ -24,7 +24,13 @@ void flush_tlb_all(void) void flush_tlb_mm(struct mm_struct *mm) { #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.asids %0"::"r"(cpu_asid(mm))); + sync_is(); + asm volatile( + "tlbi.asids %0 \n" + "sync.i \n" + : + : "r" (cpu_asid(mm)) + : "memory"); #else tlb_invalid_all(); #endif @@ -53,11 +59,17 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, end &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI + sync_is(); while (start < end) { - asm volatile("tlbi.vas %0"::"r"(start | newpid)); + asm volatile( + "tlbi.vas %0 \n" + : + : "r" (start | newpid) + : "memory"); + start += 2*PAGE_SIZE; } - sync_is(); + asm volatile("sync.i\n"); #else { unsigned long flags, oldpid; @@ -87,11 +99,17 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) end &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI + sync_is(); while (start < end) { - asm volatile("tlbi.vaas %0"::"r"(start)); + asm volatile( + "tlbi.vaas %0 \n" + : + : "r" (start) + : "memory"); + start += 2*PAGE_SIZE; } - sync_is(); + asm volatile("sync.i\n"); #else { unsigned long flags, oldpid; @@ -121,8 +139,13 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) addr &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.vas %0"::"r"(addr | newpid)); sync_is(); + asm volatile( + "tlbi.vas %0 \n" + "sync.i \n" + : + : "r" (addr | newpid) + : "memory"); #else { int oldpid, idx; @@ -147,8 +170,13 @@ void flush_tlb_one(unsigned long addr) addr &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.vaas %0"::"r"(addr)); sync_is(); + asm volatile( + "tlbi.vaas %0 \n" + "sync.i \n" + : + : "r" (addr) + : "memory"); #else { int oldpid, idx; diff --git a/arch/h8300/kernel/process.c b/arch/h8300/kernel/process.c index bc1364db58fe..46b1342ce515 100644 --- a/arch/h8300/kernel/process.c +++ b/arch/h8300/kernel/process.c @@ -112,7 +112,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, childregs = (struct pt_regs *) (THREAD_SIZE + task_stack_page(p)) - 1; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); childregs->retpc = (unsigned long) ret_from_kernel_thread; childregs->er4 = topstk; /* arg */ diff --git a/arch/hexagon/configs/comet_defconfig b/arch/hexagon/configs/comet_defconfig index e324f65f41e7..f19ae2ab0aaa 100644 --- a/arch/hexagon/configs/comet_defconfig +++ b/arch/hexagon/configs/comet_defconfig @@ -1,7 +1,6 @@ CONFIG_SMP=y CONFIG_DEFAULT_MMAP_MIN_ADDR=0 CONFIG_HZ_100=y -CONFIG_EXPERIMENTAL=y CONFIG_CROSS_COMPILE="hexagon-" CONFIG_LOCALVERSION="-smp" # CONFIG_LOCALVERSION_AUTO is not set diff --git a/arch/hexagon/kernel/process.c b/arch/hexagon/kernel/process.c index 6a980cba7b29..c61165c99ae0 100644 --- a/arch/hexagon/kernel/process.c +++ b/arch/hexagon/kernel/process.c @@ -73,7 +73,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, sizeof(*ss)); ss->lr = (unsigned long)ret_from_fork; p->thread.switch_sp = ss; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); /* r24 <- fn, r25 <- arg */ ss->r24 = usp; diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile index f3328a29e881..467b7e7f967c 100644 --- a/arch/ia64/Makefile +++ b/arch/ia64/Makefile @@ -14,7 +14,6 @@ KBUILD_DEFCONFIG := generic_defconfig NM := $(CROSS_COMPILE)nm -B -READELF := $(CROSS_COMPILE)readelf CHECKFLAGS += -D__ia64=1 -D__ia64__=1 -D_LP64 -D__LP64__ @@ -85,9 +84,3 @@ define archhelp echo ' install - Install compressed kernel image' echo '* unwcheck - Check vmlinux for invalid unwind info' endef - -archprepare: make_nr_irqs_h -PHONY += make_nr_irqs_h - -make_nr_irqs_h: - $(Q)$(MAKE) $(build)=arch/ia64/kernel include/generated/nr-irqs.h diff --git a/arch/ia64/include/asm/irq.h b/arch/ia64/include/asm/irq.h index 5acf52e90872..0eccf33dfe8b 100644 --- a/arch/ia64/include/asm/irq.h +++ b/arch/ia64/include/asm/irq.h @@ -14,7 +14,9 @@ #include #include -#include +#include + +#define NR_IRQS IA64_NATIVE_NR_IRQS static __inline__ int irq_canonicalize (int irq) diff --git a/arch/ia64/include/asm/mca.h b/arch/ia64/include/asm/mca.h index 726df17f1b51..05805249296c 100644 --- a/arch/ia64/include/asm/mca.h +++ b/arch/ia64/include/asm/mca.h @@ -14,13 +14,10 @@ #if !defined(__ASSEMBLY__) -#include +#include +#include #include - -#include -#include -#include -#include +#include #define IA64_MCA_RENDEZ_TIMEOUT (20 * 1000) /* value in milliseconds - 20 seconds */ @@ -83,7 +80,7 @@ struct ia64_sal_os_state { /* common */ unsigned long sal_ra; /* Return address in SAL, physical */ unsigned long sal_gp; /* GP of the SAL - physical */ - pal_min_state_area_t *pal_min_state; /* from R17. physical in asm, virtual in C */ + struct pal_min_state_area *pal_min_state; /* from R17. physical in asm, virtual in C */ /* Previous values of IA64_KR(CURRENT) and IA64_KR(CURRENT_STACK). * Note: if the MCA/INIT recovery code wants to resume to a new context * then it must change these values to reflect the new kernel stack. diff --git a/arch/ia64/include/asm/pal.h b/arch/ia64/include/asm/pal.h index f9d2b3b2dfad..b1d87955e8cc 100644 --- a/arch/ia64/include/asm/pal.h +++ b/arch/ia64/include/asm/pal.h @@ -750,7 +750,7 @@ typedef union pal_mc_error_info_u { * for PAL. */ -typedef struct pal_min_state_area_s { +struct pal_min_state_area { u64 pmsa_nat_bits; /* nat bits for saved GRs */ u64 pmsa_gr[15]; /* GR1 - GR15 */ u64 pmsa_bank0_gr[16]; /* GR16 - GR31 */ @@ -766,7 +766,7 @@ typedef struct pal_min_state_area_s { u64 pmsa_xfs; /* previous ifs */ u64 pmsa_br1; /* branch register 1 */ u64 pmsa_reserved[70]; /* pal_min_state_area should total to 1KB */ -} pal_min_state_area_t; +}; struct ia64_pal_retval { diff --git a/arch/ia64/include/asm/pgtable.h b/arch/ia64/include/asm/pgtable.h index 779b6972aa84..9b4efe89e62d 100644 --- a/arch/ia64/include/asm/pgtable.h +++ b/arch/ia64/include/asm/pgtable.h @@ -517,12 +517,6 @@ extern struct page *zero_page_memmap_ptr; __changed; \ }) #endif - -# ifdef CONFIG_VIRTUAL_MEM_MAP - /* arch mem_map init routine is needed due to holes in a virtual mem_map */ - extern void memmap_init (unsigned long size, int nid, unsigned long zone, - unsigned long start_pfn); -# endif /* CONFIG_VIRTUAL_MEM_MAP */ # endif /* !__ASSEMBLY__ */ /* diff --git a/arch/ia64/include/asm/sal.h b/arch/ia64/include/asm/sal.h index 08f5b6aaed73..78f4f7b40435 100644 --- a/arch/ia64/include/asm/sal.h +++ b/arch/ia64/include/asm/sal.h @@ -385,7 +385,7 @@ typedef struct sal_processor_static_info { fr : 1, reserved : 58; } valid; - pal_min_state_area_t min_state_area; + struct pal_min_state_area min_state_area; u64 br[8]; u64 cr[128]; u64 ar[128]; diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile index c89bd5f8cbf8..78717819131c 100644 --- a/arch/ia64/kernel/Makefile +++ b/arch/ia64/kernel/Makefile @@ -47,8 +47,3 @@ CFLAGS_traps.o += -mfixed-range=f2-f5,f16-f31 # The gate DSO image is built using a special linker script. include $(src)/Makefile.gate - -include/generated/nr-irqs.h: arch/$(SRCARCH)/kernel/nr-irqs.s FORCE - $(call filechk,offsets,__ASM_NR_IRQS_H__) - -targets += nr-irqs.s diff --git a/arch/ia64/kernel/asm-offsets.c b/arch/ia64/kernel/asm-offsets.c index fb0deb8a4221..be3b90fef2e9 100644 --- a/arch/ia64/kernel/asm-offsets.c +++ b/arch/ia64/kernel/asm-offsets.c @@ -245,23 +245,23 @@ void foo(void) BLANK(); DEFINE(IA64_PMSA_GR_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_gr)); + offsetof(struct pal_min_state_area, pmsa_gr)); DEFINE(IA64_PMSA_BANK1_GR_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_bank1_gr)); + offsetof(struct pal_min_state_area, pmsa_bank1_gr)); DEFINE(IA64_PMSA_PR_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_pr)); + offsetof(struct pal_min_state_area, pmsa_pr)); DEFINE(IA64_PMSA_BR0_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_br0)); + offsetof(struct pal_min_state_area, pmsa_br0)); DEFINE(IA64_PMSA_RSC_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_rsc)); + offsetof(struct pal_min_state_area, pmsa_rsc)); DEFINE(IA64_PMSA_IIP_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_iip)); + offsetof(struct pal_min_state_area, pmsa_iip)); DEFINE(IA64_PMSA_IPSR_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_ipsr)); + offsetof(struct pal_min_state_area, pmsa_ipsr)); DEFINE(IA64_PMSA_IFS_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_ifs)); + offsetof(struct pal_min_state_area, pmsa_ifs)); DEFINE(IA64_PMSA_XIP_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_xip)); + offsetof(struct pal_min_state_area, pmsa_xip)); BLANK(); /* used by fsys_gettimeofday in arch/ia64/kernel/fsys.S */ diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c index 4f47741005d2..76730f34685c 100644 --- a/arch/ia64/kernel/crash.c +++ b/arch/ia64/kernel/crash.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/ia64/kernel/efi.c b/arch/ia64/kernel/efi.c index dd7fd750bb93..c5fe21de46a8 100644 --- a/arch/ia64/kernel/efi.c +++ b/arch/ia64/kernel/efi.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index 0fea266b4d39..d4cae2fc69ca 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -97,6 +97,7 @@ #include #include #include +#include #include #include @@ -895,7 +896,7 @@ static void finish_pt_regs(struct pt_regs *regs, struct ia64_sal_os_state *sos, unsigned long *nat) { - const pal_min_state_area_t *ms = sos->pal_min_state; + const struct pal_min_state_area *ms = sos->pal_min_state; const u64 *bank; /* If ipsr.ic then use pmsa_{iip,ipsr,ifs}, else use @@ -971,7 +972,7 @@ ia64_mca_modify_original_stack(struct pt_regs *regs, char *p; ia64_va va; extern char ia64_leave_kernel[]; /* Need asm address, not function descriptor */ - const pal_min_state_area_t *ms = sos->pal_min_state; + const struct pal_min_state_area *ms = sos->pal_min_state; struct task_struct *previous_current; struct pt_regs *old_regs; struct switch_stack *old_sw; diff --git a/arch/ia64/kernel/mca_drv.c b/arch/ia64/kernel/mca_drv.c index 4d0ab323dee8..36a69b4e6169 100644 --- a/arch/ia64/kernel/mca_drv.c +++ b/arch/ia64/kernel/mca_drv.c @@ -496,7 +496,7 @@ recover_from_read_error(slidx_table_t *slidx, struct ia64_sal_os_state *sos) { u64 target_identifier; - pal_min_state_area_t *pmsa; + struct pal_min_state_area *pmsa; struct ia64_psr *psr1, *psr2; ia64_fptr_t *mca_hdlr_bh = (ia64_fptr_t*)mca_handler_bhhook; diff --git a/arch/ia64/kernel/nr-irqs.c b/arch/ia64/kernel/nr-irqs.c deleted file mode 100644 index f2633b22d3be..000000000000 --- a/arch/ia64/kernel/nr-irqs.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * calculate - * NR_IRQS = max(IA64_NATIVE_NR_IRQS, XEN_NR_IRQS, FOO_NR_IRQS...) - * depending on config. - * This must be calculated before processing asm-offset.c. - */ - -#define ASM_OFFSETS_C 1 - -#include -#include -#include - -void foo(void) -{ - union paravirt_nr_irqs_max { - char ia64_native_nr_irqs[IA64_NATIVE_NR_IRQS]; - }; - - DEFINE(NR_IRQS, sizeof (union paravirt_nr_irqs_max)); -} diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c index 4ebbfa076a26..7e1a1525e202 100644 --- a/arch/ia64/kernel/process.c +++ b/arch/ia64/kernel/process.c @@ -338,7 +338,7 @@ copy_thread(unsigned long clone_flags, unsigned long user_stack_base, ia64_drop_fpu(p); /* don't pick up stale state from a CPU's fph */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { if (unlikely(!user_stack_base)) { /* fork_idle() called us */ return 0; diff --git a/arch/ia64/kernel/syscalls/Makefile b/arch/ia64/kernel/syscalls/Makefile index 813a58cba39c..bf4bda0f63eb 100644 --- a/arch/ia64/kernel/syscalls/Makefile +++ b/arch/ia64/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -22,19 +22,20 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_offset_$(basetarget))' syshdr_offset_unistd_64 := __NR_Linux -$(uapi)/unistd_64.h: $(syscall) $(syshdr) +$(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) systbl_offset_syscall_table := 1024 -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_64.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/ia64/kernel/syscalls/syscall.tbl b/arch/ia64/kernel/syscalls/syscall.tbl index bfc00f2bd437..d89231166e19 100644 --- a/arch/ia64/kernel/syscalls/syscall.tbl +++ b/arch/ia64/kernel/syscalls/syscall.tbl @@ -362,3 +362,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index b19f47a5a305..16d0d7d22657 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -536,18 +536,20 @@ virtual_memmap_init(u64 start, u64 end, void *arg) / sizeof(struct page)); if (map_start < map_end) - memmap_init_zone((unsigned long)(map_end - map_start), + memmap_init_range((unsigned long)(map_end - map_start), args->nid, args->zone, page_to_pfn(map_start), page_to_pfn(map_end), MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); return 0; } -void __meminit -memmap_init (unsigned long size, int nid, unsigned long zone, - unsigned long start_pfn) +void __meminit memmap_init_zone(struct zone *zone) { + int nid = zone_to_nid(zone), zone_id = zone_idx(zone); + unsigned long start_pfn = zone->zone_start_pfn; + unsigned long size = zone->spanned_pages; + if (!vmem_map) { - memmap_init_zone(size, nid, zone, start_pfn, start_pfn + size, + memmap_init_range(size, nid, zone_id, start_pfn, start_pfn + size, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); } else { struct page *start; @@ -557,7 +559,7 @@ memmap_init (unsigned long size, int nid, unsigned long zone, args.start = start; args.end = start + size; args.nid = nid; - args.zone = zone; + args.zone = zone_id; efi_memmap_walk(virtual_memmap_init, &args); } diff --git a/arch/m68k/coldfire/clk.c b/arch/m68k/coldfire/clk.c index 7bc666e482eb..076a9caa9557 100644 --- a/arch/m68k/coldfire/clk.c +++ b/arch/m68k/coldfire/clk.c @@ -90,6 +90,10 @@ EXPORT_SYMBOL(clk_get); int clk_enable(struct clk *clk) { unsigned long flags; + + if (!clk) + return -EINVAL; + spin_lock_irqsave(&clk_lock, flags); if ((clk->enabled++ == 0) && clk->clk_ops) clk->clk_ops->enable(clk); diff --git a/arch/m68k/kernel/process.c b/arch/m68k/kernel/process.c index 08359a6e058f..da83cc83e791 100644 --- a/arch/m68k/kernel/process.c +++ b/arch/m68k/kernel/process.c @@ -157,7 +157,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, */ p->thread.fs = get_fs().seg; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ memset(frame, 0, sizeof(struct fork_frame)); frame->regs.sr = PS_S; diff --git a/arch/m68k/kernel/syscalls/Makefile b/arch/m68k/kernel/syscalls/Makefile index 659faefdcb1d..285aaba832d9 100644 --- a/arch/m68k/kernel/syscalls/Makefile +++ b/arch/m68k/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,18 +21,19 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_abi_$(basetarget))' \ '$(systbl_offset_$(basetarget))' -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl index 7fe4e45c864c..72bde6707dd3 100644 --- a/arch/m68k/kernel/syscalls/syscall.tbl +++ b/arch/m68k/kernel/syscalls/syscall.tbl @@ -441,3 +441,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index 25a5a3fb14aa..0660f47012bc 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -38,7 +38,6 @@ config MICROBLAZE select OF_EARLY_FLATTREE select PCI_DOMAINS_GENERIC if PCI select PCI_SYSCALL if PCI - select TRACING_SUPPORT select VIRT_TO_BUS select CPU_NO_EFFICIENT_FFS select MMU_GATHER_NO_RANGE diff --git a/arch/microblaze/kernel/module.c b/arch/microblaze/kernel/module.c index 9f12e3c2bb42..e5db3a57b9e3 100644 --- a/arch/microblaze/kernel/module.c +++ b/arch/microblaze/kernel/module.c @@ -24,9 +24,6 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, Elf32_Sym *sym; unsigned long int *location; unsigned long int value; -#if __GNUC__ < 4 - unsigned long int old_value; -#endif pr_debug("Applying add relocation section %u to %u\n", relsec, sechdrs[relsec].sh_info); @@ -49,40 +46,17 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, */ case R_MICROBLAZE_32: -#if __GNUC__ < 4 - old_value = *location; - *location = value + old_value; - - pr_debug("R_MICROBLAZE_32 (%08lx->%08lx)\n", - old_value, value); -#else *location = value; -#endif break; case R_MICROBLAZE_64: -#if __GNUC__ < 4 - /* Split relocs only required/used pre gcc4.1.1 */ - old_value = ((location[0] & 0x0000FFFF) << 16) | - (location[1] & 0x0000FFFF); - value += old_value; -#endif location[0] = (location[0] & 0xFFFF0000) | (value >> 16); location[1] = (location[1] & 0xFFFF0000) | (value & 0xFFFF); -#if __GNUC__ < 4 - pr_debug("R_MICROBLAZE_64 (%08lx->%08lx)\n", - old_value, value); -#endif break; case R_MICROBLAZE_64_PCREL: -#if __GNUC__ < 4 - old_value = (location[0] & 0xFFFF) << 16 | - (location[1] & 0xFFFF); - value -= old_value; -#endif value -= (unsigned long int)(location) + 4; location[0] = (location[0] & 0xFFFF0000) | (value >> 16); diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c index 657c2beb665e..62aa237180b6 100644 --- a/arch/microblaze/kernel/process.c +++ b/arch/microblaze/kernel/process.c @@ -59,7 +59,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, struct pt_regs *childregs = task_pt_regs(p); struct thread_info *ti = task_thread_info(p); - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* if we're creating a new kernel thread then just zeroing all * the registers. That's OK for a brand new thread.*/ memset(childregs, 0, sizeof(struct pt_regs)); diff --git a/arch/microblaze/kernel/syscalls/Makefile b/arch/microblaze/kernel/syscalls/Makefile index 659faefdcb1d..285aaba832d9 100644 --- a/arch/microblaze/kernel/syscalls/Makefile +++ b/arch/microblaze/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,18 +21,19 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_abi_$(basetarget))' \ '$(systbl_offset_$(basetarget))' -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl index a522adf194ab..d603a5ec9338 100644 --- a/arch/microblaze/kernel/syscalls/syscall.tbl +++ b/arch/microblaze/kernel/syscalls/syscall.tbl @@ -447,3 +447,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/microblaze/kernel/vmlinux.lds.S b/arch/microblaze/kernel/vmlinux.lds.S index df07b3d06cd6..fb31747ec092 100644 --- a/arch/microblaze/kernel/vmlinux.lds.S +++ b/arch/microblaze/kernel/vmlinux.lds.S @@ -45,7 +45,7 @@ SECTIONS { _etext = . ; } - . = ALIGN (4) ; + . = ALIGN (8) ; __fdt_blob : AT(ADDR(__fdt_blob) - LOAD_OFFSET) { _fdt_start = . ; /* place for fdt blob */ *(__fdt_blob) ; /* Any link-placed DTB */ diff --git a/arch/mips/bmips/setup.c b/arch/mips/bmips/setup.c index 95f8f10d8697..31bcfa4e08b9 100644 --- a/arch/mips/bmips/setup.c +++ b/arch/mips/bmips/setup.c @@ -196,4 +196,4 @@ static int __init plat_dev_init(void) return 0; } -device_initcall(plat_dev_init); +arch_initcall(plat_dev_init); diff --git a/arch/mips/configs/nlm_xlp_defconfig b/arch/mips/configs/nlm_xlp_defconfig index 72a211d2d556..32c290611723 100644 --- a/arch/mips/configs/nlm_xlp_defconfig +++ b/arch/mips/configs/nlm_xlp_defconfig @@ -549,7 +549,6 @@ CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y # CONFIG_ENABLE_MUST_CHECK is not set CONFIG_FRAME_WARN=1024 -CONFIG_UNUSED_SYMBOLS=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_DETECT_HUNG_TASK=y CONFIG_SCHEDSTATS=y diff --git a/arch/mips/configs/nlm_xlr_defconfig b/arch/mips/configs/nlm_xlr_defconfig index 4ecb157e56d4..bf9b9244929e 100644 --- a/arch/mips/configs/nlm_xlr_defconfig +++ b/arch/mips/configs/nlm_xlr_defconfig @@ -500,7 +500,6 @@ CONFIG_CRC7=m CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y # CONFIG_ENABLE_MUST_CHECK is not set -CONFIG_UNUSED_SYMBOLS=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_DETECT_HUNG_TASK=y CONFIG_SCHEDSTATS=y diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index af4c862ec5ff..7efa0d1a4c2b 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -120,7 +120,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, /* Put the stack after the struct pt_regs. */ childksp = (unsigned long) childregs; p->thread.cp0_status = (read_c0_status() & ~(ST0_CU2|ST0_CU1)) | ST0_KERNEL_CUMASK; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ unsigned long status = p->thread.cp0_status; memset(childregs, 0, sizeof(struct pt_regs)); diff --git a/arch/mips/kernel/r4k-bugs64.c b/arch/mips/kernel/r4k-bugs64.c index 1ff19f1ea5ca..35729c9e6cfa 100644 --- a/arch/mips/kernel/r4k-bugs64.c +++ b/arch/mips/kernel/r4k-bugs64.c @@ -18,7 +18,7 @@ static char bug64hit[] __initdata = "reliable operation impossible!\n%s"; static char nowar[] __initdata = - "Please report to ."; + "Please report to ."; static char r4kwar[] __initdata = "Enable CPU_R4000_WORKAROUNDS to rectify."; static char daddiwar[] __initdata = diff --git a/arch/mips/kernel/syscalls/Makefile b/arch/mips/kernel/syscalls/Makefile index a1ce8b7dbcfa..51f8b805f2ed 100644 --- a/arch/mips/kernel/syscalls/Makefile +++ b/arch/mips/kernel/syscalls/Makefile @@ -5,9 +5,9 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscalln32 := $(srctree)/$(src)/syscall_n32.tbl -syscalln64 := $(srctree)/$(src)/syscall_n64.tbl -syscallo32 := $(srctree)/$(src)/syscall_o32.tbl +syscalln32 := $(src)/syscall_n32.tbl +syscalln64 := $(src)/syscall_n64.tbl +syscallo32 := $(src)/syscall_o32.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh sysnr := $(srctree)/$(src)/syscallnr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -31,50 +31,50 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_offset_$(basetarget))' syshdr_offset_unistd_n32 := __NR_Linux -$(uapi)/unistd_n32.h: $(syscalln32) $(syshdr) +$(uapi)/unistd_n32.h: $(syscalln32) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_offset_unistd_n64 := __NR_Linux -$(uapi)/unistd_n64.h: $(syscalln64) $(syshdr) +$(uapi)/unistd_n64.h: $(syscalln64) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_offset_unistd_o32 := __NR_Linux -$(uapi)/unistd_o32.h: $(syscallo32) $(syshdr) +$(uapi)/unistd_o32.h: $(syscallo32) $(syshdr) FORCE $(call if_changed,syshdr) sysnr_pfx_unistd_nr_n32 := N32 sysnr_offset_unistd_nr_n32 := 6000 -$(kapi)/unistd_nr_n32.h: $(syscalln32) $(sysnr) +$(kapi)/unistd_nr_n32.h: $(syscalln32) $(sysnr) FORCE $(call if_changed,sysnr) sysnr_pfx_unistd_nr_n64 := 64 sysnr_offset_unistd_nr_n64 := 5000 -$(kapi)/unistd_nr_n64.h: $(syscalln64) $(sysnr) +$(kapi)/unistd_nr_n64.h: $(syscalln64) $(sysnr) FORCE $(call if_changed,sysnr) sysnr_pfx_unistd_nr_o32 := O32 sysnr_offset_unistd_nr_o32 := 4000 -$(kapi)/unistd_nr_o32.h: $(syscallo32) $(sysnr) +$(kapi)/unistd_nr_o32.h: $(syscallo32) $(sysnr) FORCE $(call if_changed,sysnr) systbl_abi_syscall_table_32_o32 := 32_o32 systbl_offset_syscall_table_32_o32 := 4000 -$(kapi)/syscall_table_32_o32.h: $(syscallo32) $(systbl) +$(kapi)/syscall_table_32_o32.h: $(syscallo32) $(systbl) FORCE $(call if_changed,systbl) systbl_abi_syscall_table_64_n32 := 64_n32 systbl_offset_syscall_table_64_n32 := 6000 -$(kapi)/syscall_table_64_n32.h: $(syscalln32) $(systbl) +$(kapi)/syscall_table_64_n32.h: $(syscalln32) $(systbl) FORCE $(call if_changed,systbl) systbl_abi_syscall_table_64_n64 := 64_n64 systbl_offset_syscall_table_64_n64 := 5000 -$(kapi)/syscall_table_64_n64.h: $(syscalln64) $(systbl) +$(kapi)/syscall_table_64_n64.h: $(syscalln64) $(systbl) FORCE $(call if_changed,systbl) systbl_abi_syscall_table_64_o32 := 64_o32 systbl_offset_syscall_table_64_o32 := 4000 -$(kapi)/syscall_table_64_o32.h: $(syscallo32) $(systbl) +$(kapi)/syscall_table_64_o32.h: $(syscallo32) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_n32.h \ @@ -88,9 +88,10 @@ kapisyshdr-y += syscall_table_32_o32.h \ unistd_nr_n64.h \ unistd_nr_o32.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl index 0f03ad223f33..8fd8c1790941 100644 --- a/arch/mips/kernel/syscalls/syscall_n32.tbl +++ b/arch/mips/kernel/syscalls/syscall_n32.tbl @@ -380,3 +380,4 @@ 439 n32 faccessat2 sys_faccessat2 440 n32 process_madvise sys_process_madvise 441 n32 epoll_pwait2 compat_sys_epoll_pwait2 +442 n32 mount_setattr sys_mount_setattr diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl index 91649690b52f..169f21438065 100644 --- a/arch/mips/kernel/syscalls/syscall_n64.tbl +++ b/arch/mips/kernel/syscalls/syscall_n64.tbl @@ -356,3 +356,4 @@ 439 n64 faccessat2 sys_faccessat2 440 n64 process_madvise sys_process_madvise 441 n64 epoll_pwait2 sys_epoll_pwait2 +442 n64 mount_setattr sys_mount_setattr diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl index 4bad0c40aed6..090d29ca80ff 100644 --- a/arch/mips/kernel/syscalls/syscall_o32.tbl +++ b/arch/mips/kernel/syscalls/syscall_o32.tbl @@ -429,3 +429,4 @@ 439 o32 faccessat2 sys_faccessat2 440 o32 process_madvise sys_process_madvise 441 o32 epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 o32 mount_setattr sys_mount_setattr diff --git a/arch/mips/lib/iomap-pci.c b/arch/mips/lib/iomap-pci.c index 210f5a95ecb1..a9cb28813f0b 100644 --- a/arch/mips/lib/iomap-pci.c +++ b/arch/mips/lib/iomap-pci.c @@ -32,7 +32,7 @@ void __iomem *__pci_ioport_map(struct pci_dev *dev, sprintf(name, "%04x:%02x", pci_domain_nr(bus), bus->number); printk(KERN_WARNING "io_map_base of root PCI bus %s unset. " "Trying to continue but you better\nfix this issue or " - "report it to linux-mips@linux-mips.org or your " + "report it to linux-mips@vger.kernel.org or your " "vendor.\n", name); #ifdef CONFIG_PCI_DOMAINS panic("To avoid data corruption io_map_base MUST be set with " diff --git a/arch/mips/loongson64/Platform b/arch/mips/loongson64/Platform index e2354e128d9a..3e660d6d3c2b 100644 --- a/arch/mips/loongson64/Platform +++ b/arch/mips/loongson64/Platform @@ -13,7 +13,7 @@ cflags-$(CONFIG_CPU_LOONGSON64) += -Wa,--trap # can't easily be used safely within the kbuild framework. # ifeq ($(call cc-ifversion, -ge, 0409, y), y) - ifeq ($(call ld-ifversion, -ge, 225000000, y), y) + ifeq ($(call ld-ifversion, -ge, 22500, y), y) cflags-$(CONFIG_CPU_LOONGSON64) += \ $(call cc-option,-march=loongson3a -U_MIPS_ISA -D_MIPS_ISA=_MIPS_ISA_MIPS64) else diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index 1754498b0717..7719d632df8d 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -157,29 +157,31 @@ unsigned long _page_cachable_default; EXPORT_SYMBOL(_page_cachable_default); #define PM(p) __pgprot(_page_cachable_default | (p)) +#define PVA(p) PM(_PAGE_VALID | _PAGE_ACCESSED | (p)) static inline void setup_protection_map(void) { protection_map[0] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); - protection_map[1] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC); - protection_map[2] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); - protection_map[3] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC); - protection_map[4] = PM(_PAGE_PRESENT); - protection_map[5] = PM(_PAGE_PRESENT); - protection_map[6] = PM(_PAGE_PRESENT); - protection_map[7] = PM(_PAGE_PRESENT); + protection_map[1] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC); + protection_map[2] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); + protection_map[3] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC); + protection_map[4] = PVA(_PAGE_PRESENT); + protection_map[5] = PVA(_PAGE_PRESENT); + protection_map[6] = PVA(_PAGE_PRESENT); + protection_map[7] = PVA(_PAGE_PRESENT); protection_map[8] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); - protection_map[9] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC); - protection_map[10] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE | + protection_map[9] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC); + protection_map[10] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE | _PAGE_NO_READ); - protection_map[11] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE); - protection_map[12] = PM(_PAGE_PRESENT); - protection_map[13] = PM(_PAGE_PRESENT); - protection_map[14] = PM(_PAGE_PRESENT | _PAGE_WRITE); - protection_map[15] = PM(_PAGE_PRESENT | _PAGE_WRITE); + protection_map[11] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE); + protection_map[12] = PVA(_PAGE_PRESENT); + protection_map[13] = PVA(_PAGE_PRESENT); + protection_map[14] = PVA(_PAGE_PRESENT); + protection_map[15] = PVA(_PAGE_PRESENT); } +#undef _PVA #undef PM void cpu_cache_init(void) diff --git a/arch/mips/mm/pgtable-32.c b/arch/mips/mm/pgtable-32.c index bd4b0656add3..61891af25019 100644 --- a/arch/mips/mm/pgtable-32.c +++ b/arch/mips/mm/pgtable-32.c @@ -45,7 +45,6 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; - flush_tlb_all(); } #endif /* defined(CONFIG_TRANSPARENT_HUGEPAGE) */ diff --git a/arch/mips/mm/pgtable-64.c b/arch/mips/mm/pgtable-64.c index 183ff9f9c026..7536f7804c44 100644 --- a/arch/mips/mm/pgtable-64.c +++ b/arch/mips/mm/pgtable-64.c @@ -100,7 +100,6 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; - flush_tlb_all(); } void __init pagetable_init(void) diff --git a/arch/mips/sgi-ip32/ip32-irq.c b/arch/mips/sgi-ip32/ip32-irq.c index 1bbd5bfb5458..e21ea1de05e3 100644 --- a/arch/mips/sgi-ip32/ip32-irq.c +++ b/arch/mips/sgi-ip32/ip32-irq.c @@ -343,7 +343,7 @@ static void ip32_unknown_interrupt(void) printk("Register dump:\n"); show_regs(get_irq_regs()); - printk("Please mail this report to linux-mips@linux-mips.org\n"); + printk("Please mail this report to linux-mips@vger.kernel.org\n"); printk("Spinning..."); while(1) ; } diff --git a/arch/mips/vdso/Kconfig b/arch/mips/vdso/Kconfig index 7aec721398d5..a665f6108cb5 100644 --- a/arch/mips/vdso/Kconfig +++ b/arch/mips/vdso/Kconfig @@ -12,7 +12,7 @@ # the lack of relocations. As such, we disable the VDSO for microMIPS builds. config MIPS_LD_CAN_LINK_VDSO - def_bool LD_VERSION >= 225000000 || LD_IS_LLD + def_bool LD_VERSION >= 22500 || LD_IS_LLD config MIPS_DISABLE_VDSO def_bool CPU_MICROMIPS || (!CPU_MIPSR6 && !MIPS_LD_CAN_LINK_VDSO) diff --git a/arch/nds32/configs/defconfig b/arch/nds32/configs/defconfig index 40313a635075..f9a89cf00aa6 100644 --- a/arch/nds32/configs/defconfig +++ b/arch/nds32/configs/defconfig @@ -1,4 +1,3 @@ -CONFIG_CROSS_COMPILE="nds32le-linux-" CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_HIGH_RES_TIMERS=y diff --git a/arch/nds32/kernel/process.c b/arch/nds32/kernel/process.c index e01ad5d17224..c1327e552ec6 100644 --- a/arch/nds32/kernel/process.c +++ b/arch/nds32/kernel/process.c @@ -156,7 +156,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); /* kernel thread fn */ p->thread.cpu_context.r6 = stack_start; diff --git a/arch/nds32/kernel/setup.c b/arch/nds32/kernel/setup.c index c356e484dcab..af82e996f412 100644 --- a/arch/nds32/kernel/setup.c +++ b/arch/nds32/kernel/setup.c @@ -52,7 +52,7 @@ EXPORT_SYMBOL(elf_hwcap); /* * The following string table, must sync with HWCAP_xx bitmask, - * which is defined in + * which is defined above */ static const char *hwcap_str[] = { "mfusr_pc", diff --git a/arch/nds32/kernel/time.c b/arch/nds32/kernel/time.c index ac9d78ce3a81..574a3d0a8539 100644 --- a/arch/nds32/kernel/time.c +++ b/arch/nds32/kernel/time.c @@ -2,7 +2,7 @@ // Copyright (C) 2005-2017 Andes Technology Corporation #include -#include +#include void __init time_init(void) { diff --git a/arch/nds32/kernel/traps.c b/arch/nds32/kernel/traps.c index 6a9772ba7392..ee0d9ae192a5 100644 --- a/arch/nds32/kernel/traps.c +++ b/arch/nds32/kernel/traps.c @@ -25,17 +25,8 @@ extern void show_pte(struct mm_struct *mm, unsigned long addr); void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) { unsigned long first; - mm_segment_t fs; int i; - /* - * We need to switch to kernel mode so that we can use __get_user - * to safely read from kernel space. Note that we now dump the - * code first, just in case the backtrace kills us. - */ - fs = get_fs(); - set_fs(KERNEL_DS); - pr_emerg("%s(0x%08lx to 0x%08lx)\n", lvl, bottom, top); for (first = bottom & ~31; first < top; first += 32) { @@ -48,7 +39,9 @@ void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) for (p = first, i = 0; i < 8 && p < top; i++, p += 4) { if (p >= bottom && p < top) { unsigned long val; - if (__get_user(val, (unsigned long *)p) == 0) + + if (get_kernel_nofault(val, + (unsigned long *)p) == 0) sprintf(str + i * 9, " %08lx", val); else sprintf(str + i * 9, " ????????"); @@ -56,46 +49,10 @@ void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) } pr_emerg("%s%04lx:%s\n", lvl, first & 0xffff, str); } - - set_fs(fs); } EXPORT_SYMBOL(dump_mem); -static void dump_instr(struct pt_regs *regs) -{ - unsigned long addr = instruction_pointer(regs); - mm_segment_t fs; - char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; - int i; - - return; - /* - * We need to switch to kernel mode so that we can use __get_user - * to safely read from kernel space. Note that we now dump the - * code first, just in case the backtrace kills us. - */ - fs = get_fs(); - set_fs(KERNEL_DS); - - pr_emerg("Code: "); - for (i = -4; i < 1; i++) { - unsigned int val, bad; - - bad = __get_user(val, &((u32 *) addr)[i]); - - if (!bad) { - p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val); - } else { - p += sprintf(p, "bad PC value"); - break; - } - } - pr_emerg("Code: %s\n", str); - - set_fs(fs); -} - #define LOOP_TIMES (100) static void __dump(struct task_struct *tsk, unsigned long *base_reg, const char *loglvl) @@ -179,7 +136,6 @@ void die(const char *str, struct pt_regs *regs, int err) if (!user_mode(regs) || in_interrupt()) { dump_mem("Stack: ", regs->sp, (regs->sp + PAGE_SIZE) & PAGE_MASK); - dump_instr(regs); dump_stack(); } diff --git a/arch/nios2/kernel/entry.S b/arch/nios2/kernel/entry.S index da8442450e46..0794cd7803df 100644 --- a/arch/nios2/kernel/entry.S +++ b/arch/nios2/kernel/entry.S @@ -389,7 +389,10 @@ ENTRY(ret_from_interrupt) */ ENTRY(sys_clone) SAVE_SWITCH_STACK + subi sp, sp, 4 /* make space for tls pointer */ + stw r8, 0(sp) /* pass tls pointer (r8) via stack (5th argument) */ call nios2_clone + addi sp, sp, 4 RESTORE_SWITCH_STACK ret diff --git a/arch/nios2/kernel/process.c b/arch/nios2/kernel/process.c index 50b4eb19a6cc..c5f916ca6845 100644 --- a/arch/nios2/kernel/process.c +++ b/arch/nios2/kernel/process.c @@ -109,7 +109,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, struct switch_stack *childstack = ((struct switch_stack *)childregs) - 1; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childstack, 0, sizeof(struct switch_stack) + sizeof(struct pt_regs)); diff --git a/arch/nios2/kernel/setup.c b/arch/nios2/kernel/setup.c index 3c6e3c813a0b..d2f21957e99c 100644 --- a/arch/nios2/kernel/setup.c +++ b/arch/nios2/kernel/setup.c @@ -32,8 +32,6 @@ EXPORT_SYMBOL(memory_start); unsigned long memory_end; EXPORT_SYMBOL(memory_end); -unsigned long memory_size; - static struct pt_regs fake_regs = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; @@ -141,16 +139,22 @@ asmlinkage void __init nios2_boot_init(unsigned r4, unsigned r5, unsigned r6, parse_early_param(); } +static void __init find_limits(unsigned long *min, unsigned long *max_low, + unsigned long *max_high) +{ + *max_low = PFN_DOWN(memblock_get_current_limit()); + *min = PFN_UP(memblock_start_of_DRAM()); + *max_high = PFN_DOWN(memblock_end_of_DRAM()); +} + void __init setup_arch(char **cmdline_p) { int dram_start; console_verbose(); - dram_start = memblock_start_of_DRAM(); - memory_size = memblock_phys_mem_size(); - memory_start = PAGE_ALIGN((unsigned long)__pa(_end)); - memory_end = (unsigned long) CONFIG_NIOS2_MEM_BASE + memory_size; + memory_start = memblock_start_of_DRAM(); + memory_end = memblock_end_of_DRAM(); init_mm.start_code = (unsigned long) _stext; init_mm.end_code = (unsigned long) _etext; @@ -161,11 +165,10 @@ void __init setup_arch(char **cmdline_p) /* Keep a copy of command line */ *cmdline_p = boot_command_line; - min_low_pfn = PFN_UP(memory_start); - max_low_pfn = PFN_DOWN(memory_end); + find_limits(&min_low_pfn, &max_low_pfn, &max_pfn); max_mapnr = max_low_pfn; - memblock_reserve(dram_start, memory_start - dram_start); + memblock_reserve(__pa_symbol(_stext), _end - _stext); #ifdef CONFIG_BLK_DEV_INITRD if (initrd_start) { memblock_reserve(virt_to_phys((void *)initrd_start), diff --git a/arch/nios2/kernel/sys_nios2.c b/arch/nios2/kernel/sys_nios2.c index cd390ec4f88b..b1ca85699952 100644 --- a/arch/nios2/kernel/sys_nios2.c +++ b/arch/nios2/kernel/sys_nios2.c @@ -22,6 +22,7 @@ asmlinkage int sys_cacheflush(unsigned long addr, unsigned long len, unsigned int op) { struct vm_area_struct *vma; + struct mm_struct *mm = current->mm; if (len == 0) return 0; @@ -34,16 +35,22 @@ asmlinkage int sys_cacheflush(unsigned long addr, unsigned long len, if (addr + len < addr) return -EFAULT; + if (mmap_read_lock_killable(mm)) + return -EINTR; + /* * Verify that the specified address region actually belongs * to this process. */ - vma = find_vma(current->mm, addr); - if (vma == NULL || addr < vma->vm_start || addr + len > vma->vm_end) + vma = find_vma(mm, addr); + if (vma == NULL || addr < vma->vm_start || addr + len > vma->vm_end) { + mmap_read_unlock(mm); return -EFAULT; + } flush_cache_range(vma, addr, addr + len); + mmap_read_unlock(mm); return 0; } diff --git a/arch/openrisc/Kbuild b/arch/openrisc/Kbuild new file mode 100644 index 000000000000..4234b4c03e72 --- /dev/null +++ b/arch/openrisc/Kbuild @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-y += lib/ kernel/ mm/ +obj-y += boot/dts/ diff --git a/arch/openrisc/Makefile b/arch/openrisc/Makefile index bf10141c7426..410e7abfac69 100644 --- a/arch/openrisc/Makefile +++ b/arch/openrisc/Makefile @@ -24,6 +24,10 @@ LIBGCC := $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name) KBUILD_CFLAGS += -pipe -ffixed-r10 -D__linux__ +all: vmlinux.bin + +boot := arch/$(ARCH)/boot + ifeq ($(CONFIG_OPENRISC_HAVE_INST_MUL),y) KBUILD_CFLAGS += $(call cc-option,-mhard-mul) else @@ -38,14 +42,13 @@ endif head-y := arch/openrisc/kernel/head.o -core-y += arch/openrisc/lib/ \ - arch/openrisc/kernel/ \ - arch/openrisc/mm/ +core-y += arch/openrisc/ libs-y += $(LIBGCC) -ifneq '$(CONFIG_OPENRISC_BUILTIN_DTB)' '""' -BUILTIN_DTB := y -else -BUILTIN_DTB := n -endif -core-$(BUILTIN_DTB) += arch/openrisc/boot/dts/ +PHONY += vmlinux.bin + +vmlinux.bin: vmlinux + $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ + +archclean: + $(Q)$(MAKE) $(clean)=$(boot) diff --git a/arch/openrisc/boot/.gitignore b/arch/openrisc/boot/.gitignore new file mode 100644 index 000000000000..007d6fea3145 --- /dev/null +++ b/arch/openrisc/boot/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +vmlinux.bin diff --git a/arch/openrisc/boot/Makefile b/arch/openrisc/boot/Makefile new file mode 100644 index 000000000000..5b28538f4dd1 --- /dev/null +++ b/arch/openrisc/boot/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for bootable kernel images +# + +targets += vmlinux.bin + +OBJCOPYFLAGS_vmlinux.bin := -O binary +$(obj)/vmlinux.bin: vmlinux FORCE + $(call if_changed,objcopy) diff --git a/arch/openrisc/kernel/process.c b/arch/openrisc/kernel/process.c index 3c98728cce24..eb62429681fc 100644 --- a/arch/openrisc/kernel/process.c +++ b/arch/openrisc/kernel/process.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -49,10 +50,16 @@ */ struct thread_info *current_thread_info_set[NR_CPUS] = { &init_thread_info, }; -void machine_restart(void) +void machine_restart(char *cmd) { - printk(KERN_INFO "*** MACHINE RESTART ***\n"); - __asm__("l.nop 1"); + do_kernel_restart(cmd); + + /* Give a grace period for failure to restart of 1s */ + mdelay(1000); + + /* Whoops - the platform was unable to reboot. Tell the user! */ + pr_emerg("Reboot failed -- System halted\n"); + while (1); } /* @@ -167,7 +174,7 @@ copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, sp -= sizeof(struct pt_regs); kregs = (struct pt_regs *)sp; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(kregs, 0, sizeof(struct pt_regs)); kregs->gpr[20] = usp; /* fn, kernel thread */ kregs->gpr[22] = arg; diff --git a/arch/openrisc/kernel/smp.c b/arch/openrisc/kernel/smp.c index 29c82ef2e207..48e1092a64de 100644 --- a/arch/openrisc/kernel/smp.c +++ b/arch/openrisc/kernel/smp.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -60,22 +61,32 @@ void __init smp_prepare_boot_cpu(void) void __init smp_init_cpus(void) { - int i; + struct device_node *cpu; + u32 cpu_id; - for (i = 0; i < NR_CPUS; i++) - set_cpu_possible(i, true); + for_each_of_cpu_node(cpu) { + if (of_property_read_u32(cpu, "reg", &cpu_id)) { + pr_warn("%s missing reg property", cpu->full_name); + continue; + } + + if (cpu_id < NR_CPUS) + set_cpu_possible(cpu_id, true); + } } void __init smp_prepare_cpus(unsigned int max_cpus) { - int i; + unsigned int cpu; /* * Initialise the present map, which describes the set of CPUs * actually populated at the present time. */ - for (i = 0; i < max_cpus; i++) - set_cpu_present(i, true); + for_each_possible_cpu(cpu) { + if (cpu < max_cpus) + set_cpu_present(cpu, true); + } } void __init smp_cpus_done(unsigned int max_cpus) diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index ecef9aff9d72..4e53ac46e857 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -61,8 +61,10 @@ config PARISC select HAVE_KRETPROBES select HAVE_DYNAMIC_FTRACE if $(cc-option,-fpatchable-function-entry=1,1) select HAVE_FTRACE_MCOUNT_RECORD if HAVE_DYNAMIC_FTRACE + select FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY if DYNAMIC_FTRACE select HAVE_KPROBES_ON_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_SOFTIRQ_ON_OWN_STACK if IRQSTACKS select SET_FS help diff --git a/arch/parisc/configs/generic-32bit_defconfig b/arch/parisc/configs/generic-32bit_defconfig index 3cbcfad5f724..7611d48c599e 100644 --- a/arch/parisc/configs/generic-32bit_defconfig +++ b/arch/parisc/configs/generic-32bit_defconfig @@ -22,7 +22,6 @@ CONFIG_PCI_LBA=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_UNUSED_SYMBOLS=y # CONFIG_BLK_DEV_BSG is not set # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m diff --git a/arch/parisc/configs/generic-64bit_defconfig b/arch/parisc/configs/generic-64bit_defconfig index 8f81fcbf04c4..53054b81461a 100644 --- a/arch/parisc/configs/generic-64bit_defconfig +++ b/arch/parisc/configs/generic-64bit_defconfig @@ -31,7 +31,6 @@ CONFIG_MODULE_FORCE_LOAD=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_MODVERSIONS=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_BLK_DEV_INTEGRITY=y CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set diff --git a/arch/parisc/include/asm/hardirq.h b/arch/parisc/include/asm/hardirq.h index fad29aa6f45f..1e4fbd0fd944 100644 --- a/arch/parisc/include/asm/hardirq.h +++ b/arch/parisc/include/asm/hardirq.h @@ -12,10 +12,6 @@ #include #include -#ifdef CONFIG_IRQSTACKS -#define __ARCH_HAS_DO_SOFTIRQ -#endif - typedef struct { unsigned int __softirq_pending; unsigned int kernel_stack_usage; diff --git a/arch/parisc/kernel/irq.c b/arch/parisc/kernel/irq.c index 1dfb439b0692..0d46b19dc4d3 100644 --- a/arch/parisc/kernel/irq.c +++ b/arch/parisc/kernel/irq.c @@ -17,6 +17,7 @@ #include #include +#include #include #include diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index fda1c1a6a444..b144fbe29bc1 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -200,7 +200,7 @@ copy_thread(unsigned long clone_flags, unsigned long usp, extern void * const ret_from_kernel_thread; extern void * const child_return; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ memset(cregs, 0, sizeof(struct pt_regs)); if (!usp) /* idle thread */ diff --git a/arch/parisc/kernel/syscalls/Makefile b/arch/parisc/kernel/syscalls/Makefile index c22a21c39f30..283f64407b07 100644 --- a/arch/parisc/kernel/syscalls/Makefile +++ b/arch/parisc/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -22,24 +22,24 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_offset_$(basetarget))' syshdr_abis_unistd_32 := common,32 -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abis_unistd_64 := common,64 -$(uapi)/unistd_64.h: $(syscall) $(syshdr) +$(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) systbl_abis_syscall_table_32 := common,32 -$(kapi)/syscall_table_32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_64 := common,64 -$(kapi)/syscall_table_64.h: $(syscall) $(systbl) +$(kapi)/syscall_table_64.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_c32 := common,32 systbl_abi_syscall_table_c32 := c32 -$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h unistd_64.h @@ -47,9 +47,10 @@ kapisyshdr-y += syscall_table_32.h \ syscall_table_64.h \ syscall_table_c32.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl index 6bcc31966b44..271a92519683 100644 --- a/arch/parisc/kernel/syscalls/syscall.tbl +++ b/arch/parisc/kernel/syscalls/syscall.tbl @@ -439,3 +439,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 9141f03060ce..386ae12d8523 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -196,7 +196,6 @@ config PPC select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13) select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2) select HAVE_CONTEXT_TRACKING if PPC64 - select HAVE_TIF_NOHZ if PPC64 select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_STACKOVERFLOW select HAVE_DYNAMIC_FTRACE @@ -236,6 +235,7 @@ config PPC select MMU_GATHER_PAGE_SIZE select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if PPC_BOOK3S_64 && CPU_LITTLE_ENDIAN + select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_SYSCALL_TRACEPOINTS select HAVE_VIRT_CPU_ACCOUNTING select HAVE_IRQ_TIME_ACCOUNTING @@ -503,18 +503,14 @@ config HOTPLUG_CPU Say N if you are unsure. config PPC_QUEUED_SPINLOCKS - bool "Queued spinlocks" + bool "Queued spinlocks" if EXPERT depends on SMP + default PPC_BOOK3S_64 help Say Y here to use queued spinlocks which give better scalability and fairness on large SMP and NUMA systems without harming single threaded performance. - This option is currently experimental, the code is more complex and - less tested so it defaults to "N" for the moment. - - If unsure, say "N". - config ARCH_CPU_PROBE_RELEASE def_bool y depends on HOTPLUG_CPU @@ -718,18 +714,6 @@ config ARCH_MEMORY_PROBE def_bool y depends on MEMORY_HOTPLUG -config STDBINUTILS - bool "Using standard binutils settings" - depends on 44x - default y - help - Turning this option off allows you to select 256KB PAGE_SIZE on 44x. - Note, that kernel will be able to run only those applications, - which had been compiled using binutils later than 2.17.50.0.3 with - '-zmax-page-size' set to 256K (the default is 64K). Or, if using - the older binutils, you can patch them with a trivial patch, which - changes the ELF_MAXPAGESIZE definition from 0x10000 to 0x40000. - choice prompt "Page size" default PPC_4K_PAGES @@ -769,17 +753,15 @@ config PPC_64K_PAGES select HAVE_ARCH_SOFT_DIRTY if PPC_BOOK3S_64 config PPC_256K_PAGES - bool "256k page size" - depends on 44x && !STDBINUTILS + bool "256k page size (Requires non-standard binutils settings)" + depends on 44x && !PPC_47x help Make the page size 256k. - As the ELF standard only requires alignment to support page - sizes up to 64k, you will need to compile all of your user - space applications with a non-standard binutils settings - (see the STDBINUTILS description for details). - - Say N unless you know what you are doing. + The kernel will only be able to run applications that have been + compiled with '-zmax-page-size' set to 256K (the default is 64K) using + binutils later than 2.17.50.0.3, or by patching the ELF_MAXPAGESIZE + definition from 0x10000 to 0x40000 in older versions. endchoice diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug index b88900f4832f..ae084357994e 100644 --- a/arch/powerpc/Kconfig.debug +++ b/arch/powerpc/Kconfig.debug @@ -88,6 +88,7 @@ config PPC_IRQ_SOFT_MASK_DEBUG config XMON bool "Include xmon kernel debugger" depends on DEBUG_KERNEL + select CONSOLE_POLL if SERIAL_CPM_CONSOLE help Include in-kernel hooks for the xmon kernel monitor/debugger. Unless you are intending to debug the kernel, say N here. diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index b959fdaec713..5f8544cf724a 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -65,7 +65,7 @@ UTS_MACHINE := $(subst $(space),,$(machine-y)) ifdef CONFIG_PPC32 KBUILD_LDFLAGS_MODULE += arch/powerpc/lib/crtsavres.o else -ifeq ($(call ld-ifversion, -ge, 225000000, y),y) +ifeq ($(call ld-ifversion, -ge, 22500, y),y) # Have the linker provide sfpr if possible. # There is a corresponding test in arch/powerpc/lib/Makefile KBUILD_LDFLAGS_MODULE += --save-restore-funcs diff --git a/arch/powerpc/configs/44x/akebono_defconfig b/arch/powerpc/configs/44x/akebono_defconfig index 72b8f93a9bdd..4bc549c6edc5 100644 --- a/arch/powerpc/configs/44x/akebono_defconfig +++ b/arch/powerpc/configs/44x/akebono_defconfig @@ -20,6 +20,7 @@ CONFIG_IRQ_ALL_CPUS=y # CONFIG_COMPACTION is not set # CONFIG_SUSPEND is not set CONFIG_NET=y +CONFIG_NETDEVICES=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y @@ -40,7 +41,9 @@ CONFIG_BLK_DEV_RAM_SIZE=35000 # CONFIG_SCSI_PROC_FS is not set CONFIG_BLK_DEV_SD=y # CONFIG_SCSI_LOWLEVEL is not set +CONFIG_ATA=y # CONFIG_SATA_PMP is not set +CONFIG_SATA_AHCI_PLATFORM=y # CONFIG_ATA_SFF is not set # CONFIG_NET_VENDOR_3COM is not set # CONFIG_NET_VENDOR_ADAPTEC is not set @@ -97,6 +100,8 @@ CONFIG_USB_OHCI_HCD=y # CONFIG_USB_OHCI_HCD_PCI is not set CONFIG_USB_STORAGE=y CONFIG_MMC=y +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_PLTFM=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_M41T80=y CONFIG_EXT2_FS=y diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig index 10c055eaebf0..6677ac0da45a 100644 --- a/arch/powerpc/configs/ppc6xx_defconfig +++ b/arch/powerpc/configs/ppc6xx_defconfig @@ -1071,7 +1071,6 @@ CONFIG_NLS_ISO8859_15=m CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_DEBUG_INFO=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_HEADERS_INSTALL=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index a6e650a97d8f..ffedea7e4bef 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -129,7 +129,7 @@ static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data, src += bytes; len -= bytes; - }; + } memcpy((char *)sctx->buf, src, len); return 0; diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h index d0b832cbbec8..939f3c94c8f3 100644 --- a/arch/powerpc/include/asm/asm-prototypes.h +++ b/arch/powerpc/include/asm/asm-prototypes.h @@ -56,35 +56,6 @@ int exit_vmx_usercopy(void); int enter_vmx_ops(void); void *exit_vmx_ops(void *dest); -/* Traps */ -long machine_check_early(struct pt_regs *regs); -long hmi_exception_realmode(struct pt_regs *regs); -void SMIException(struct pt_regs *regs); -void handle_hmi_exception(struct pt_regs *regs); -void instruction_breakpoint_exception(struct pt_regs *regs); -void RunModeException(struct pt_regs *regs); -void single_step_exception(struct pt_regs *regs); -void program_check_exception(struct pt_regs *regs); -void alignment_exception(struct pt_regs *regs); -void StackOverflow(struct pt_regs *regs); -void stack_overflow_exception(struct pt_regs *regs); -void kernel_fp_unavailable_exception(struct pt_regs *regs); -void altivec_unavailable_exception(struct pt_regs *regs); -void vsx_unavailable_exception(struct pt_regs *regs); -void fp_unavailable_tm(struct pt_regs *regs); -void altivec_unavailable_tm(struct pt_regs *regs); -void vsx_unavailable_tm(struct pt_regs *regs); -void facility_unavailable_exception(struct pt_regs *regs); -void TAUException(struct pt_regs *regs); -void altivec_assist_exception(struct pt_regs *regs); -void unrecoverable_exception(struct pt_regs *regs); -void kernel_bad_stack(struct pt_regs *regs); -void system_reset_exception(struct pt_regs *regs); -void machine_check_exception(struct pt_regs *regs); -void emulation_assist_interrupt(struct pt_regs *regs); -long do_slb_fault(struct pt_regs *regs, unsigned long ea); -void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err); - /* signals, syscalls and interrupts */ long sys_swapcontext(struct ucontext __user *old_ctx, struct ucontext __user *new_ctx, diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h index a0117a9d5b06..73bc5d2c431d 100644 --- a/arch/powerpc/include/asm/book3s/32/kup.h +++ b/arch/powerpc/include/asm/book3s/32/kup.h @@ -95,12 +95,12 @@ static inline void kuap_update_sr(u32 sr, u32 addr, u32 end) addr &= 0xf0000000; /* align addr to start of segment */ barrier(); /* make sure thread.kuap is updated before playing with SRs */ while (addr < end) { - mtsrin(sr, addr); + mtsr(sr, addr); sr += 0x111; /* next VSID */ sr &= 0xf0ffffff; /* clear VSID overflow */ addr += 0x10000000; /* address of next segment */ } - isync(); /* Context sync required after mtsrin() */ + isync(); /* Context sync required after mtsr() */ } static __always_inline void allow_user_access(void __user *to, const void __user *from, @@ -122,7 +122,7 @@ static __always_inline void allow_user_access(void __user *to, const void __user end = min(addr + size, TASK_SIZE); current->thread.kuap = (addr & 0xf0000000) | ((((end - 1) >> 28) + 1) & 0xf); - kuap_update_sr(mfsrin(addr) & ~SR_KS, addr, end); /* Clear Ks */ + kuap_update_sr(mfsr(addr) & ~SR_KS, addr, end); /* Clear Ks */ } static __always_inline void prevent_user_access(void __user *to, const void __user *from, @@ -151,7 +151,7 @@ static __always_inline void prevent_user_access(void __user *to, const void __us } current->thread.kuap = 0; - kuap_update_sr(mfsrin(addr) | SR_KS, addr, end); /* set Ks */ + kuap_update_sr(mfsr(addr) | SR_KS, addr, end); /* set Ks */ } static inline unsigned long prevent_user_access_return(void) diff --git a/arch/powerpc/include/asm/book3s/32/mmu-hash.h b/arch/powerpc/include/asm/book3s/32/mmu-hash.h index 685c589e723f..b85f8e114a9c 100644 --- a/arch/powerpc/include/asm/book3s/32/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/32/mmu-hash.h @@ -94,7 +94,7 @@ typedef struct { } mm_context_t; void update_bats(void); -static inline void cleanup_cpu_mmu_context(void) { }; +static inline void cleanup_cpu_mmu_context(void) { } /* patch sites */ extern s32 patch__hash_page_A0, patch__hash_page_A1, patch__hash_page_A2; diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h index 7d1ef7b9754e..8bd905050896 100644 --- a/arch/powerpc/include/asm/book3s/64/kup.h +++ b/arch/powerpc/include/asm/book3s/64/kup.h @@ -339,7 +339,7 @@ static inline unsigned long get_kuap(void) * This has no effect in terms of actually blocking things on hash, * so it doesn't break anything. */ - if (!early_mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) + if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) return AMR_KUAP_BLOCKED; return mfspr(SPRN_AMR); @@ -347,7 +347,7 @@ static inline unsigned long get_kuap(void) static inline void set_kuap(unsigned long value) { - if (!early_mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) + if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) return; /* diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h index 066b1d34c7bc..f911bdb68d8b 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -454,6 +454,8 @@ static inline unsigned long hpt_hash(unsigned long vpn, #define HPTE_NOHPTE_UPDATE 0x2 #define HPTE_USE_KERNEL_KEY 0x4 +long hpte_insert_repeating(unsigned long hash, unsigned long vpn, unsigned long pa, + unsigned long rlags, unsigned long vflags, int psize, int ssize); extern int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize, int subpage_prot); @@ -467,6 +469,8 @@ extern int hash_page_mm(struct mm_struct *mm, unsigned long ea, unsigned long flags); extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap, unsigned long dsisr); +void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc); +int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, unsigned long msr); int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize, unsigned int shift, unsigned int mmu_psize); @@ -521,6 +525,7 @@ void slb_dump_contents(struct slb_entry *slb_ptr); extern void slb_vmalloc_update(void); extern void slb_set_size(u16 size); +void preload_new_slb_context(unsigned long start, unsigned long sp); #endif /* __ASSEMBLY__ */ /* diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h index 995bbcdd0ef8..eace8c3f7b0a 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu.h +++ b/arch/powerpc/include/asm/book3s/64/mmu.h @@ -239,7 +239,7 @@ static inline void setup_initial_memory_limit(phys_addr_t first_memblock_base, #ifdef CONFIG_PPC_PSERIES extern void radix_init_pseries(void); #else -static inline void radix_init_pseries(void) { }; +static inline void radix_init_pseries(void) { } #endif #ifdef CONFIG_HOTPLUG_CPU diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index a39886681629..058601efbc8a 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -388,11 +388,28 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG #define ptep_test_and_clear_young(__vma, __addr, __ptep) \ ({ \ - int __r; \ - __r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \ - __r; \ + __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \ }) +/* + * On Book3S CPUs, clearing the accessed bit without a TLB flush + * doesn't cause data corruption. [ It could cause incorrect + * page aging and the (mistaken) reclaim of hot pages, but the + * chance of that should be relatively low. ] + * + * So as a performance optimization don't flush the TLB when + * clearing the accessed bit, it will eventually be flushed by + * a context switch or a VM operation anyway. [ In the rare + * event of it not getting flushed for a long time the delay + * shouldn't really matter because there's no real memory + * pressure for swapout to react to. ] + */ +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +#define ptep_clear_flush_young ptep_test_and_clear_young + +#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH +#define pmdp_clear_flush_young pmdp_test_and_clear_young + static inline int __pte_write(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE)); diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index 94439e0cefc9..8b33601cdb9d 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -35,7 +35,7 @@ extern void radix__flush_pwc_lpid(unsigned int lpid); extern void radix__flush_all_lpid(unsigned int lpid); extern void radix__flush_all_lpid_guest(unsigned int lpid); #else -static inline void radix__tlbiel_all(unsigned int action) { WARN_ON(1); }; +static inline void radix__tlbiel_all(unsigned int action) { WARN_ON(1); } static inline void radix__flush_tlb_lpid_page(unsigned int lpid, unsigned long addr, unsigned long page_size) diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h index dcb5c3839d2f..215973b4cb26 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -31,7 +31,7 @@ static inline void tlbiel_all(void) hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); } #else -static inline void tlbiel_all(void) { BUG(); }; +static inline void tlbiel_all(void) { BUG(); } #endif static inline void tlbiel_all_lpid(bool radix) diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h index 464f8ca8a5c9..d1635ffbb179 100644 --- a/arch/powerpc/include/asm/bug.h +++ b/arch/powerpc/include/asm/bug.h @@ -111,12 +111,15 @@ #ifndef __ASSEMBLY__ struct pt_regs; -extern int do_page_fault(struct pt_regs *, unsigned long, unsigned long); -extern void bad_page_fault(struct pt_regs *, unsigned long, int); -void __bad_page_fault(struct pt_regs *regs, unsigned long address, int sig); +long do_page_fault(struct pt_regs *); +long hash__do_page_fault(struct pt_regs *); +void bad_page_fault(struct pt_regs *, int); +void __bad_page_fault(struct pt_regs *regs, int sig); +void do_bad_page_fault_segv(struct pt_regs *regs); extern void _exception(int, struct pt_regs *, int, unsigned long); extern void _exception_pkey(struct pt_regs *, unsigned long, int); extern void die(const char *, struct pt_regs *, long); +void die_mce(const char *str, struct pt_regs *regs, long err); extern bool die_will_crash(void); extern void panic_flush_kmsg_start(void); extern void panic_flush_kmsg_end(void); diff --git a/arch/powerpc/include/asm/cacheflush.h b/arch/powerpc/include/asm/cacheflush.h index 138e46d8c04e..f63495109f63 100644 --- a/arch/powerpc/include/asm/cacheflush.h +++ b/arch/powerpc/include/asm/cacheflush.h @@ -8,6 +8,12 @@ #include #include +/* + * This flag is used to indicate that the page pointed to by a pte is clean + * and does not require cleaning before returning it to the user. + */ +#define PG_dcache_clean PG_arch_1 + #ifdef CONFIG_PPC_BOOK3S_64 /* * Book3s has no ptesync after setting a pte, so without this ptesync it's diff --git a/arch/powerpc/include/asm/cputime.h b/arch/powerpc/include/asm/cputime.h index ed75d1c318e3..504f7fe6711a 100644 --- a/arch/powerpc/include/asm/cputime.h +++ b/arch/powerpc/include/asm/cputime.h @@ -87,6 +87,17 @@ static notrace inline void account_cpu_user_exit(void) acct->starttime_user = tb; } +static notrace inline void account_stolen_time(void) +{ +#ifdef CONFIG_PPC_SPLPAR + if (firmware_has_feature(FW_FEATURE_SPLPAR)) { + struct lppaca *lp = local_paca->lppaca_ptr; + + if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx))) + accumulate_stolen_time(); + } +#endif +} #endif /* __KERNEL__ */ #else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ @@ -96,5 +107,8 @@ static inline void account_cpu_user_entry(void) static inline void account_cpu_user_exit(void) { } +static notrace inline void account_stolen_time(void) +{ +} #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ #endif /* __POWERPC_CPUTIME_H */ diff --git a/arch/powerpc/include/asm/debug.h b/arch/powerpc/include/asm/debug.h index ec57daf87f40..86a14736c76c 100644 --- a/arch/powerpc/include/asm/debug.h +++ b/arch/powerpc/include/asm/debug.h @@ -50,10 +50,6 @@ bool ppc_breakpoint_available(void); #ifdef CONFIG_PPC_ADV_DEBUG_REGS extern void do_send_trap(struct pt_regs *regs, unsigned long address, unsigned long error_code, int brkpt); -#else - -extern void do_break(struct pt_regs *regs, unsigned long address, - unsigned long error_code); #endif #endif /* _ASM_POWERPC_DEBUG_H */ diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h index aa6a5ef5d483..7604673787d6 100644 --- a/arch/powerpc/include/asm/firmware.h +++ b/arch/powerpc/include/asm/firmware.h @@ -137,7 +137,7 @@ extern unsigned int __start___fw_ftr_fixup, __stop___fw_ftr_fixup; #ifdef CONFIG_PPC_PSERIES void pseries_probe_fw_features(void); #else -static inline void pseries_probe_fw_features(void) { }; +static inline void pseries_probe_fw_features(void) { } #endif #endif /* __ASSEMBLY__ */ diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h index 013165e62618..f18c543bc01d 100644 --- a/arch/powerpc/include/asm/hugetlb.h +++ b/arch/powerpc/include/asm/hugetlb.h @@ -17,8 +17,6 @@ extern bool hugetlb_disabled; void hugetlbpage_init_default(void); -void flush_dcache_icache_hugepage(struct page *page); - int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, unsigned long len); diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index 0363734ff56e..56a98936a6a9 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -38,6 +38,8 @@ #define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE) #endif +#endif /* CONFIG_PPC64 */ + /* * flags for paca->irq_soft_mask */ @@ -46,18 +48,56 @@ #define IRQS_PMI_DISABLED 2 #define IRQS_ALL_DISABLED (IRQS_DISABLED | IRQS_PMI_DISABLED) -#endif /* CONFIG_PPC64 */ - #ifndef __ASSEMBLY__ -extern void replay_system_reset(void); -extern void replay_soft_interrupts(void); +static inline void __hard_irq_enable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x)) + wrtee(MSR_EE); + else if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_EIE); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(MSR_EE | MSR_RI, 1); + else + mtmsr(mfmsr() | MSR_EE); +} -extern void timer_interrupt(struct pt_regs *); -extern void timer_broadcast_interrupt(void); -extern void performance_monitor_exception(struct pt_regs *regs); -extern void WatchdogException(struct pt_regs *regs); -extern void unknown_exception(struct pt_regs *regs); +static inline void __hard_irq_disable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x)) + wrtee(0); + else if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_EID); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(MSR_RI, 1); + else + mtmsr(mfmsr() & ~MSR_EE); +} + +static inline void __hard_EE_RI_disable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x)) + wrtee(0); + else if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_NRI); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(0, 1); + else + mtmsr(mfmsr() & ~(MSR_EE | MSR_RI)); +} + +static inline void __hard_RI_enable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x)) + return; + + if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_EID); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(MSR_RI, 1); + else + mtmsr(mfmsr() | MSR_RI); +} #ifdef CONFIG_PPC64 #include @@ -221,18 +261,6 @@ static inline bool arch_irqs_disabled(void) #endif /* CONFIG_PPC_BOOK3S */ -#ifdef CONFIG_PPC_BOOK3E -#define __hard_irq_enable() wrtee(MSR_EE) -#define __hard_irq_disable() wrtee(0) -#define __hard_EE_RI_disable() wrtee(0) -#define __hard_RI_enable() do { } while (0) -#else -#define __hard_irq_enable() __mtmsrd(MSR_EE|MSR_RI, 1) -#define __hard_irq_disable() __mtmsrd(MSR_RI, 1) -#define __hard_EE_RI_disable() __mtmsrd(0, 1) -#define __hard_RI_enable() __mtmsrd(MSR_RI, 1) -#endif - #define hard_irq_disable() do { \ unsigned long flags; \ __hard_irq_disable(); \ @@ -296,8 +324,17 @@ extern void irq_set_pending_from_srr1(unsigned long srr1); extern void force_external_irq_replay(void); +static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val) +{ + regs->softe = val; +} #else /* CONFIG_PPC64 */ +static inline notrace unsigned long irq_soft_mask_return(void) +{ + return 0; +} + static inline unsigned long arch_local_save_flags(void) { return mfmsr(); @@ -327,22 +364,12 @@ static inline unsigned long arch_local_irq_save(void) static inline void arch_local_irq_disable(void) { - if (IS_ENABLED(CONFIG_BOOKE)) - wrtee(0); - else if (IS_ENABLED(CONFIG_PPC_8xx)) - wrtspr(SPRN_EID); - else - mtmsr(mfmsr() & ~MSR_EE); + __hard_irq_disable(); } static inline void arch_local_irq_enable(void) { - if (IS_ENABLED(CONFIG_BOOKE)) - wrtee(MSR_EE); - else if (IS_ENABLED(CONFIG_PPC_8xx)) - wrtspr(SPRN_EIE); - else - mtmsr(mfmsr() | MSR_EE); + __hard_irq_enable(); } static inline bool arch_irqs_disabled_flags(unsigned long flags) @@ -364,6 +391,9 @@ static inline bool arch_irq_disabled_regs(struct pt_regs *regs) static inline void may_hard_irq_enable(void) { } +static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val) +{ +} #endif /* CONFIG_PPC64 */ #define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h new file mode 100644 index 000000000000..aedfba29e43a --- /dev/null +++ b/arch/powerpc/include/asm/interrupt.h @@ -0,0 +1,449 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_INTERRUPT_H +#define _ASM_POWERPC_INTERRUPT_H + +#include +#include +#include +#include +#include +#include + +struct interrupt_state { +#ifdef CONFIG_PPC_BOOK3E_64 + enum ctx_state ctx_state; +#endif +}; + +static inline void booke_restore_dbcr0(void) +{ +#ifdef CONFIG_PPC_ADV_DEBUG_REGS + unsigned long dbcr0 = current->thread.debug.dbcr0; + + if (IS_ENABLED(CONFIG_PPC32) && unlikely(dbcr0 & DBCR0_IDM)) { + mtspr(SPRN_DBSR, -1); + mtspr(SPRN_DBCR0, global_dbcr0[smp_processor_id()]); + } +#endif +} + +static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrupt_state *state) +{ + /* + * Book3E reconciles irq soft mask in asm + */ +#ifdef CONFIG_PPC_BOOK3S_64 + if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED) + trace_hardirqs_off(); + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; + + if (user_mode(regs)) { + CT_WARN_ON(ct_state() != CONTEXT_USER); + user_exit_irqoff(); + + account_cpu_user_entry(); + account_stolen_time(); + } else { + /* + * CT_WARN_ON comes here via program_check_exception, + * so avoid recursion. + */ + if (TRAP(regs) != 0x700) + CT_WARN_ON(ct_state() != CONTEXT_KERNEL); + } +#endif + +#ifdef CONFIG_PPC_BOOK3E_64 + state->ctx_state = exception_enter(); + if (user_mode(regs)) + account_cpu_user_entry(); +#endif +} + +/* + * Care should be taken to note that interrupt_exit_prepare and + * interrupt_async_exit_prepare do not necessarily return immediately to + * regs context (e.g., if regs is usermode, we don't necessarily return to + * user mode). Other interrupts might be taken between here and return, + * context switch / preemption may occur in the exit path after this, or a + * signal may be delivered, etc. + * + * The real interrupt exit code is platform specific, e.g., + * interrupt_exit_user_prepare / interrupt_exit_kernel_prepare for 64s. + * + * However interrupt_nmi_exit_prepare does return directly to regs, because + * NMIs do not do "exit work" or replay soft-masked interrupts. + */ +static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt_state *state) +{ +#ifdef CONFIG_PPC_BOOK3E_64 + exception_exit(state->ctx_state); +#endif + + /* + * Book3S exits to user via interrupt_exit_user_prepare(), which does + * context tracking, which is a cleaner way to handle PREEMPT=y + * and avoid context entry/exit in e.g., preempt_schedule_irq()), + * which is likely to be where the core code wants to end up. + * + * The above comment explains why we can't do the + * + * if (user_mode(regs)) + * user_exit_irqoff(); + * + * sequence here. + */ +} + +static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct interrupt_state *state) +{ +#ifdef CONFIG_PPC_BOOK3S_64 + if (cpu_has_feature(CPU_FTR_CTRL) && + !test_thread_local_flags(_TLF_RUNLATCH)) + __ppc64_runlatch_on(); +#endif + + interrupt_enter_prepare(regs, state); + irq_enter(); +} + +static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct interrupt_state *state) +{ + irq_exit(); + interrupt_exit_prepare(regs, state); +} + +struct interrupt_nmi_state { +#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3S_64 + u8 irq_soft_mask; + u8 irq_happened; +#endif + u8 ftrace_enabled; +#endif +}; + +static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) +{ +#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3S_64 + state->irq_soft_mask = local_paca->irq_soft_mask; + state->irq_happened = local_paca->irq_happened; + + /* + * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does + * the right thing, and set IRQ_HARD_DIS. We do not want to reconcile + * because that goes through irq tracing which we don't want in NMI. + */ + local_paca->irq_soft_mask = IRQS_ALL_DISABLED; + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; + + /* Don't do any per-CPU operations until interrupt state is fixed */ +#endif + /* Allow DEC and PMI to be traced when they are soft-NMI */ + if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260) { + state->ftrace_enabled = this_cpu_get_ftrace_enabled(); + this_cpu_set_ftrace_enabled(0); + } +#endif + + /* + * Do not use nmi_enter() for pseries hash guest taking a real-mode + * NMI because not everything it touches is within the RMA limit. + */ + if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || + !firmware_has_feature(FW_FEATURE_LPAR) || + radix_enabled() || (mfmsr() & MSR_DR)) + nmi_enter(); +} + +static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) +{ + if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || + !firmware_has_feature(FW_FEATURE_LPAR) || + radix_enabled() || (mfmsr() & MSR_DR)) + nmi_exit(); + +#ifdef CONFIG_PPC64 + if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260) + this_cpu_set_ftrace_enabled(state->ftrace_enabled); + +#ifdef CONFIG_PPC_BOOK3S_64 + /* Check we didn't change the pending interrupt mask. */ + WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened); + local_paca->irq_happened = state->irq_happened; + local_paca->irq_soft_mask = state->irq_soft_mask; +#endif +#endif +} + +/* + * Don't use noinstr here like x86, but rather add NOKPROBE_SYMBOL to each + * function definition. The reason for this is the noinstr section is placed + * after the main text section, i.e., very far away from the interrupt entry + * asm. That creates problems with fitting linker stubs when building large + * kernels. + */ +#define interrupt_handler __visible noinline notrace __no_kcsan __no_sanitize_address + +/** + * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + */ +#define DECLARE_INTERRUPT_HANDLER_RAW(func) \ + __visible long func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_RAW - Define raw interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + * + * @func is called from ASM entry code. + * + * This is a plain function which does no tracing, reconciling, etc. + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + * + * raw interrupt handlers must not enable or disable interrupts, or + * schedule, tracing and instrumentation (ftrace, lockdep, etc) would + * not be advisable either, although may be possible in a pinch, the + * trace will look odd at least. + * + * A raw handler may call one of the other interrupt handler functions + * to be converted into that interrupt context without these restrictions. + * + * On PPC64, _RAW handlers may return with fast_interrupt_return. + * + * Specific handlers may have additional restrictions. + */ +#define DEFINE_INTERRUPT_HANDLER_RAW(func) \ +static __always_inline long ____##func(struct pt_regs *regs); \ + \ +interrupt_handler long func(struct pt_regs *regs) \ +{ \ + long ret; \ + \ + ret = ____##func (regs); \ + \ + return ret; \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline long ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function + * @func: Function name of the entry point + */ +#define DECLARE_INTERRUPT_HANDLER(func) \ + __visible void func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER - Define synchronous interrupt handler function + * @func: Function name of the entry point + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER(func) \ +static __always_inline void ____##func(struct pt_regs *regs); \ + \ +interrupt_handler void func(struct pt_regs *regs) \ +{ \ + struct interrupt_state state; \ + \ + interrupt_enter_prepare(regs, &state); \ + \ + ____##func (regs); \ + \ + interrupt_exit_prepare(regs, &state); \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline void ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER_RET - Declare synchronous interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + */ +#define DECLARE_INTERRUPT_HANDLER_RET(func) \ + __visible long func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_RET - Define synchronous interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER_RET(func) \ +static __always_inline long ____##func(struct pt_regs *regs); \ + \ +interrupt_handler long func(struct pt_regs *regs) \ +{ \ + struct interrupt_state state; \ + long ret; \ + \ + interrupt_enter_prepare(regs, &state); \ + \ + ret = ____##func (regs); \ + \ + interrupt_exit_prepare(regs, &state); \ + \ + return ret; \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline long ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER_ASYNC - Declare asynchronous interrupt handler function + * @func: Function name of the entry point + */ +#define DECLARE_INTERRUPT_HANDLER_ASYNC(func) \ + __visible void func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_ASYNC - Define asynchronous interrupt handler function + * @func: Function name of the entry point + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER_ASYNC(func) \ +static __always_inline void ____##func(struct pt_regs *regs); \ + \ +interrupt_handler void func(struct pt_regs *regs) \ +{ \ + struct interrupt_state state; \ + \ + interrupt_async_enter_prepare(regs, &state); \ + \ + ____##func (regs); \ + \ + interrupt_async_exit_prepare(regs, &state); \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline void ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER_NMI - Declare NMI interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + */ +#define DECLARE_INTERRUPT_HANDLER_NMI(func) \ + __visible long func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_NMI - Define NMI interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER_NMI(func) \ +static __always_inline long ____##func(struct pt_regs *regs); \ + \ +interrupt_handler long func(struct pt_regs *regs) \ +{ \ + struct interrupt_nmi_state state; \ + long ret; \ + \ + interrupt_nmi_enter_prepare(regs, &state); \ + \ + ret = ____##func (regs); \ + \ + interrupt_nmi_exit_prepare(regs, &state); \ + \ + return ret; \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline long ____##func(struct pt_regs *regs) + + +/* Interrupt handlers */ +/* kernel/traps.c */ +DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception); +#ifdef CONFIG_PPC_BOOK3S_64 +DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception); +#else +DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception); +#endif +DECLARE_INTERRUPT_HANDLER(SMIException); +DECLARE_INTERRUPT_HANDLER(handle_hmi_exception); +DECLARE_INTERRUPT_HANDLER(unknown_exception); +DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception); +DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception); +DECLARE_INTERRUPT_HANDLER(RunModeException); +DECLARE_INTERRUPT_HANDLER(single_step_exception); +DECLARE_INTERRUPT_HANDLER(program_check_exception); +DECLARE_INTERRUPT_HANDLER(emulation_assist_interrupt); +DECLARE_INTERRUPT_HANDLER(alignment_exception); +DECLARE_INTERRUPT_HANDLER(StackOverflow); +DECLARE_INTERRUPT_HANDLER(stack_overflow_exception); +DECLARE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(altivec_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(vsx_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(facility_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(fp_unavailable_tm); +DECLARE_INTERRUPT_HANDLER(altivec_unavailable_tm); +DECLARE_INTERRUPT_HANDLER(vsx_unavailable_tm); +DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi); +DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async); +DECLARE_INTERRUPT_HANDLER_RAW(performance_monitor_exception); +DECLARE_INTERRUPT_HANDLER(DebugException); +DECLARE_INTERRUPT_HANDLER(altivec_assist_exception); +DECLARE_INTERRUPT_HANDLER(CacheLockingException); +DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException); +DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException); +DECLARE_INTERRUPT_HANDLER(unrecoverable_exception); +DECLARE_INTERRUPT_HANDLER(WatchdogException); +DECLARE_INTERRUPT_HANDLER(kernel_bad_stack); + +/* slb.c */ +DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault); +DECLARE_INTERRUPT_HANDLER(do_bad_slb_fault); + +/* hash_utils.c */ +DECLARE_INTERRUPT_HANDLER_RAW(do_hash_fault); + +/* fault.c */ +DECLARE_INTERRUPT_HANDLER_RET(do_page_fault); +DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv); + +/* process.c */ +DECLARE_INTERRUPT_HANDLER(do_break); + +/* time.c */ +DECLARE_INTERRUPT_HANDLER_ASYNC(timer_interrupt); + +/* mce.c */ +DECLARE_INTERRUPT_HANDLER_NMI(machine_check_early); +DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode); + +DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException); + +void replay_system_reset(void); +void replay_soft_interrupts(void); + +static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs) +{ + if (!arch_irq_disabled_regs(regs)) + local_irq_enable(); +} + +#endif /* _ASM_POWERPC_INTERRUPT_H */ diff --git a/arch/powerpc/include/asm/irq.h b/arch/powerpc/include/asm/irq.h index 4f983ca4030a..f3f264e441a7 100644 --- a/arch/powerpc/include/asm/irq.h +++ b/arch/powerpc/include/asm/irq.h @@ -37,8 +37,6 @@ extern int distribute_irqs; struct pt_regs; -#define __ARCH_HAS_DO_SOFTIRQ - #if defined(CONFIG_BOOKE) || defined(CONFIG_40x) /* * Per-cpu stacks for handling critical, debug and machine check diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index 55d6ede30c19..9ab344d29a54 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -136,6 +136,7 @@ int load_crashdump_segments_ppc64(struct kimage *image, int setup_purgatory_ppc64(struct kimage *image, const void *slave_code, const void *fdt, unsigned long kernel_load_addr, unsigned long fdt_load_addr); +unsigned int kexec_fdt_totalsize_ppc64(struct kimage *image); int setup_new_fdt_ppc64(const struct kimage *image, void *fdt, unsigned long initrd_load_addr, unsigned long initrd_len, const char *cmdline); diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h index bf221a2a523e..7ec21af49a45 100644 --- a/arch/powerpc/include/asm/kup.h +++ b/arch/powerpc/include/asm/kup.h @@ -91,6 +91,7 @@ static __always_inline void setup_kup(void) static inline void allow_read_from_user(const void __user *from, unsigned long size) { + barrier_nospec(); allow_user_access(NULL, from, size, KUAP_READ); } @@ -102,6 +103,7 @@ static inline void allow_write_to_user(void __user *to, unsigned long size) static inline void allow_read_write_user(void __user *to, const void __user *from, unsigned long size) { + barrier_nospec(); allow_user_access(to, from, size, KUAP_READ_WRITE); } diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index d32ec9ae73bd..2f5f919f6cd3 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -277,6 +277,13 @@ extern int kvmppc_hcall_impl_hv_realmode(unsigned long cmd); extern void kvmppc_copy_to_svcpu(struct kvm_vcpu *vcpu); extern void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu); +long kvmppc_read_intr(void); +void kvmppc_bad_interrupt(struct pt_regs *regs); +void kvmhv_p9_set_lpcr(struct kvm_split_mode *sip); +void kvmhv_p9_restore_lpcr(struct kvm_split_mode *sip); +void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr); +void kvmppc_inject_interrupt_hv(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags); + #ifdef CONFIG_PPC_TRANSACTIONAL_MEM void kvmppc_save_tm_pr(struct kvm_vcpu *vcpu); void kvmppc_restore_tm_pr(struct kvm_vcpu *vcpu); diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index df4bda867bab..8aacd76bb702 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -629,9 +629,9 @@ extern int h_ipi_redirect; static inline struct kvmppc_passthru_irqmap *kvmppc_get_passthru_irqmap( struct kvm *kvm) { return NULL; } -static inline void kvmppc_alloc_host_rm_ops(void) {}; -static inline void kvmppc_free_host_rm_ops(void) {}; -static inline void kvmppc_free_pimap(struct kvm *kvm) {}; +static inline void kvmppc_alloc_host_rm_ops(void) {} +static inline void kvmppc_free_host_rm_ops(void) {} +static inline void kvmppc_free_pimap(struct kvm *kvm) {} static inline int kvmppc_xics_rm_complete(struct kvm_vcpu *vcpu, u32 hcall) { return 0; } static inline int kvmppc_xics_enabled(struct kvm_vcpu *vcpu) @@ -883,9 +883,9 @@ static inline void kvmppc_mmu_flush_icache(kvm_pfn_t pfn) /* Clear i-cache for new pages */ page = pfn_to_page(pfn); - if (!test_bit(PG_arch_1, &page->flags)) { + if (!test_bit(PG_dcache_clean, &page->flags)) { flush_dcache_icache_page(page); - set_bit(PG_arch_1, &page->flags); + set_bit(PG_dcache_clean, &page->flags); } } diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index cf6ebbc16cb4..764f2732a821 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -59,6 +59,9 @@ struct machdep_calls { int (*pcibios_root_bridge_prepare)(struct pci_host_bridge *bridge); + /* finds all the pci_controllers present at boot */ + void (*discover_phbs)(void); + /* To setup PHBs when using automatic OF platform driver for PCI */ int (*pci_setup_phb)(struct pci_controller *host); diff --git a/arch/powerpc/include/asm/mce.h b/arch/powerpc/include/asm/mce.h index e6c27ae843dc..331d944280b8 100644 --- a/arch/powerpc/include/asm/mce.h +++ b/arch/powerpc/include/asm/mce.h @@ -204,7 +204,18 @@ struct mce_error_info { bool ignore_event; }; -#define MAX_MC_EVT 100 +#define MAX_MC_EVT 10 + +struct mce_info { + int mce_nest_count; + struct machine_check_event mce_event[MAX_MC_EVT]; + /* Queue for delayed MCE events. */ + int mce_queue_count; + struct machine_check_event mce_event_queue[MAX_MC_EVT]; + /* Queue for delayed MCE UE events. */ + int mce_ue_count; + struct machine_check_event mce_ue_event_queue[MAX_MC_EVT]; +}; /* Release flags for get_mce_event() */ #define MCE_EVENT_RELEASE true @@ -234,4 +245,11 @@ long __machine_check_early_realmode_p8(struct pt_regs *regs); long __machine_check_early_realmode_p9(struct pt_regs *regs); long __machine_check_early_realmode_p10(struct pt_regs *regs); #endif /* CONFIG_PPC_BOOK3S_64 */ + +#ifdef CONFIG_PPC_BOOK3S_64 +void mce_init(void); +#else +static inline void mce_init(void) { }; +#endif /* CONFIG_PPC_BOOK3S_64 */ + #endif /* __ASM_PPC64_MCE_H__ */ diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index d5821834dba9..652ce85f9410 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -282,9 +282,6 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, } #define pkey_mm_init(mm) -#define thread_pkey_regs_save(thread) -#define thread_pkey_regs_restore(new_thread, old_thread) -#define thread_pkey_regs_init(thread) #define arch_dup_pkeys(oldmm, mm) static inline u64 pte_to_hpte_pkey_bits(u64 pteflags, unsigned long flags) diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h index 84b4cfe73edd..160abcb8e9fa 100644 --- a/arch/powerpc/include/asm/nmi.h +++ b/arch/powerpc/include/asm/nmi.h @@ -4,6 +4,7 @@ #ifdef CONFIG_PPC_WATCHDOG extern void arch_touch_nmi_watchdog(void); +long soft_nmi_interrupt(struct pt_regs *regs); #else static inline void arch_touch_nmi_watchdog(void) {} #endif diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index 9454d29ff4b4..ec18ac818e3a 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -29,6 +29,7 @@ #include #include #include +#include #include @@ -108,8 +109,7 @@ struct paca_struct { */ /* used for most interrupts/exceptions */ u64 exgen[EX_SIZE] __attribute__((aligned(0x80))); - u64 exslb[EX_SIZE]; /* used for SLB/segment table misses - * on the linear mapping */ + /* SLB related definitions */ u16 vmalloc_sllp; u8 slb_cache_ptr; @@ -273,6 +273,9 @@ struct paca_struct { #ifdef CONFIG_MMIOWB struct mmiowb_state mmiowb_state; #endif +#ifdef CONFIG_PPC_BOOK3S_64 + struct mce_info *mce_info; +#endif /* CONFIG_PPC_BOOK3S_64 */ } ____cacheline_aligned; extern void copy_mm_to_paca(struct mm_struct *mm); @@ -285,9 +288,9 @@ extern void free_unused_pacas(void); #else /* CONFIG_PPC64 */ -static inline void allocate_paca_ptrs(void) { }; -static inline void allocate_paca(int cpu) { }; -static inline void free_unused_pacas(void) { }; +static inline void allocate_paca_ptrs(void) { } +static inline void allocate_paca(int cpu) { } +static inline void free_unused_pacas(void) { } #endif /* CONFIG_PPC64 */ diff --git a/arch/powerpc/include/asm/paravirt.h b/arch/powerpc/include/asm/paravirt.h index edc08f04aef7..5d1726bb28e7 100644 --- a/arch/powerpc/include/asm/paravirt.h +++ b/arch/powerpc/include/asm/paravirt.h @@ -10,6 +10,7 @@ #endif #ifdef CONFIG_PPC_SPLPAR +#include #include #include diff --git a/arch/powerpc/include/asm/perf_event.h b/arch/powerpc/include/asm/perf_event.h index daec64d41b44..164e910bf654 100644 --- a/arch/powerpc/include/asm/perf_event.h +++ b/arch/powerpc/include/asm/perf_event.h @@ -14,6 +14,7 @@ #include #else static inline bool is_sier_available(void) { return false; } +static inline unsigned long get_pmcs_ext_regs(int idx) { return 0; } #endif #ifdef CONFIG_FSL_EMB_PERF_EVENT @@ -40,6 +41,7 @@ static inline bool is_sier_available(void) { return false; } /* To support perf_regs sier update */ extern bool is_sier_available(void); +extern unsigned long get_pmcs_ext_regs(int idx); /* To define perf extended regs mask value */ extern u64 PERF_REG_EXTENDED_MASK; #define PERF_REG_EXTENDED_MASK PERF_REG_EXTENDED_MASK diff --git a/arch/powerpc/include/asm/perf_event_server.h b/arch/powerpc/include/asm/perf_event_server.h index 3b7baba01c92..00e7e671bb4b 100644 --- a/arch/powerpc/include/asm/perf_event_server.h +++ b/arch/powerpc/include/asm/perf_event_server.h @@ -36,9 +36,9 @@ struct power_pmu { unsigned long test_adder; int (*compute_mmcr)(u64 events[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]); + struct perf_event *pevents[], u32 flags); int (*get_constraint)(u64 event_id, unsigned long *mskp, - unsigned long *valp); + unsigned long *valp, u64 event_config1); int (*get_alternatives)(u64 event_id, unsigned int flags, u64 alt[]); void (*get_mem_data_src)(union perf_mem_data_src *dsrc, @@ -83,6 +83,7 @@ struct power_pmu { #define PPMU_NO_SIAR 0x00000100 /* Do not use SIAR */ #define PPMU_ARCH_31 0x00000200 /* Has MMCR3, SIER2 and SIER3 */ #define PPMU_P10_DD1 0x00000400 /* Is power10 DD1 processor version */ +#define PPMU_HAS_ATTR_CONFIG1 0x00000800 /* Using config1 attribute */ /* * Values for flags to get_alternatives() diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index f7613f43c9cf..4eed82172e33 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -162,6 +162,9 @@ static inline bool is_ioremap_addr(const void *x) return addr >= IOREMAP_BASE && addr < IOREMAP_END; } + +struct seq_file; +void arch_report_meminfo(struct seq_file *m); #endif /* CONFIG_PPC64 */ #endif /* __ASSEMBLY__ */ diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index a7951049e129..59a2c7dbc78f 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -169,10 +169,4 @@ static inline bool arch_pkeys_enabled(void) } extern void pkey_mm_init(struct mm_struct *mm); -extern bool arch_supports_pkeys(int cap); -extern unsigned int arch_usable_pkeys(void); -extern void thread_pkey_regs_save(struct thread_struct *thread); -extern void thread_pkey_regs_restore(struct thread_struct *new_thread, - struct thread_struct *old_thread); -extern void thread_pkey_regs_init(struct thread_struct *thread); #endif /*_ASM_POWERPC_KEYS_H */ diff --git a/arch/powerpc/include/asm/ppc-pci.h b/arch/powerpc/include/asm/ppc-pci.h index 7f4be5a05eb3..2b9edbf6e929 100644 --- a/arch/powerpc/include/asm/ppc-pci.h +++ b/arch/powerpc/include/asm/ppc-pci.h @@ -13,10 +13,6 @@ extern unsigned long isa_io_base; -extern void pci_setup_phb_io(struct pci_controller *hose, int primary); -extern void pci_setup_phb_io_dynamic(struct pci_controller *hose, int primary); - - extern struct list_head hose_list; extern struct pci_dev *isa_bridge_pcidev; /* may be NULL if no ISA bus */ @@ -32,9 +28,6 @@ struct pci_dn; void *pci_traverse_device_nodes(struct device_node *start, void *(*fn)(struct device_node *, void *), void *data); -void *traverse_pci_dn(struct pci_dn *root, - void *(*fn)(struct pci_dn *, void *), - void *data); extern void pci_devs_phb_init_dynamic(struct pci_controller *phb); /* From rtas_pci.h */ diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index cc1bca571332..3dceb64fc9af 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -25,7 +25,6 @@ #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE #define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb) #define ACCOUNT_CPU_USER_EXIT(ptr, ra, rb) -#define ACCOUNT_STOLEN_TIME #else #define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb) \ MFTB(ra); /* get timebase */ \ @@ -44,29 +43,6 @@ PPC_LL ra, ACCOUNT_SYSTEM_TIME(ptr); \ add ra,ra,rb; /* add on to system time */ \ PPC_STL ra, ACCOUNT_SYSTEM_TIME(ptr) - -#ifdef CONFIG_PPC_SPLPAR -#define ACCOUNT_STOLEN_TIME \ -BEGIN_FW_FTR_SECTION; \ - beq 33f; \ - /* from user - see if there are any DTL entries to process */ \ - ld r10,PACALPPACAPTR(r13); /* get ptr to VPA */ \ - ld r11,PACA_DTL_RIDX(r13); /* get log read index */ \ - addi r10,r10,LPPACA_DTLIDX; \ - LDX_BE r10,0,r10; /* get log write index */ \ - cmpd cr1,r11,r10; \ - beq+ cr1,33f; \ - bl accumulate_stolen_time; \ - ld r12,_MSR(r1); \ - andi. r10,r12,MSR_PR; /* Restore cr0 (coming from user) */ \ -33: \ -END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) - -#else /* CONFIG_PPC_SPLPAR */ -#define ACCOUNT_STOLEN_TIME - -#endif /* CONFIG_PPC_SPLPAR */ - #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ /* diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h index 58f9dc060a7b..975ba260006a 100644 --- a/arch/powerpc/include/asm/ptrace.h +++ b/arch/powerpc/include/asm/ptrace.h @@ -70,6 +70,9 @@ struct pt_regs }; #endif + +#define STACK_FRAME_WITH_PT_REGS (STACK_FRAME_OVERHEAD + sizeof(struct pt_regs)) + #ifdef __powerpc64__ /* @@ -229,6 +232,11 @@ static inline bool trap_is_scv(struct pt_regs *regs) return (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && TRAP(regs) == 0x3000); } +static inline bool trap_is_unsupported_scv(struct pt_regs *regs) +{ + return IS_ENABLED(CONFIG_PPC_BOOK3S_64) && TRAP(regs) == 0x7ff0; +} + static inline bool trap_is_syscall(struct pt_regs *regs) { return (trap_is_scv(regs) || TRAP(regs) == 0xc00); diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index e40a921d78f9..da103e92c112 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -1375,6 +1375,7 @@ #define mtmsr(v) asm volatile("mtmsr %0" : \ : "r" ((unsigned long)(v)) \ : "memory") +#define __mtmsrd(v, l) BUILD_BUG() #define __MTMSR "mtmsr" #endif @@ -1413,13 +1414,24 @@ static inline void msr_check_and_clear(unsigned long bits) } #ifdef CONFIG_PPC32 -#define mfsrin(v) ({unsigned int rval; \ - asm volatile("mfsrin %0,%1" : "=r" (rval) : "r" (v)); \ - rval;}) - -static inline void mtsrin(u32 val, u32 idx) +static inline u32 mfsr(u32 idx) { - asm volatile("mtsrin %0, %1" : : "r" (val), "r" (idx)); + u32 val; + + if (__builtin_constant_p(idx)) + asm volatile("mfsr %0, %1" : "=r" (val): "i" (idx >> 28)); + else + asm volatile("mfsrin %0, %1" : "=r" (val): "r" (idx)); + + return val; +} + +static inline void mtsr(u32 val, u32 idx) +{ + if (__builtin_constant_p(idx)) + asm volatile("mtsr %1, %0" : : "r" (val), "i" (idx >> 28)); + else + asm volatile("mtsrin %0, %1" : : "r" (val), "r" (idx)); } #endif diff --git a/arch/powerpc/include/asm/reg_booke.h b/arch/powerpc/include/asm/reg_booke.h index 262782f08fd4..17b8dcd9a40d 100644 --- a/arch/powerpc/include/asm/reg_booke.h +++ b/arch/powerpc/include/asm/reg_booke.h @@ -691,6 +691,9 @@ #define mttmr(rn, v) asm volatile(MTTMR(rn, %0) : \ : "r" ((unsigned long)(v)) \ : "memory") + +extern unsigned long global_dbcr0[]; + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_POWERPC_REG_BOOKE_H__ */ diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h index 332e1000ca0f..658448ca5b8a 100644 --- a/arch/powerpc/include/asm/rtas.h +++ b/arch/powerpc/include/asm/rtas.h @@ -369,7 +369,7 @@ void rtas_initialize(void); #else static inline int page_is_rtas_user_buf(unsigned long pfn) { return 0;} static inline void pSeries_coalesce_init(void) { } -static inline void rtas_initialize(void) { }; +static inline void rtas_initialize(void) { } #endif extern int call_rtas(const char *, int, int, unsigned long *, ...); diff --git a/arch/powerpc/include/asm/setup.h b/arch/powerpc/include/asm/setup.h index a466749703f1..e89bfebd4e00 100644 --- a/arch/powerpc/include/asm/setup.h +++ b/arch/powerpc/include/asm/setup.h @@ -58,7 +58,7 @@ void do_rfi_flush_fixups(enum l1d_flush_type types); #ifdef CONFIG_PPC_BARRIER_NOSPEC void setup_barrier_nospec(void); #else -static inline void setup_barrier_nospec(void) { }; +static inline void setup_barrier_nospec(void) { } #endif void do_uaccess_flush_fixups(enum l1d_flush_type types); void do_entry_flush_fixups(enum l1d_flush_type types); @@ -68,13 +68,13 @@ extern bool barrier_nospec_enabled; #ifdef CONFIG_PPC_BARRIER_NOSPEC void do_barrier_nospec_fixups_range(bool enable, void *start, void *end); #else -static inline void do_barrier_nospec_fixups_range(bool enable, void *start, void *end) { }; +static inline void do_barrier_nospec_fixups_range(bool enable, void *start, void *end) { } #endif #ifdef CONFIG_PPC_FSL_BOOK3E void setup_spectre_v2(void); #else -static inline void setup_spectre_v2(void) {}; +static inline void setup_spectre_v2(void) {} #endif void do_btb_flush_fixups(void); diff --git a/arch/powerpc/include/asm/simple_spinlock.h b/arch/powerpc/include/asm/simple_spinlock.h index 9c3c30534333..5b862de29dff 100644 --- a/arch/powerpc/include/asm/simple_spinlock.h +++ b/arch/powerpc/include/asm/simple_spinlock.h @@ -90,8 +90,8 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock) void splpar_spin_yield(arch_spinlock_t *lock); void splpar_rw_yield(arch_rwlock_t *lock); #else /* SPLPAR */ -static inline void splpar_spin_yield(arch_spinlock_t *lock) {}; -static inline void splpar_rw_yield(arch_rwlock_t *lock) {}; +static inline void splpar_spin_yield(arch_spinlock_t *lock) {} +static inline void splpar_rw_yield(arch_rwlock_t *lock) {} #endif static inline void spin_yield(arch_spinlock_t *lock) diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h index c4e2d53acd2b..7a13bc20f0a0 100644 --- a/arch/powerpc/include/asm/smp.h +++ b/arch/powerpc/include/asm/smp.h @@ -236,7 +236,7 @@ static inline void set_hard_smp_processor_id(int cpu, int phys) #if defined(CONFIG_PPC64) && (defined(CONFIG_SMP) || defined(CONFIG_KEXEC_CORE)) extern void smp_release_cpus(void); #else -static inline void smp_release_cpus(void) { }; +static inline void smp_release_cpus(void) { } #endif extern int smt_enabled_at_boot; diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index 3d8a47af7a25..386d576673a1 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -94,7 +94,6 @@ void arch_setup_new_exec(void); #define TIF_PATCH_PENDING 6 /* pending live patching update */ #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ #define TIF_SINGLESTEP 8 /* singlestepping active */ -#define TIF_NOHZ 9 /* in adaptive nohz mode */ #define TIF_SECCOMP 10 /* secure computing */ #define TIF_RESTOREALL 11 /* Restore all regs (implies NOERROR) */ #define TIF_NOERROR 12 /* Force successful syscall return */ @@ -128,11 +127,10 @@ void arch_setup_new_exec(void); #define _TIF_UPROBE (1< #include +#include #include #include #include #ifdef CONFIG_SMP -void doorbell_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(doorbell_exception) { struct pt_regs *old_regs = set_irq_regs(regs); - irq_enter(); trace_doorbell_entry(regs); ppc_msgsync(); @@ -35,13 +35,12 @@ void doorbell_exception(struct pt_regs *regs) smp_ipi_demux_relaxed(); /* already performed the barrier */ trace_doorbell_exit(regs); - irq_exit(); + set_irq_regs(old_regs); } #else /* CONFIG_SMP */ -void doorbell_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(doorbell_exception) { printk(KERN_WARNING "Received doorbell on non-smp system\n"); } #endif /* CONFIG_SMP */ - diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index 813713c9120c..cd60bc1c8701 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -1596,6 +1596,35 @@ static int proc_eeh_show(struct seq_file *m, void *v) } #ifdef CONFIG_DEBUG_FS + + +static struct pci_dev *eeh_debug_lookup_pdev(struct file *filp, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + uint32_t domain, bus, dev, fn; + struct pci_dev *pdev; + char buf[20]; + int ret; + + memset(buf, 0, sizeof(buf)); + ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count); + if (!ret) + return ERR_PTR(-EFAULT); + + ret = sscanf(buf, "%x:%x:%x.%x", &domain, &bus, &dev, &fn); + if (ret != 4) { + pr_err("%s: expected 4 args, got %d\n", __func__, ret); + return ERR_PTR(-EINVAL); + } + + pdev = pci_get_domain_bus_and_slot(domain, bus, (dev << 3) | fn); + if (!pdev) + return ERR_PTR(-ENODEV); + + return pdev; +} + static int eeh_enable_dbgfs_set(void *data, u64 val) { if (val) @@ -1688,26 +1717,13 @@ static ssize_t eeh_dev_check_write(struct file *filp, const char __user *user_buf, size_t count, loff_t *ppos) { - uint32_t domain, bus, dev, fn; struct pci_dev *pdev; struct eeh_dev *edev; - char buf[20]; int ret; - memset(buf, 0, sizeof(buf)); - ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count); - if (!ret) - return -EFAULT; - - ret = sscanf(buf, "%x:%x:%x.%x", &domain, &bus, &dev, &fn); - if (ret != 4) { - pr_err("%s: expected 4 args, got %d\n", __func__, ret); - return -EINVAL; - } - - pdev = pci_get_domain_bus_and_slot(domain, bus, (dev << 3) | fn); - if (!pdev) - return -ENODEV; + pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); edev = pci_dev_to_eeh_dev(pdev); if (!edev) { @@ -1717,8 +1733,8 @@ static ssize_t eeh_dev_check_write(struct file *filp, } ret = eeh_dev_check_failure(edev); - pci_info(pdev, "eeh_dev_check_failure(%04x:%02x:%02x.%01x) = %d\n", - domain, bus, dev, fn, ret); + pci_info(pdev, "eeh_dev_check_failure(%s) = %d\n", + pci_name(pdev), ret); pci_dev_put(pdev); @@ -1829,25 +1845,12 @@ static ssize_t eeh_dev_break_write(struct file *filp, const char __user *user_buf, size_t count, loff_t *ppos) { - uint32_t domain, bus, dev, fn; struct pci_dev *pdev; - char buf[20]; int ret; - memset(buf, 0, sizeof(buf)); - ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count); - if (!ret) - return -EFAULT; - - ret = sscanf(buf, "%x:%x:%x.%x", &domain, &bus, &dev, &fn); - if (ret != 4) { - pr_err("%s: expected 4 args, got %d\n", __func__, ret); - return -EINVAL; - } - - pdev = pci_get_domain_bus_and_slot(domain, bus, (dev << 3) | fn); - if (!pdev) - return -ENODEV; + pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); ret = eeh_debugfs_break_device(pdev); pci_dev_put(pdev); @@ -1865,6 +1868,53 @@ static const struct file_operations eeh_dev_break_fops = { .read = eeh_debugfs_dev_usage, }; +static ssize_t eeh_dev_can_recover(struct file *filp, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct pci_driver *drv; + struct pci_dev *pdev; + size_t ret; + + pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); + + /* + * In order for error recovery to work the driver needs to implement + * .error_detected(), so it can quiesce IO to the device, and + * .slot_reset() so it can re-initialise the device after a reset. + * + * Ideally they'd implement .resume() too, but some drivers which + * we need to support (notably IPR) don't so I guess we can tolerate + * that. + * + * .mmio_enabled() is mostly there as a work-around for devices which + * take forever to re-init after a hot reset. Implementing that is + * strictly optional. + */ + drv = pci_dev_driver(pdev); + if (drv && + drv->err_handler && + drv->err_handler->error_detected && + drv->err_handler->slot_reset) { + ret = count; + } else { + ret = -EOPNOTSUPP; + } + + pci_dev_put(pdev); + + return ret; +} + +static const struct file_operations eeh_dev_can_recover_fops = { + .open = simple_open, + .llseek = no_llseek, + .write = eeh_dev_can_recover, + .read = eeh_debugfs_dev_usage, +}; + #endif static int __init eeh_init_proc(void) @@ -1889,6 +1939,9 @@ static int __init eeh_init_proc(void) debugfs_create_file_unsafe("eeh_force_recover", 0600, powerpc_debugfs_root, NULL, &eeh_force_recover_fops); + debugfs_create_file_unsafe("eeh_dev_can_recover", 0600, + powerpc_debugfs_root, NULL, + &eeh_dev_can_recover_fops); eeh_cache_debugfs_init(); #endif } diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index 1c9b0ccc2172..78c430b7f9d9 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -175,14 +175,11 @@ transfer_to_handler: addi r11,r11,global_dbcr0@l #ifdef CONFIG_SMP lwz r9,TASK_CPU(r2) - slwi r9,r9,3 + slwi r9,r9,2 add r11,r11,r9 #endif lwz r12,0(r11) mtspr SPRN_DBCR0,r12 - lwz r12,4(r11) - addi r12,r12,-1 - stw r12,4(r11) #endif b 3f @@ -276,8 +273,7 @@ reenable_mmu: * We save a bunch of GPRs, * r3 can be different from GPR3(r1) at this point, r9 and r11 * contains the old MSR and handler address respectively, - * r4 & r5 can contain page fault arguments that need to be passed - * along as well. r0, r6-r8, r12, CCR, CTR, XER etc... are left + * r0, r4-r8, r12, CCR, CTR, XER etc... are left * clobbered as they aren't useful past this point. */ @@ -285,15 +281,11 @@ reenable_mmu: stw r9,8(r1) stw r11,12(r1) stw r3,16(r1) - stw r4,20(r1) - stw r5,24(r1) /* If we are disabling interrupts (normal case), simply log it with * lockdep */ 1: bl trace_hardirqs_off - lwz r5,24(r1) - lwz r4,20(r1) lwz r3,16(r1) lwz r11,12(r1) lwz r9,8(r1) @@ -334,132 +326,29 @@ stack_ovf: _ASM_NOKPROBE_SYMBOL(stack_ovf) #endif -#ifdef CONFIG_TRACE_IRQFLAGS -trace_syscall_entry_irq_off: - /* - * Syscall shouldn't happen while interrupts are disabled, - * so let's do a warning here. - */ -0: trap - EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING - bl trace_hardirqs_on - - /* Now enable for real */ - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE) - mtmsr r10 - - REST_GPR(0, r1) - REST_4GPRS(3, r1) - REST_2GPRS(7, r1) - b DoSyscall -#endif /* CONFIG_TRACE_IRQFLAGS */ - .globl transfer_to_syscall transfer_to_syscall: -#ifdef CONFIG_TRACE_IRQFLAGS - andi. r12,r9,MSR_EE - beq- trace_syscall_entry_irq_off -#endif /* CONFIG_TRACE_IRQFLAGS */ + SAVE_NVGPRS(r1) +#ifdef CONFIG_PPC_BOOK3S_32 + kuep_lock r11, r12 +#endif -/* - * Handle a system call. - */ - .stabs "arch/powerpc/kernel/",N_SO,0,0,0f - .stabs "entry_32.S",N_SO,0,0,0f -0: + /* Calling convention has r9 = orig r0, r10 = regs */ + addi r10,r1,STACK_FRAME_OVERHEAD + mr r9,r0 + stw r10,THREAD+PT_REGS(r2) + bl system_call_exception -_GLOBAL(DoSyscall) - stw r3,ORIG_GPR3(r1) - li r12,0 - stw r12,RESULT(r1) -#ifdef CONFIG_TRACE_IRQFLAGS - /* Make sure interrupts are enabled */ - mfmsr r11 - andi. r12,r11,MSR_EE - /* We came in with interrupts disabled, we WARN and mark them enabled - * for lockdep now */ -0: tweqi r12, 0 - EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING -#endif /* CONFIG_TRACE_IRQFLAGS */ - lwz r11,TI_FLAGS(r2) - andi. r11,r11,_TIF_SYSCALL_DOTRACE - bne- syscall_dotrace -syscall_dotrace_cont: - cmplwi 0,r0,NR_syscalls - lis r10,sys_call_table@h - ori r10,r10,sys_call_table@l - slwi r0,r0,2 - bge- 66f - - barrier_nospec_asm - /* - * Prevent the load of the handler below (based on the user-passed - * system call number) being speculatively executed until the test - * against NR_syscalls and branch to .66f above has - * committed. - */ - - lwzx r10,r10,r0 /* Fetch system call handler [ptr] */ - mtlr r10 - addi r9,r1,STACK_FRAME_OVERHEAD - PPC440EP_ERR42 - blrl /* Call handler */ - .globl ret_from_syscall ret_from_syscall: -#ifdef CONFIG_DEBUG_RSEQ - /* Check whether the syscall is issued inside a restartable sequence */ - stw r3,GPR3(r1) - addi r3,r1,STACK_FRAME_OVERHEAD - bl rseq_syscall - lwz r3,GPR3(r1) -#endif - mr r6,r3 - /* disable interrupts so current_thread_info()->flags can't change */ - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) /* doesn't include MSR_EE */ - /* Note: We don't bother telling lockdep about it */ - mtmsr r10 - lwz r9,TI_FLAGS(r2) - li r8,-MAX_ERRNO - andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK) - bne- syscall_exit_work - cmplw 0,r3,r8 - blt+ syscall_exit_cont - lwz r11,_CCR(r1) /* Load CR */ - neg r3,r3 - oris r11,r11,0x1000 /* Set SO bit in CR */ - stw r11,_CCR(r1) -syscall_exit_cont: - lwz r8,_MSR(r1) -#ifdef CONFIG_TRACE_IRQFLAGS - /* If we are going to return from the syscall with interrupts - * off, we trace that here. It shouldn't normally happen. - */ - andi. r10,r8,MSR_EE - bne+ 1f - stw r3,GPR3(r1) - bl trace_hardirqs_off - lwz r3,GPR3(r1) -1: -#endif /* CONFIG_TRACE_IRQFLAGS */ -#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) - /* If the process has its own DBCR0 value, load it up. The internal - debug mode bit tells us that dbcr0 should be loaded. */ - lwz r0,THREAD+THREAD_DBCR0(r2) - andis. r10,r0,DBCR0_IDM@h - bnel- load_dbcr0 -#endif + addi r4,r1,STACK_FRAME_OVERHEAD + li r5,0 + bl syscall_exit_prepare #ifdef CONFIG_PPC_47x lis r4,icache_44x_need_flush@ha lwz r5,icache_44x_need_flush@l(r4) cmplwi cr0,r5,0 bne- 2f #endif /* CONFIG_PPC_47x */ -1: -BEGIN_FTR_SECTION - lwarx r7,0,r1 -END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) - stwcx. r0,0,r1 /* to clear the reservation */ - ACCOUNT_CPU_USER_EXIT(r2, r5, r7) #ifdef CONFIG_PPC_BOOK3S_32 kuep_unlock r5, r7 #endif @@ -467,21 +356,36 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) lwz r4,_LINK(r1) lwz r5,_CCR(r1) mtlr r4 - mtcr r5 lwz r7,_NIP(r1) - lwz r2,GPR2(r1) - lwz r1,GPR1(r1) + lwz r8,_MSR(r1) + cmpwi r3,0 + lwz r3,GPR3(r1) syscall_exit_finish: -#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) - mtspr SPRN_NRI, r0 -#endif mtspr SPRN_SRR0,r7 mtspr SPRN_SRR1,r8 + + bne 3f + mtcr r5 + +1: lwz r2,GPR2(r1) + lwz r1,GPR1(r1) rfi #ifdef CONFIG_40x b . /* Prevent prefetch past rfi */ #endif -_ASM_NOKPROBE_SYMBOL(syscall_exit_finish) + +3: mtcr r5 + lwz r4,_CTR(r1) + lwz r5,_XER(r1) + REST_NVGPRS(r1) + mtctr r4 + mtxer r5 + lwz r0,GPR0(r1) + lwz r3,GPR3(r1) + REST_8GPRS(4,r1) + lwz r12,GPR12(r1) + b 1b + #ifdef CONFIG_44x 2: li r7,0 iccci r0,r0 @@ -489,9 +393,6 @@ _ASM_NOKPROBE_SYMBOL(syscall_exit_finish) b 1b #endif /* CONFIG_44x */ -66: li r3,-ENOSYS - b ret_from_syscall - .globl ret_from_fork ret_from_fork: REST_NVGPRS(r1) @@ -510,157 +411,6 @@ ret_from_kernel_thread: li r3,0 b ret_from_syscall -/* Traced system call support */ -syscall_dotrace: - SAVE_NVGPRS(r1) - li r0,0xc00 - stw r0,_TRAP(r1) - addi r3,r1,STACK_FRAME_OVERHEAD - bl do_syscall_trace_enter - /* - * Restore argument registers possibly just changed. - * We use the return value of do_syscall_trace_enter - * for call number to look up in the table (r0). - */ - mr r0,r3 - lwz r3,GPR3(r1) - lwz r4,GPR4(r1) - lwz r5,GPR5(r1) - lwz r6,GPR6(r1) - lwz r7,GPR7(r1) - lwz r8,GPR8(r1) - REST_NVGPRS(r1) - - cmplwi r0,NR_syscalls - /* Return code is already in r3 thanks to do_syscall_trace_enter() */ - bge- ret_from_syscall - b syscall_dotrace_cont - -syscall_exit_work: - andi. r0,r9,_TIF_RESTOREALL - beq+ 0f - REST_NVGPRS(r1) - b 2f -0: cmplw 0,r3,r8 - blt+ 1f - andi. r0,r9,_TIF_NOERROR - bne- 1f - lwz r11,_CCR(r1) /* Load CR */ - neg r3,r3 - oris r11,r11,0x1000 /* Set SO bit in CR */ - stw r11,_CCR(r1) - -1: stw r6,RESULT(r1) /* Save result */ - stw r3,GPR3(r1) /* Update return value */ -2: andi. r0,r9,(_TIF_PERSYSCALL_MASK) - beq 4f - - /* Clear per-syscall TIF flags if any are set. */ - - li r11,_TIF_PERSYSCALL_MASK - addi r12,r2,TI_FLAGS -3: lwarx r8,0,r12 - andc r8,r8,r11 - stwcx. r8,0,r12 - bne- 3b - -4: /* Anything which requires enabling interrupts? */ - andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP) - beq ret_from_except - - /* Re-enable interrupts. There is no need to trace that with - * lockdep as we are supposed to have IRQs on at this point - */ - ori r10,r10,MSR_EE - mtmsr r10 - - /* Save NVGPRS if they're not saved already */ - lwz r4,_TRAP(r1) - andi. r4,r4,1 - beq 5f - SAVE_NVGPRS(r1) - li r4,0xc00 - stw r4,_TRAP(r1) -5: - addi r3,r1,STACK_FRAME_OVERHEAD - bl do_syscall_trace_leave - b ret_from_except_full - - /* - * System call was called from kernel. We get here with SRR1 in r9. - * Mark the exception as recoverable once we have retrieved SRR0, - * trap a warning and return ENOSYS with CR[SO] set. - */ - .globl ret_from_kernel_syscall -ret_from_kernel_syscall: - mfspr r9, SPRN_SRR0 - mfspr r10, SPRN_SRR1 -#if !defined(CONFIG_4xx) && !defined(CONFIG_BOOKE) - LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_IR|MSR_DR)) - mtmsr r11 -#endif - -0: trap - EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING - - li r3, ENOSYS - crset so -#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) - mtspr SPRN_NRI, r0 -#endif - mtspr SPRN_SRR0, r9 - mtspr SPRN_SRR1, r10 - rfi -#ifdef CONFIG_40x - b . /* Prevent prefetch past rfi */ -#endif -_ASM_NOKPROBE_SYMBOL(ret_from_kernel_syscall) - -/* - * The fork/clone functions need to copy the full register set into - * the child process. Therefore we need to save all the nonvolatile - * registers (r13 - r31) before calling the C code. - */ - .globl ppc_fork -ppc_fork: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ - stw r0,_TRAP(r1) /* register set saved */ - b sys_fork - - .globl ppc_vfork -ppc_vfork: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ - stw r0,_TRAP(r1) /* register set saved */ - b sys_vfork - - .globl ppc_clone -ppc_clone: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ - stw r0,_TRAP(r1) /* register set saved */ - b sys_clone - - .globl ppc_clone3 -ppc_clone3: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ - stw r0,_TRAP(r1) /* register set saved */ - b sys_clone3 - - .globl ppc_swapcontext -ppc_swapcontext: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ - stw r0,_TRAP(r1) /* register set saved */ - b sys_swapcontext - /* * Top-level page fault handling. * This is in assembler because if do_page_fault tells us that @@ -670,10 +420,6 @@ ppc_swapcontext: .globl handle_page_fault handle_page_fault: addi r3,r1,STACK_FRAME_OVERHEAD -#ifdef CONFIG_PPC_BOOK3S_32 - andis. r0,r5,DSISR_DABRMATCH@h - bne- handle_dabr_fault -#endif bl do_page_fault cmpwi r3,0 beq+ ret_from_except @@ -681,23 +427,11 @@ handle_page_fault: lwz r0,_TRAP(r1) clrrwi r0,r0,1 stw r0,_TRAP(r1) - mr r5,r3 + mr r4,r3 /* err arg for bad_page_fault */ addi r3,r1,STACK_FRAME_OVERHEAD - lwz r4,_DAR(r1) bl __bad_page_fault b ret_from_except_full -#ifdef CONFIG_PPC_BOOK3S_32 - /* We have a data breakpoint exception - handle it */ -handle_dabr_fault: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - clrrwi r0,r0,1 - stw r0,_TRAP(r1) - bl do_break - b ret_from_except_full -#endif - /* * This routine switches between two different tasks. The process * state of one is saved on its kernel stack. Then the state @@ -1237,14 +971,11 @@ load_dbcr0: addi r11,r11,global_dbcr0@l #ifdef CONFIG_SMP lwz r9,TASK_CPU(r2) - slwi r9,r9,3 + slwi r9,r9,2 add r11,r11,r9 #endif stw r10,0(r11) mtspr SPRN_DBCR0,r0 - lwz r10,4(r11) - addi r10,r10,1 - stw r10,4(r11) li r11,-1 mtspr SPRN_DBSR,r11 /* clear all pending debug events */ blr @@ -1253,7 +984,7 @@ load_dbcr0: .align 4 .global global_dbcr0 global_dbcr0: - .space 8*NR_CPUS + .space 4*NR_CPUS .previous #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */ diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 33ddfeef4fe9..6c4d9e276c4d 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -108,7 +108,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) li r11,\trapnr std r11,_TRAP(r1) std r12,_CCR(r1) - std r3,ORIG_GPR3(r1) addi r10,r1,STACK_FRAME_OVERHEAD ld r11,exception_marker@toc(r2) std r11,-16(r10) /* "regshere" marker */ @@ -225,6 +224,12 @@ _ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate) b system_call_vectored_common #endif + .balign IFETCH_ALIGN_BYTES + .globl system_call_common_real +system_call_common_real: + ld r10,PACAKMSR(r13) /* get MSR value for kernel */ + mtmsrd r10 + .balign IFETCH_ALIGN_BYTES .globl system_call_common system_call_common: @@ -278,7 +283,6 @@ END_BTB_FLUSH_SECTION std r10,_LINK(r1) std r11,_TRAP(r1) std r12,_CCR(r1) - std r3,ORIG_GPR3(r1) addi r10,r1,STACK_FRAME_OVERHEAD ld r11,exception_marker@toc(r2) std r11,-16(r10) /* "regshere" marker */ diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index 74d07dc0bb48..e8eb9992a270 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -398,7 +398,6 @@ exc_##n##_common: \ std r10,_NIP(r1); /* save SRR0 to stackframe */ \ std r11,_MSR(r1); /* save SRR1 to stackframe */ \ beq 2f; /* if from kernel mode */ \ - ACCOUNT_CPU_USER_ENTRY(r13,r10,r11);/* accounting (uses cr0+eq) */ \ 2: ld r3,excf+EX_R10(r13); /* get back r10 */ \ ld r4,excf+EX_R11(r13); /* get back r11 */ \ mfspr r5,scratch; /* get back r13 */ \ @@ -791,7 +790,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) EXCEPTION_COMMON_CRIT(0xd00) std r14,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD - mr r4,r14 ld r14,PACA_EXCRIT+EX_R14(r13) ld r15,PACA_EXCRIT+EX_R15(r13) bl save_nvgprs @@ -864,7 +862,6 @@ kernel_dbg_exc: INTS_DISABLE std r14,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD - mr r4,r14 ld r14,PACA_EXDBG+EX_R14(r13) ld r15,PACA_EXDBG+EX_R15(r13) bl save_nvgprs @@ -1011,8 +1008,6 @@ storage_fault_common: std r14,_DAR(r1) std r15,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD - mr r4,r14 - mr r5,r15 ld r14,PACA_EXGEN+EX_R14(r13) ld r15,PACA_EXGEN+EX_R15(r13) bl do_page_fault @@ -1020,9 +1015,8 @@ storage_fault_common: bne- 1f b ret_from_except_lite 1: bl save_nvgprs - mr r5,r3 + mr r4,r3 addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) bl __bad_page_fault b ret_from_except diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 6e53f7638737..60d3051a8bc8 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -139,7 +139,6 @@ name: #define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ #define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ #define __ISTACK(name) .L_ISTACK_ ## name -#define IRECONCILE .L_IRECONCILE_\name\() /* Do RECONCILE_IRQ_STATE */ #define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ #define INT_DEFINE_BEGIN(n) \ @@ -203,9 +202,6 @@ do_define_int n .ifndef ISTACK ISTACK=1 .endif - .ifndef IRECONCILE - IRECONCILE=1 - .endif .ifndef IKUAP IKUAP=1 .endif @@ -581,7 +577,6 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real) kuap_save_amr_and_lock r9, r10, cr1, cr0 .endif beq 101f /* if from kernel mode */ - ACCOUNT_CPU_USER_ENTRY(r13, r9, r10) BEGIN_FTR_SECTION ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ std r9,_PPR(r1) @@ -649,14 +644,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) ld r11,exception_marker@toc(r2) std r10,RESULT(r1) /* clear regs->result */ std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ - - .if ISTACK - ACCOUNT_STOLEN_TIME - .endif - - .if IRECONCILE - RECONCILE_IRQ_STATE(r10, r11) - .endif .endm /* @@ -705,14 +692,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) ld r1,GPR1(r1) .endm -#define RUNLATCH_ON \ -BEGIN_FTR_SECTION \ - ld r3, PACA_THREAD_INFO(r13); \ - ld r4,TI_LOCAL_FLAGS(r3); \ - andi. r0,r4,_TLF_RUNLATCH; \ - beql ppc64_runlatch_on_trampoline; \ -END_FTR_SECTION_IFSET(CPU_FTR_CTRL) - /* * When the idle code in power4_idle puts the CPU into NAP mode, * it has to do so in a loop, and relies on the external interrupt @@ -935,7 +914,6 @@ INT_DEFINE_BEGIN(system_reset) */ ISET_RI=0 ISTACK=0 - IRECONCILE=0 IKVM_REAL=1 INT_DEFINE_END(system_reset) @@ -1022,20 +1000,6 @@ EXC_COMMON_BEGIN(system_reset_common) ld r1,PACA_NMI_EMERG_SP(r13) subi r1,r1,INT_FRAME_SIZE __GEN_COMMON_BODY system_reset - /* - * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does - * the right thing. We do not want to reconcile because that goes - * through irq tracing which we don't want in NMI. - * - * Save PACAIRQHAPPENED to RESULT (otherwise unused), and set HARD_DIS - * as we are running with MSR[EE]=0. - */ - li r10,IRQS_ALL_DISABLED - stb r10,PACAIRQSOFTMASK(r13) - lbz r10,PACAIRQHAPPENED(r13) - std r10,RESULT(r1) - ori r10,r10,PACA_IRQ_HARD_DIS - stb r10,PACAIRQHAPPENED(r13) addi r3,r1,STACK_FRAME_OVERHEAD bl system_reset_exception @@ -1051,14 +1015,6 @@ EXC_COMMON_BEGIN(system_reset_common) subi r10,r10,1 sth r10,PACA_IN_NMI(r13) - /* - * Restore soft mask settings. - */ - ld r10,RESULT(r1) - stb r10,PACAIRQHAPPENED(r13) - ld r10,SOFTE(r1) - stb r10,PACAIRQSOFTMASK(r13) - kuap_kernel_restore r9, r10 EXCEPTION_RESTORE_REGS RFI_TO_USER_OR_KERNEL @@ -1123,7 +1079,6 @@ INT_DEFINE_BEGIN(machine_check_early) ISTACK=0 IDAR=1 IDSISR=1 - IRECONCILE=0 IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ INT_DEFINE_END(machine_check_early) @@ -1205,30 +1160,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) li r10,MSR_RI mtmsrd r10,1 - /* - * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see - * system_reset_common) - */ - li r10,IRQS_ALL_DISABLED - stb r10,PACAIRQSOFTMASK(r13) - lbz r10,PACAIRQHAPPENED(r13) - std r10,RESULT(r1) - ori r10,r10,PACA_IRQ_HARD_DIS - stb r10,PACAIRQHAPPENED(r13) - addi r3,r1,STACK_FRAME_OVERHEAD bl machine_check_early std r3,RESULT(r1) /* Save result */ ld r12,_MSR(r1) - /* - * Restore soft mask settings. - */ - ld r10,RESULT(r1) - stb r10,PACAIRQHAPPENED(r13) - ld r10,SOFTE(r1) - stb r10,PACAIRQSOFTMASK(r13) - #ifdef CONFIG_PPC_P7_NAP /* * Check if thread was in power saving mode. We come here when any @@ -1401,14 +1337,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) * * Handling: * - Hash MMU - * Go to do_hash_page first to see if the HPT can be filled from an entry in - * the Linux page table. Hash faults can hit in kernel mode in a fairly + * Go to do_hash_fault, which attempts to fill the HPT from an entry in the + * Linux page table. Hash faults can hit in kernel mode in a fairly * arbitrary state (e.g., interrupts disabled, locks held) when accessing * "non-bolted" regions, e.g., vmalloc space. However these should always be - * backed by Linux page tables. + * backed by Linux page table entries. * - * If none is found, do a Linux page fault. Linux page faults can happen in - * kernel mode due to user copy operations of course. + * If no entry is found the Linux page fault handler is invoked (by + * do_hash_fault). Linux page faults can happen in kernel mode due to user + * copy operations of course. * * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest * MMU context, which may cause a DSI in the host, which must go to the @@ -1437,15 +1374,24 @@ EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) EXC_VIRT_END(data_access, 0x4300, 0x80) EXC_COMMON_BEGIN(data_access_common) GEN_COMMON data_access - ld r4,_DAR(r1) - ld r5,_DSISR(r1) + ld r4,_DSISR(r1) + addi r3,r1,STACK_FRAME_OVERHEAD + andis. r0,r4,DSISR_DABRMATCH@h + bne- 1f BEGIN_MMU_FTR_SECTION - ld r6,_MSR(r1) - li r3,0x300 - b do_hash_page /* Try to handle as hpte fault */ + bl do_hash_fault MMU_FTR_SECTION_ELSE - b handle_page_fault + bl do_page_fault ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) + b interrupt_return + +1: bl do_break + /* + * do_break() may have changed the NV GPRS while handling a breakpoint. + * If so, we need to restore them with their updated values. + */ + REST_NVGPRS(r1) + b interrupt_return GEN_KVM data_access @@ -1466,14 +1412,9 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) * on user-handler data structures. * * KVM: Same as 0x300, DSLB must test for KVM guest. - * - * A dedicated save area EXSLB is used (XXX: but it actually need not be - * these days, we could use EXGEN). */ INT_DEFINE_BEGIN(data_access_slb) IVEC=0x380 - IAREA=PACA_EXSLB - IRECONCILE=0 IDAR=1 IKVM_SKIP=1 IKVM_REAL=1 @@ -1487,10 +1428,9 @@ EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) EXC_VIRT_END(data_access_slb, 0x4380, 0x80) EXC_COMMON_BEGIN(data_access_slb_common) GEN_COMMON data_access_slb - ld r4,_DAR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION /* HPT case, do SLB fault */ + addi r3,r1,STACK_FRAME_OVERHEAD bl do_slb_fault cmpdi r3,0 bne- 1f @@ -1501,9 +1441,6 @@ MMU_FTR_SECTION_ELSE li r3,-EFAULT ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) std r3,RESULT(r1) - RECONCILE_IRQ_STATE(r10, r11) - ld r4,_DAR(r1) - ld r5,RESULT(r1) addi r3,r1,STACK_FRAME_OVERHEAD bl do_bad_slb_fault b interrupt_return @@ -1538,15 +1475,13 @@ EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) EXC_VIRT_END(instruction_access, 0x4400, 0x80) EXC_COMMON_BEGIN(instruction_access_common) GEN_COMMON instruction_access - ld r4,_DAR(r1) - ld r5,_DSISR(r1) + addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION - ld r6,_MSR(r1) - li r3,0x400 - b do_hash_page /* Try to handle as hpte fault */ + bl do_hash_fault MMU_FTR_SECTION_ELSE - b handle_page_fault + bl do_page_fault ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) + b interrupt_return GEN_KVM instruction_access @@ -1562,8 +1497,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) */ INT_DEFINE_BEGIN(instruction_access_slb) IVEC=0x480 - IAREA=PACA_EXSLB - IRECONCILE=0 IISIDE=1 IDAR=1 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE @@ -1579,10 +1512,9 @@ EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) EXC_COMMON_BEGIN(instruction_access_slb_common) GEN_COMMON instruction_access_slb - ld r4,_DAR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION /* HPT case, do SLB fault */ + addi r3,r1,STACK_FRAME_OVERHEAD bl do_slb_fault cmpdi r3,0 bne- 1f @@ -1593,9 +1525,6 @@ MMU_FTR_SECTION_ELSE li r3,-EFAULT ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) std r3,RESULT(r1) - RECONCILE_IRQ_STATE(r10, r11) - ld r4,_DAR(r1) - ld r5,RESULT(r1) addi r3,r1,STACK_FRAME_OVERHEAD bl do_bad_slb_fault b interrupt_return @@ -1643,7 +1572,6 @@ EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) EXC_COMMON_BEGIN(hardware_interrupt_common) GEN_COMMON hardware_interrupt FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl do_IRQ b interrupt_return @@ -1697,6 +1625,51 @@ INT_DEFINE_BEGIN(program_check) INT_DEFINE_END(program_check) EXC_REAL_BEGIN(program_check, 0x700, 0x100) + +#ifdef CONFIG_CPU_LITTLE_ENDIAN + /* + * There's a short window during boot where although the kernel is + * running little endian, any exceptions will cause the CPU to switch + * back to big endian. For example a WARN() boils down to a trap + * instruction, which will cause a program check, and we end up here but + * with the CPU in big endian mode. The first instruction of the program + * check handler (in GEN_INT_ENTRY below) is an mtsprg, which when + * executed in the wrong endian is an lhzu with a ~3GB displacement from + * r3. The content of r3 is random, so that is a load from some random + * location, and depending on the system can easily lead to a checkstop, + * or an infinitely recursive page fault. + * + * So to handle that case we have a trampoline here that can detect we + * are in the wrong endian and flip us back to the correct endian. We + * can't flip MSR[LE] using mtmsr, so we have to use rfid. That requires + * backing up SRR0/1 as well as a GPR. To do that we use SPRG0/2/3, as + * SPRG1 is already used for the paca. SPRG3 is user readable, but this + * trampoline is only active very early in boot, and SPRG3 will be + * reinitialised in vdso_getcpu_init() before userspace starts. + */ +BEGIN_FTR_SECTION + tdi 0,0,0x48 // Trap never, or in reverse endian: b . + 8 + b 1f // Skip trampoline if endian is correct + .long 0xa643707d // mtsprg 0, r11 Backup r11 + .long 0xa6027a7d // mfsrr0 r11 + .long 0xa643727d // mtsprg 2, r11 Backup SRR0 in SPRG2 + .long 0xa6027b7d // mfsrr1 r11 + .long 0xa643737d // mtsprg 3, r11 Backup SRR1 in SPRG3 + .long 0xa600607d // mfmsr r11 + .long 0x01006b69 // xori r11, r11, 1 Invert MSR[LE] + .long 0xa6037b7d // mtsrr1 r11 + .long 0x34076039 // li r11, 0x734 + .long 0xa6037a7d // mtsrr0 r11 + .long 0x2400004c // rfid + mfsprg r11, 3 + mtsrr1 r11 // Restore SRR1 + mfsprg r11, 2 + mtsrr0 r11 // Restore SRR0 + mfsprg r11, 0 // Restore r11 +1: +END_FTR_SECTION(0, 1) // nop out after boot +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ + GEN_INT_ENTRY program_check, virt=0 EXC_REAL_END(program_check, 0x700, 0x100) EXC_VIRT_BEGIN(program_check, 0x4700, 0x100) @@ -1755,7 +1728,6 @@ EXC_COMMON_BEGIN(program_check_common) */ INT_DEFINE_BEGIN(fp_unavailable) IVEC=0x800 - IRECONCILE=0 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif @@ -1770,7 +1742,6 @@ EXC_VIRT_END(fp_unavailable, 0x4800, 0x100) EXC_COMMON_BEGIN(fp_unavailable_common) GEN_COMMON fp_unavailable bne 1f /* if from user, just load it up */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl kernel_fp_unavailable_exception 0: trap @@ -1789,7 +1760,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) b fast_interrupt_return #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2: /* User process was in a transaction */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl fp_unavailable_tm b interrupt_return @@ -1832,7 +1802,6 @@ EXC_VIRT_END(decrementer, 0x4900, 0x80) EXC_COMMON_BEGIN(decrementer_common) GEN_COMMON decrementer FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl timer_interrupt b interrupt_return @@ -1854,7 +1823,6 @@ INT_DEFINE_BEGIN(hdecrementer) IVEC=0x980 IHSRR=1 ISTACK=0 - IRECONCILE=0 IKVM_REAL=1 IKVM_VIRT=1 INT_DEFINE_END(hdecrementer) @@ -1919,12 +1887,11 @@ EXC_VIRT_END(doorbell_super, 0x4a00, 0x100) EXC_COMMON_BEGIN(doorbell_super_common) GEN_COMMON doorbell_super FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_DOORBELL bl doorbell_exception #else - bl unknown_exception + bl unknown_async_exception #endif b interrupt_return @@ -2001,12 +1968,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) HMT_MEDIUM .if ! \virt - __LOAD_HANDLER(r10, system_call_common) - mtspr SPRN_SRR0,r10 - ld r10,PACAKMSR(r13) - mtspr SPRN_SRR1,r10 - RFI_TO_KERNEL - b . /* prevent speculative execution */ + __LOAD_HANDLER(r10, system_call_common_real) + mtctr r10 + bctr .else li r10,MSR_RI mtmsrd r10,1 /* Set RI (EE=0) */ @@ -2137,9 +2101,7 @@ EXC_COMMON_BEGIN(h_data_storage_common) GEN_COMMON h_data_storage addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION - ld r4,_DAR(r1) - li r5,SIGSEGV - bl bad_page_fault + bl do_bad_page_fault_segv MMU_FTR_SECTION_ELSE bl unknown_exception ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) @@ -2230,7 +2192,6 @@ INT_DEFINE_BEGIN(hmi_exception_early) IHSRR=1 IREALMODE_COMMON=1 ISTACK=0 - IRECONCILE=0 IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ IKVM_REAL=1 INT_DEFINE_END(hmi_exception_early) @@ -2277,7 +2238,6 @@ EXC_COMMON_BEGIN(hmi_exception_early_common) EXC_COMMON_BEGIN(hmi_exception_common) GEN_COMMON hmi_exception FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl handle_hmi_exception b interrupt_return @@ -2307,12 +2267,11 @@ EXC_VIRT_END(h_doorbell, 0x4e80, 0x20) EXC_COMMON_BEGIN(h_doorbell_common) GEN_COMMON h_doorbell FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_DOORBELL bl doorbell_exception #else - bl unknown_exception + bl unknown_async_exception #endif b interrupt_return @@ -2341,7 +2300,6 @@ EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) EXC_COMMON_BEGIN(h_virt_irq_common) GEN_COMMON h_virt_irq FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl do_IRQ b interrupt_return @@ -2388,7 +2346,6 @@ EXC_VIRT_END(performance_monitor, 0x4f00, 0x20) EXC_COMMON_BEGIN(performance_monitor_common) GEN_COMMON performance_monitor FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl performance_monitor_exception b interrupt_return @@ -2404,7 +2361,6 @@ EXC_COMMON_BEGIN(performance_monitor_common) */ INT_DEFINE_BEGIN(altivec_unavailable) IVEC=0xf20 - IRECONCILE=0 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif @@ -2434,7 +2390,6 @@ BEGIN_FTR_SECTION b fast_interrupt_return #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2: /* User process was in a transaction */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl altivec_unavailable_tm b interrupt_return @@ -2442,7 +2397,6 @@ BEGIN_FTR_SECTION 1: END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) #endif - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl altivec_unavailable_exception b interrupt_return @@ -2458,7 +2412,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) */ INT_DEFINE_BEGIN(vsx_unavailable) IVEC=0xf40 - IRECONCILE=0 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif @@ -2487,7 +2440,6 @@ BEGIN_FTR_SECTION b load_up_vsx #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2: /* User process was in a transaction */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl vsx_unavailable_tm b interrupt_return @@ -2495,7 +2447,6 @@ BEGIN_FTR_SECTION 1: END_FTR_SECTION_IFSET(CPU_FTR_VSX) #endif - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl vsx_unavailable_exception b interrupt_return @@ -2830,7 +2781,6 @@ EXC_VIRT_NONE(0x5800, 0x100) INT_DEFINE_BEGIN(soft_nmi) IVEC=0x900 ISTACK=0 - IRECONCILE=0 /* Soft-NMI may fire under local_irq_disable */ INT_DEFINE_END(soft_nmi) /* @@ -2849,17 +2799,6 @@ EXC_COMMON_BEGIN(soft_nmi_common) subi r1,r1,INT_FRAME_SIZE __GEN_COMMON_BODY soft_nmi - /* - * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see - * system_reset_common) - */ - li r10,IRQS_ALL_DISABLED - stb r10,PACAIRQSOFTMASK(r13) - lbz r10,PACAIRQHAPPENED(r13) - std r10,RESULT(r1) - ori r10,r10,PACA_IRQ_HARD_DIS - stb r10,PACAIRQHAPPENED(r13) - addi r3,r1,STACK_FRAME_OVERHEAD bl soft_nmi_interrupt @@ -2867,14 +2806,6 @@ EXC_COMMON_BEGIN(soft_nmi_common) li r9,0 mtmsrd r9,1 - /* - * Restore soft mask settings. - */ - ld r10,RESULT(r1) - stb r10,PACAIRQHAPPENED(r13) - ld r10,SOFTE(r1) - stb r10,PACAIRQSOFTMASK(r13) - kuap_kernel_restore r9, r10 EXCEPTION_RESTORE_REGS hsrr=0 RFI_TO_KERNEL @@ -3148,9 +3079,6 @@ kvmppc_skip_Hinterrupt: * come here. */ -EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline) - b __ppc64_runlatch_on - USE_FIXED_SECTION(virt_trampolines) /* * All code below __end_interrupts is treated as soft-masked. If @@ -3221,99 +3149,3 @@ disable_machine_check: RFI_TO_KERNEL 1: mtlr r0 blr - -/* - * Hash table stuff - */ - .balign IFETCH_ALIGN_BYTES -do_hash_page: -#ifdef CONFIG_PPC_BOOK3S_64 - lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h - ori r0,r0,DSISR_BAD_FAULT_64S@l - and. r0,r5,r0 /* weird error? */ - bne- handle_page_fault /* if not, try to insert a HPTE */ - - /* - * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then - * don't call hash_page, just fail the fault. This is required to - * prevent re-entrancy problems in the hash code, namely perf - * interrupts hitting while something holds H_PAGE_BUSY, and taking a - * hash fault. See the comment in hash_preload(). - */ - ld r11, PACA_THREAD_INFO(r13) - lwz r0,TI_PREEMPT(r11) - andis. r0,r0,NMI_MASK@h - bne 77f - - /* - * r3 contains the trap number - * r4 contains the faulting address - * r5 contains dsisr - * r6 msr - * - * at return r3 = 0 for success, 1 for page fault, negative for error - */ - bl __hash_page /* build HPTE if possible */ - cmpdi r3,0 /* see if __hash_page succeeded */ - - /* Success */ - beq interrupt_return /* Return from exception on success */ - - /* Error */ - blt- 13f - - /* Reload DAR/DSISR into r4/r5 for the DABR check below */ - ld r4,_DAR(r1) - ld r5,_DSISR(r1) -#endif /* CONFIG_PPC_BOOK3S_64 */ - -/* Here we have a page fault that hash_page can't handle. */ -handle_page_fault: -11: andis. r0,r5,DSISR_DABRMATCH@h - bne- handle_dabr_fault - addi r3,r1,STACK_FRAME_OVERHEAD - bl do_page_fault - cmpdi r3,0 - beq+ interrupt_return - mr r5,r3 - addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) - bl __bad_page_fault - b interrupt_return - -/* We have a data breakpoint exception - handle it */ -handle_dabr_fault: - ld r4,_DAR(r1) - ld r5,_DSISR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD - bl do_break - /* - * do_break() may have changed the NV GPRS while handling a breakpoint. - * If so, we need to restore them with their updated values. - */ - REST_NVGPRS(r1) - b interrupt_return - - -#ifdef CONFIG_PPC_BOOK3S_64 -/* We have a page fault that hash_page could handle but HV refused - * the PTE insertion - */ -13: mr r5,r3 - addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) - bl low_hash_fault - b interrupt_return -#endif - -/* - * We come here as a result of a DSI at a point where we don't want - * to call hash_page, such as when we are accessing memory (possibly - * user memory) inside a PMU interrupt that occurred while interrupts - * were soft-disabled. We want to invoke the exception handler for - * the access, or panic if there isn't a handler. - */ -77: addi r3,r1,STACK_FRAME_OVERHEAD - li r5,SIGSEGV - bl bad_page_fault - b interrupt_return diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h index a2f72c966baf..5d4706c14572 100644 --- a/arch/powerpc/kernel/head_32.h +++ b/arch/powerpc/kernel/head_32.h @@ -47,7 +47,7 @@ lwz r1,TASK_STACK-THREAD(r1) addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE 1: - mtcrf 0x7f, r1 + mtcrf 0x3f, r1 bt 32 - THREAD_ALIGN_SHIFT, stack_overflow #else subi r11, r1, INT_FRAME_SIZE /* use r1 if kernel */ @@ -116,114 +116,44 @@ .endm .macro SYSCALL_ENTRY trapno - mfspr r12,SPRN_SPRG_THREAD mfspr r9, SPRN_SRR1 -#ifdef CONFIG_VMAP_STACK - mfspr r11, SPRN_SRR0 - mtctr r11 - andi. r11, r9, MSR_PR + mfspr r10, SPRN_SRR0 + LOAD_REG_IMMEDIATE(r11, MSR_KERNEL) /* can take exceptions */ + lis r12, 1f@h + ori r12, r12, 1f@l + mtspr SPRN_SRR1, r11 + mtspr SPRN_SRR0, r12 + mfspr r12,SPRN_SPRG_THREAD mr r11, r1 lwz r1,TASK_STACK-THREAD(r12) - beq- 99f - addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE - li r10, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */ - mtmsr r10 - isync tovirt(r12, r12) + addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE + rfi +1: stw r11,GPR1(r1) stw r11,0(r1) mr r11, r1 -#else - andi. r11, r9, MSR_PR - lwz r11,TASK_STACK-THREAD(r12) - beq- 99f - addi r11, r11, THREAD_SIZE - INT_FRAME_SIZE - tophys(r11, r11) - stw r1,GPR1(r11) - stw r1,0(r11) - tovirt(r1, r11) /* set new kernel sp */ -#endif + stw r10,_NIP(r11) mflr r10 stw r10, _LINK(r11) -#ifdef CONFIG_VMAP_STACK - mfctr r10 -#else - mfspr r10,SPRN_SRR0 -#endif - stw r10,_NIP(r11) mfcr r10 rlwinm r10,r10,0,4,2 /* Clear SO bit in CR */ stw r10,_CCR(r11) /* save registers */ #ifdef CONFIG_40x rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ -#else -#ifdef CONFIG_VMAP_STACK - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~MSR_IR) /* can take exceptions */ -#else - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~(MSR_IR|MSR_DR)) /* can take exceptions */ -#endif - mtmsr r10 /* (except for mach check in rtas) */ #endif lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */ stw r2,GPR2(r11) addi r10,r10,STACK_FRAME_REGS_MARKER@l stw r9,_MSR(r11) - li r2, \trapno + 1 + li r2, \trapno stw r10,8(r11) stw r2,_TRAP(r11) SAVE_GPR(0, r11) SAVE_4GPRS(3, r11) SAVE_2GPRS(7, r11) - addi r11,r1,STACK_FRAME_OVERHEAD addi r2,r12,-THREAD - stw r11,PT_REGS(r12) -#if defined(CONFIG_40x) - /* Check to see if the dbcr0 register is set up to debug. Use the - internal debug mode bit to do this. */ - lwz r12,THREAD_DBCR0(r12) - andis. r12,r12,DBCR0_IDM@h -#endif - ACCOUNT_CPU_USER_ENTRY(r2, r11, r12) -#if defined(CONFIG_40x) - beq+ 3f - /* From user and task is ptraced - load up global dbcr0 */ - li r12,-1 /* clear all pending debug events */ - mtspr SPRN_DBSR,r12 - lis r11,global_dbcr0@ha - tophys(r11,r11) - addi r11,r11,global_dbcr0@l - lwz r12,0(r11) - mtspr SPRN_DBCR0,r12 - lwz r12,4(r11) - addi r12,r12,-1 - stw r12,4(r11) -#endif - -3: - tovirt_novmstack r2, r2 /* set r2 to current */ - lis r11, transfer_to_syscall@h - ori r11, r11, transfer_to_syscall@l -#ifdef CONFIG_TRACE_IRQFLAGS - /* - * If MSR is changing we need to keep interrupts disabled at this point - * otherwise we might risk taking an interrupt before we tell lockdep - * they are enabled. - */ - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL) - rlwimi r10, r9, 0, MSR_EE -#else - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE) -#endif -#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) - mtspr SPRN_NRI, r0 -#endif - mtspr SPRN_SRR1,r10 - mtspr SPRN_SRR0,r11 - rfi /* jump to handler, enable MMU */ -#ifdef CONFIG_40x - b . /* Prevent prefetch past rfi */ -#endif -99: b ret_from_kernel_syscall + b transfer_to_syscall /* jump to handler */ .endm .macro save_dar_dsisr_on_stack reg1, reg2, sp diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S index a1ae00689e0f..24724a7dad49 100644 --- a/arch/powerpc/kernel/head_40x.S +++ b/arch/powerpc/kernel/head_40x.S @@ -179,9 +179,9 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0300, DataStorage) EXCEPTION_PROLOG - mfspr r5, SPRN_ESR /* Grab the ESR, save it, pass arg3 */ + mfspr r5, SPRN_ESR /* Grab the ESR, save it */ stw r5, _ESR(r11) - mfspr r4, SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */ + mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */ stw r4, _DEAR(r11) EXC_XFER_LITE(0x300, handle_page_fault) @@ -191,9 +191,9 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0400, InstructionAccess) EXCEPTION_PROLOG - mr r4,r12 /* Pass SRR0 as arg2 */ - stw r4, _DEAR(r11) - li r5,0 /* Pass zero as arg3 */ + li r5,0 + stw r5, _ESR(r11) /* Zero ESR */ + stw r12, _DEAR(r11) /* SRR0 as DEAR */ EXC_XFER_LITE(0x400, handle_page_fault) /* 0x0500 - External Interrupt Exception */ @@ -476,6 +476,7 @@ _ENTRY(saved_ksp_limit) /* continue normal handling for a critical exception... */ 2: mfspr r4,SPRN_DBSR + stw r4,_ESR(r11) /* DebugException takes DBSR in _ESR */ addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_TEMPLATE(DebugException, 0x2002, \ (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S index 8e36718f3167..813fa305c33b 100644 --- a/arch/powerpc/kernel/head_44x.S +++ b/arch/powerpc/kernel/head_44x.S @@ -376,7 +376,7 @@ interrupt_base: /* Load the next available TLB index */ lwz r13,tlb_44x_index@l(r10) - bne 2f /* Bail if permission mismach */ + bne 2f /* Bail if permission mismatch */ /* Increment, rollover, and store TLB index */ addi r13,r13,1 @@ -471,7 +471,7 @@ interrupt_base: /* Load the next available TLB index */ lwz r13,tlb_44x_index@l(r10) - bne 2f /* Bail if permission mismach */ + bne 2f /* Bail if permission mismatch */ /* Increment, rollover, and store TLB index */ addi r13,r13,1 diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index 52702f3db6df..46dff3f9c31f 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -165,7 +165,7 @@ SystemCall: /* On the MPC8xx, this is a software emulation interrupt. It occurs * for all unimplemented and illegal instructions. */ - EXCEPTION(0x1000, SoftEmu, program_check_exception, EXC_XFER_STD) + EXCEPTION(0x1000, SoftEmu, emulation_assist_interrupt, EXC_XFER_STD) . = 0x1100 /* @@ -312,14 +312,14 @@ DataStoreTLBMiss: . = 0x1300 InstructionTLBError: EXCEPTION_PROLOG - mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ andis. r10,r9,SRR1_ISI_NOPT@h beq+ .Litlbie - tlbie r4 + tlbie r12 /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */ .Litlbie: - stw r4, _DAR(r11) + stw r12, _DAR(r11) + stw r5, _DSISR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* This is the data TLB error on the MPC8xx. This could be due to @@ -364,10 +364,9 @@ do_databreakpoint: addi r3,r1,STACK_FRAME_OVERHEAD mfspr r4,SPRN_BAR stw r4,_DAR(r11) -#ifdef CONFIG_VMAP_STACK - lwz r5,_DSISR(r11) -#else +#ifndef CONFIG_VMAP_STACK mfspr r5,SPRN_DSISR + stw r5,_DSISR(r11) #endif EXC_XFER_STD(0x1c00, do_break) diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S index 858fbc8b19f3..727fdab557c9 100644 --- a/arch/powerpc/kernel/head_book3s_32.S +++ b/arch/powerpc/kernel/head_book3s_32.S @@ -238,8 +238,8 @@ __secondary_hold_acknowledge: /* System reset */ /* core99 pmac starts the seconary here by changing the vector, and - putting it back to what it was (unknown_exception) when done. */ - EXCEPTION(0x100, Reset, unknown_exception, EXC_XFER_STD) + putting it back to what it was (unknown_async_exception) when done. */ + EXCEPTION(0x100, Reset, unknown_async_exception, EXC_XFER_STD) /* Machine check */ /* @@ -278,12 +278,6 @@ MachineCheck: 7: EXCEPTION_PROLOG_2 addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_CHRP -#ifdef CONFIG_VMAP_STACK - mfspr r4, SPRN_SPRG_THREAD - tovirt(r4, r4) - lwz r4, RTAS_SP(r4) - cmpwi cr1, r4, 0 -#endif beq cr1, machine_check_tramp twi 31, 0, 0 #else @@ -295,6 +289,7 @@ MachineCheck: DO_KVM 0x300 DataAccess: #ifdef CONFIG_VMAP_STACK +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION mtspr SPRN_SPRG_SCRATCH2,r10 mfspr r10, SPRN_SPRG_THREAD @@ -311,12 +306,14 @@ BEGIN_MMU_FTR_SECTION MMU_FTR_SECTION_ELSE b 1f ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) +#endif 1: EXCEPTION_PROLOG_0 handle_dar_dsisr=1 EXCEPTION_PROLOG_1 b handle_page_fault_tramp_1 #else /* CONFIG_VMAP_STACK */ EXCEPTION_PROLOG handle_dar_dsisr=1 get_and_save_dar_dsisr_on_stack r4, r5, r11 +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION andis. r0, r5, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h bne handle_page_fault_tramp_2 /* if not, try to put a PTE */ @@ -324,8 +321,11 @@ BEGIN_MMU_FTR_SECTION bl hash_page b handle_page_fault_tramp_1 MMU_FTR_SECTION_ELSE +#endif b handle_page_fault_tramp_2 +#ifdef CONFIG_PPC_BOOK3S_604 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) +#endif #endif /* CONFIG_VMAP_STACK */ /* Instruction access exception. */ @@ -341,12 +341,14 @@ InstructionAccess: mfspr r11, SPRN_SRR1 /* check whether user or kernel */ stw r11, SRR1(r10) mfcr r10 +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION andis. r11, r11, SRR1_ISI_NOPT@h /* no pte found? */ bne hash_page_isi .Lhash_page_isi_cont: mfspr r11, SPRN_SRR1 /* check whether user or kernel */ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) +#endif andi. r11, r11, MSR_PR EXCEPTION_PROLOG_1 @@ -357,13 +359,15 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) beq 1f /* if so, try to put a PTE */ li r3,0 /* into the hash table */ mr r4,r12 /* SRR0 is fault address */ +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION bl hash_page END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) +#endif #endif /* CONFIG_VMAP_STACK */ -1: mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ - stw r4, _DAR(r11) + stw r5, _DSISR(r11) + stw r12, _DAR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* External interrupt */ @@ -640,7 +644,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU) #endif #ifndef CONFIG_TAU_INT -#define TAUException unknown_exception +#define TAUException unknown_async_exception #endif EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception, EXC_XFER_STD) @@ -685,13 +689,16 @@ handle_page_fault_tramp_1: #ifdef CONFIG_VMAP_STACK EXCEPTION_PROLOG_2 handle_dar_dsisr=1 #endif - lwz r4, _DAR(r11) lwz r5, _DSISR(r11) /* fall through */ handle_page_fault_tramp_2: + andis. r0, r5, DSISR_DABRMATCH@h + bne- 1f EXC_XFER_LITE(0x300, handle_page_fault) +1: EXC_XFER_STD(0x300, do_break) #ifdef CONFIG_VMAP_STACK +#ifdef CONFIG_PPC_BOOK3S_604 .macro save_regs_thread thread stw r0, THR0(\thread) stw r3, THR3(\thread) @@ -763,6 +770,7 @@ fast_hash_page_return: mfspr r11, SPRN_SPRG_SCRATCH1 mfspr r10, SPRN_SPRG_SCRATCH0 rfi +#endif /* CONFIG_PPC_BOOK3S_604 */ stack_overflow: vmap_stack_overflow_exception diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h index 74e230c200fb..47857795f50a 100644 --- a/arch/powerpc/kernel/head_booke.h +++ b/arch/powerpc/kernel/head_booke.h @@ -106,10 +106,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) #endif mfspr r9, SPRN_SRR1 BOOKE_CLEAR_BTB(r11) - andi. r11, r9, MSR_PR lwz r11, TASK_STACK - THREAD(r10) rlwinm r12,r12,0,4,2 /* Clear SO bit in CR */ - beq- 99f ALLOC_STACK_FRAME(r11, THREAD_SIZE - INT_FRAME_SIZE) stw r12, _CCR(r11) /* save various registers */ mflr r12 @@ -124,60 +122,15 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) stw r2,GPR2(r11) addi r12, r12, STACK_FRAME_REGS_MARKER@l stw r9,_MSR(r11) - li r2, \trapno + 1 + li r2, \trapno stw r12, 8(r11) stw r2,_TRAP(r11) SAVE_GPR(0, r11) SAVE_4GPRS(3, r11) SAVE_2GPRS(7, r11) - addi r11,r1,STACK_FRAME_OVERHEAD addi r2,r10,-THREAD - stw r11,PT_REGS(r10) - /* Check to see if the dbcr0 register is set up to debug. Use the - internal debug mode bit to do this. */ - lwz r12,THREAD_DBCR0(r10) - andis. r12,r12,DBCR0_IDM@h - ACCOUNT_CPU_USER_ENTRY(r2, r11, r12) - beq+ 3f - /* From user and task is ptraced - load up global dbcr0 */ - li r12,-1 /* clear all pending debug events */ - mtspr SPRN_DBSR,r12 - lis r11,global_dbcr0@ha - tophys(r11,r11) - addi r11,r11,global_dbcr0@l -#ifdef CONFIG_SMP - lwz r10, TASK_CPU(r2) - slwi r10, r10, 3 - add r11, r11, r10 -#endif - lwz r12,0(r11) - mtspr SPRN_DBCR0,r12 - lwz r12,4(r11) - addi r12,r12,-1 - stw r12,4(r11) - -3: - tovirt(r2, r2) /* set r2 to current */ - lis r11, transfer_to_syscall@h - ori r11, r11, transfer_to_syscall@l -#ifdef CONFIG_TRACE_IRQFLAGS - /* - * If MSR is changing we need to keep interrupts disabled at this point - * otherwise we might risk taking an interrupt before we tell lockdep - * they are enabled. - */ - lis r10, MSR_KERNEL@h - ori r10, r10, MSR_KERNEL@l - rlwimi r10, r9, 0, MSR_EE -#else - lis r10, (MSR_KERNEL | MSR_EE)@h - ori r10, r10, (MSR_KERNEL | MSR_EE)@l -#endif - mtspr SPRN_SRR1,r10 - mtspr SPRN_SRR0,r11 - rfi /* jump to handler, enable MMU */ -99: b ret_from_kernel_syscall + b transfer_to_syscall /* jump to handler */ .endm /* To handle the additional exception priority levels on 40x and Book-E @@ -406,6 +359,7 @@ label: \ /* continue normal handling for a debug exception... */ \ 2: mfspr r4,SPRN_DBSR; \ + stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\ addi r3,r1,STACK_FRAME_OVERHEAD; \ EXC_XFER_TEMPLATE(DebugException, 0x2008, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), debug_transfer_to_handler, ret_from_debug_exc) @@ -459,6 +413,7 @@ label: \ /* continue normal handling for a critical exception... */ \ 2: mfspr r4,SPRN_DBSR; \ + stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\ addi r3,r1,STACK_FRAME_OVERHEAD; \ EXC_XFER_TEMPLATE(DebugException, 0x2002, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), crit_transfer_to_handler, ret_from_crit_exc) @@ -476,9 +431,7 @@ label: NORMAL_EXCEPTION_PROLOG(INST_STORAGE); \ mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \ stw r5,_ESR(r11); \ - mr r4,r12; /* Pass SRR0 as arg2 */ \ - stw r4, _DEAR(r11); \ - li r5,0; /* Pass zero as arg3 */ \ + stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \ EXC_XFER_LITE(0x0400, handle_page_fault) #define ALIGNMENT_EXCEPTION \ diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S index fdd4d274c245..3f4a40cccef5 100644 --- a/arch/powerpc/kernel/head_fsl_booke.S +++ b/arch/powerpc/kernel/head_fsl_booke.S @@ -364,12 +364,12 @@ interrupt_base: /* Data Storage Interrupt */ START_EXCEPTION(DataStorage) NORMAL_EXCEPTION_PROLOG(DATA_STORAGE) - mfspr r5,SPRN_ESR /* Grab the ESR, save it, pass arg3 */ + mfspr r5,SPRN_ESR /* Grab the ESR, save it */ stw r5,_ESR(r11) - mfspr r4,SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */ + mfspr r4,SPRN_DEAR /* Grab the DEAR, save it */ + stw r4, _DEAR(r11) andis. r10,r5,(ESR_ILK|ESR_DLK)@h bne 1f - stw r4, _DEAR(r11) EXC_XFER_LITE(0x0300, handle_page_fault) 1: addi r3,r1,STACK_FRAME_OVERHEAD diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S index 22f249b6f58d..f9e6d83e6720 100644 --- a/arch/powerpc/kernel/idle_book3s.S +++ b/arch/powerpc/kernel/idle_book3s.S @@ -52,28 +52,32 @@ _GLOBAL(isa300_idle_stop_mayloss) std r1,PACAR1(r13) mflr r4 mfcr r5 - /* use stack red zone rather than a new frame for saving regs */ - std r2,-8*0(r1) - std r14,-8*1(r1) - std r15,-8*2(r1) - std r16,-8*3(r1) - std r17,-8*4(r1) - std r18,-8*5(r1) - std r19,-8*6(r1) - std r20,-8*7(r1) - std r21,-8*8(r1) - std r22,-8*9(r1) - std r23,-8*10(r1) - std r24,-8*11(r1) - std r25,-8*12(r1) - std r26,-8*13(r1) - std r27,-8*14(r1) - std r28,-8*15(r1) - std r29,-8*16(r1) - std r30,-8*17(r1) - std r31,-8*18(r1) - std r4,-8*19(r1) - std r5,-8*20(r1) + /* + * Use the stack red zone rather than a new frame for saving regs since + * in the case of no GPR loss the wakeup code branches directly back to + * the caller without deallocating the stack frame first. + */ + std r2,-8*1(r1) + std r14,-8*2(r1) + std r15,-8*3(r1) + std r16,-8*4(r1) + std r17,-8*5(r1) + std r18,-8*6(r1) + std r19,-8*7(r1) + std r20,-8*8(r1) + std r21,-8*9(r1) + std r22,-8*10(r1) + std r23,-8*11(r1) + std r24,-8*12(r1) + std r25,-8*13(r1) + std r26,-8*14(r1) + std r27,-8*15(r1) + std r28,-8*16(r1) + std r29,-8*17(r1) + std r30,-8*18(r1) + std r31,-8*19(r1) + std r4,-8*20(r1) + std r5,-8*21(r1) /* 168 bytes */ PPC_STOP b . /* catch bugs */ @@ -89,8 +93,8 @@ _GLOBAL(isa300_idle_stop_mayloss) */ _GLOBAL(idle_return_gpr_loss) ld r1,PACAR1(r13) - ld r4,-8*19(r1) - ld r5,-8*20(r1) + ld r4,-8*20(r1) + ld r5,-8*21(r1) mtlr r4 mtcr r5 /* @@ -98,25 +102,25 @@ _GLOBAL(idle_return_gpr_loss) * from PACATOC. This could be avoided for that less common case * if KVM saved its r2. */ - ld r2,-8*0(r1) - ld r14,-8*1(r1) - ld r15,-8*2(r1) - ld r16,-8*3(r1) - ld r17,-8*4(r1) - ld r18,-8*5(r1) - ld r19,-8*6(r1) - ld r20,-8*7(r1) - ld r21,-8*8(r1) - ld r22,-8*9(r1) - ld r23,-8*10(r1) - ld r24,-8*11(r1) - ld r25,-8*12(r1) - ld r26,-8*13(r1) - ld r27,-8*14(r1) - ld r28,-8*15(r1) - ld r29,-8*16(r1) - ld r30,-8*17(r1) - ld r31,-8*18(r1) + ld r2,-8*1(r1) + ld r14,-8*2(r1) + ld r15,-8*3(r1) + ld r16,-8*4(r1) + ld r17,-8*5(r1) + ld r18,-8*6(r1) + ld r19,-8*7(r1) + ld r20,-8*8(r1) + ld r21,-8*9(r1) + ld r22,-8*10(r1) + ld r23,-8*11(r1) + ld r24,-8*12(r1) + ld r25,-8*13(r1) + ld r26,-8*14(r1) + ld r27,-8*15(r1) + ld r28,-8*16(r1) + ld r29,-8*17(r1) + ld r30,-8*18(r1) + ld r31,-8*19(r1) blr /* @@ -154,28 +158,32 @@ _GLOBAL(isa206_idle_insn_mayloss) std r1,PACAR1(r13) mflr r4 mfcr r5 - /* use stack red zone rather than a new frame for saving regs */ - std r2,-8*0(r1) - std r14,-8*1(r1) - std r15,-8*2(r1) - std r16,-8*3(r1) - std r17,-8*4(r1) - std r18,-8*5(r1) - std r19,-8*6(r1) - std r20,-8*7(r1) - std r21,-8*8(r1) - std r22,-8*9(r1) - std r23,-8*10(r1) - std r24,-8*11(r1) - std r25,-8*12(r1) - std r26,-8*13(r1) - std r27,-8*14(r1) - std r28,-8*15(r1) - std r29,-8*16(r1) - std r30,-8*17(r1) - std r31,-8*18(r1) - std r4,-8*19(r1) - std r5,-8*20(r1) + /* + * Use the stack red zone rather than a new frame for saving regs since + * in the case of no GPR loss the wakeup code branches directly back to + * the caller without deallocating the stack frame first. + */ + std r2,-8*1(r1) + std r14,-8*2(r1) + std r15,-8*3(r1) + std r16,-8*4(r1) + std r17,-8*5(r1) + std r18,-8*6(r1) + std r19,-8*7(r1) + std r20,-8*8(r1) + std r21,-8*9(r1) + std r22,-8*10(r1) + std r23,-8*11(r1) + std r24,-8*12(r1) + std r25,-8*13(r1) + std r26,-8*14(r1) + std r27,-8*15(r1) + std r28,-8*16(r1) + std r29,-8*17(r1) + std r30,-8*18(r1) + std r31,-8*19(r1) + std r4,-8*20(r1) + std r5,-8*21(r1) cmpwi r3,PNV_THREAD_NAP bne 1f IDLE_STATE_ENTER_SEQ_NORET(PPC_NAP) diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/interrupt.c similarity index 69% rename from arch/powerpc/kernel/syscall_64.c rename to arch/powerpc/kernel/interrupt.c index 7c85ed04a164..398cd86b6ada 100644 --- a/arch/powerpc/kernel/syscall_64.c +++ b/arch/powerpc/kernel/interrupt.c @@ -1,10 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#include #include +#include + #include #include #include +#include #include +#include #include #include #include @@ -24,16 +29,21 @@ notrace long system_call_exception(long r3, long r4, long r5, { syscall_fn f; + regs->orig_gpr3 = r3; + if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED); + CT_WARN_ON(ct_state() == CONTEXT_KERNEL); + user_exit_irqoff(); + trace_hardirqs_off(); /* finish reconciling */ - if (IS_ENABLED(CONFIG_PPC_BOOK3S)) + if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x)) BUG_ON(!(regs->msr & MSR_RI)); BUG_ON(!(regs->msr & MSR_PR)); BUG_ON(!FULL_REGS(regs)); - BUG_ON(regs->softe != IRQS_ENABLED); + BUG_ON(arch_irq_disabled_regs(regs)); #ifdef CONFIG_PPC_PKEY if (mmu_has_feature(MMU_FTR_PKEY)) { @@ -59,19 +69,15 @@ notrace long system_call_exception(long r3, long r4, long r5, isync(); } else #endif +#ifdef CONFIG_PPC64 kuap_check_amr(); +#endif + + booke_restore_dbcr0(); account_cpu_user_entry(); -#ifdef CONFIG_PPC_SPLPAR - if (IS_ENABLED(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && - firmware_has_feature(FW_FEATURE_SPLPAR)) { - struct lppaca *lp = local_paca->lppaca_ptr; - - if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx))) - accumulate_stolen_time(); - } -#endif + account_stolen_time(); /* * This is not required for the syscall exit path, but makes the @@ -79,12 +85,12 @@ notrace long system_call_exception(long r3, long r4, long r5, * frame, or if the unwinder was taught the first stack frame always * returns to user with IRQS_ENABLED, this store could be avoided! */ - regs->softe = IRQS_ENABLED; + irq_soft_mask_regs_set_state(regs, IRQS_ENABLED); local_irq_enable(); if (unlikely(current_thread_info()->flags & _TIF_SYSCALL_DOTRACE)) { - if (unlikely(regs->trap == 0x7ff0)) { + if (unlikely(trap_is_unsupported_scv(regs))) { /* Unsupported scv vector */ _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); return regs->gpr[3]; @@ -107,7 +113,7 @@ notrace long system_call_exception(long r3, long r4, long r5, r8 = regs->gpr[8]; } else if (unlikely(r0 >= NR_syscalls)) { - if (unlikely(regs->trap == 0x7ff0)) { + if (unlikely(trap_is_unsupported_scv(regs))) { /* Unsupported scv vector */ _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); return regs->gpr[3]; @@ -118,7 +124,7 @@ notrace long system_call_exception(long r3, long r4, long r5, /* May be faster to do array_index_nospec? */ barrier_nospec(); - if (unlikely(is_32bit_task())) { + if (unlikely(is_compat_task())) { f = (void *)compat_sys_call_table[r0]; r3 &= 0x00000000ffffffffULL; @@ -138,8 +144,12 @@ notrace long system_call_exception(long r3, long r4, long r5, /* * local irqs must be disabled. Returns false if the caller must re-enable * them, check for new work, and try again. + * + * This should be called with local irqs disabled, but if they were previously + * enabled when the interrupt handler returns (indicating a process-context / + * synchronous interrupt) then irqs_enabled should be true. */ -static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri) +static notrace inline bool __prep_irq_for_enabled_exit(bool clear_ri) { /* This must be done with RI=1 because tracing may touch vmaps */ trace_hardirqs_on(); @@ -149,6 +159,7 @@ static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri) __hard_EE_RI_disable(); else __hard_irq_disable(); +#ifdef CONFIG_PPC64 if (unlikely(lazy_irq_pending_nocheck())) { /* Took an interrupt, may have more exit work to do. */ if (clear_ri) @@ -160,10 +171,63 @@ static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri) } local_paca->irq_happened = 0; irq_soft_mask_set(IRQS_ENABLED); - +#endif return true; } +static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri, bool irqs_enabled) +{ + if (__prep_irq_for_enabled_exit(clear_ri)) + return true; + + /* + * Must replay pending soft-masked interrupts now. Don't just + * local_irq_enabe(); local_irq_disable(); because if we are + * returning from an asynchronous interrupt here, another one + * might hit after irqs are enabled, and it would exit via this + * same path allowing another to fire, and so on unbounded. + * + * If interrupts were enabled when this interrupt exited, + * indicating a process context (synchronous) interrupt, + * local_irq_enable/disable can be used, which will enable + * interrupts rather than keeping them masked (unclear how + * much benefit this is over just replaying for all cases, + * because we immediately disable again, so all we're really + * doing is allowing hard interrupts to execute directly for + * a very small time, rather than being masked and replayed). + */ + if (irqs_enabled) { + local_irq_enable(); + local_irq_disable(); + } else { + replay_soft_interrupts(); + } + + return false; +} + +static notrace void booke_load_dbcr0(void) +{ +#ifdef CONFIG_PPC_ADV_DEBUG_REGS + unsigned long dbcr0 = current->thread.debug.dbcr0; + + if (likely(!(dbcr0 & DBCR0_IDM))) + return; + + /* + * Check to see if the dbcr0 register is set up to debug. + * Use the internal debug mode bit to do this. + */ + mtmsr(mfmsr() & ~MSR_DE); + if (IS_ENABLED(CONFIG_PPC32)) { + isync(); + global_dbcr0[smp_processor_id()] = mfspr(SPRN_DBCR0); + } + mtspr(SPRN_DBCR0, dbcr0); + mtspr(SPRN_DBSR, -1); +#endif +} + /* * This should be called after a syscall returns, with r3 the return value * from the syscall. If this function returns non-zero, the system call @@ -177,20 +241,24 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv) { - unsigned long *ti_flagsp = ¤t_thread_info()->flags; unsigned long ti_flags; unsigned long ret = 0; + bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv; + CT_WARN_ON(ct_state() == CONTEXT_USER); + +#ifdef CONFIG_PPC64 kuap_check_amr(); +#endif regs->result = r3; /* Check whether the syscall is issued inside a restartable sequence */ rseq_syscall(regs); - ti_flags = *ti_flagsp; + ti_flags = current_thread_info()->flags; - if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && !scv) { + if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) { if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) { r3 = -r3; regs->ccr |= 0x10000000; /* Set SO bit in CR */ @@ -202,7 +270,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, ret = _TIF_RESTOREALL; else regs->gpr[3] = r3; - clear_bits(_TIF_PERSYSCALL_MASK, ti_flagsp); + clear_bits(_TIF_PERSYSCALL_MASK, ¤t_thread_info()->flags); } else { regs->gpr[3] = r3; } @@ -212,9 +280,10 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, ret |= _TIF_RESTOREALL; } -again: local_irq_disable(); - ti_flags = READ_ONCE(*ti_flagsp); + +again: + ti_flags = READ_ONCE(current_thread_info()->flags); while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { local_irq_enable(); if (ti_flags & _TIF_NEED_RESCHED) { @@ -230,7 +299,7 @@ again: do_notify_resume(regs, ti_flags); } local_irq_disable(); - ti_flags = READ_ONCE(*ti_flagsp); + ti_flags = READ_ONCE(current_thread_info()->flags); } if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) { @@ -257,9 +326,13 @@ again: } } + user_enter_irqoff(); + /* scv need not set RI=0 because SRRs are not used */ - if (unlikely(!prep_irq_for_enabled_exit(!scv))) { + if (unlikely(!__prep_irq_for_enabled_exit(is_not_scv))) { + user_exit_irqoff(); local_irq_enable(); + local_irq_disable(); goto again; } @@ -267,9 +340,11 @@ again: local_paca->tm_scratch = regs->msr; #endif + booke_load_dbcr0(); + account_cpu_user_exit(); -#ifdef CONFIG_PPC_BOOK3S /* BOOK3E not yet using this */ +#ifdef CONFIG_PPC_BOOK3S_64 /* BOOK3E and ppc32 not using this */ /* * We do this at the end so that we do context switch with KERNEL AMR */ @@ -278,33 +353,32 @@ again: return ret; } -#ifdef CONFIG_PPC_BOOK3S /* BOOK3E not yet using this */ +#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not yet using this */ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr) { -#ifdef CONFIG_PPC_BOOK3E - struct thread_struct *ts = ¤t->thread; -#endif - unsigned long *ti_flagsp = ¤t_thread_info()->flags; unsigned long ti_flags; unsigned long flags; unsigned long ret = 0; - if (IS_ENABLED(CONFIG_PPC_BOOK3S)) + if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x)) BUG_ON(!(regs->msr & MSR_RI)); BUG_ON(!(regs->msr & MSR_PR)); BUG_ON(!FULL_REGS(regs)); - BUG_ON(regs->softe != IRQS_ENABLED); + BUG_ON(arch_irq_disabled_regs(regs)); + CT_WARN_ON(ct_state() == CONTEXT_USER); /* * We don't need to restore AMR on the way back to userspace for KUAP. * AMR can only have been unlocked if we interrupted the kernel. */ +#ifdef CONFIG_PPC64 kuap_check_amr(); +#endif local_irq_save(flags); again: - ti_flags = READ_ONCE(*ti_flagsp); + ti_flags = READ_ONCE(current_thread_info()->flags); while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { local_irq_enable(); /* returning to user: may enable */ if (ti_flags & _TIF_NEED_RESCHED) { @@ -315,7 +389,7 @@ again: do_notify_resume(regs, ti_flags); } local_irq_disable(); - ti_flags = READ_ONCE(*ti_flagsp); + ti_flags = READ_ONCE(current_thread_info()->flags); } if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) { @@ -336,23 +410,16 @@ again: } } - if (unlikely(!prep_irq_for_enabled_exit(true))) { + user_enter_irqoff(); + + if (unlikely(!__prep_irq_for_enabled_exit(true))) { + user_exit_irqoff(); local_irq_enable(); local_irq_disable(); goto again; } -#ifdef CONFIG_PPC_BOOK3E - if (unlikely(ts->debug.dbcr0 & DBCR0_IDM)) { - /* - * Check to see if the dbcr0 register is set up to debug. - * Use the internal debug mode bit to do this. - */ - mtmsr(mfmsr() & ~MSR_DE); - mtspr(SPRN_DBCR0, ts->debug.dbcr0); - mtspr(SPRN_DBSR, -1); - } -#endif + booke_load_dbcr0(); #ifdef CONFIG_PPC_TRANSACTIONAL_MEM local_paca->tm_scratch = regs->msr; @@ -363,7 +430,9 @@ again: /* * We do this at the end so that we do context switch with KERNEL AMR */ +#ifdef CONFIG_PPC64 kuap_user_restore(regs); +#endif return ret; } @@ -372,56 +441,56 @@ void preempt_schedule_irq(void); notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr) { - unsigned long *ti_flagsp = ¤t_thread_info()->flags; unsigned long flags; unsigned long ret = 0; +#ifdef CONFIG_PPC64 unsigned long amr; +#endif - if (IS_ENABLED(CONFIG_PPC_BOOK3S) && unlikely(!(regs->msr & MSR_RI))) + if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) && + unlikely(!(regs->msr & MSR_RI))) unrecoverable_exception(regs); BUG_ON(regs->msr & MSR_PR); BUG_ON(!FULL_REGS(regs)); + /* + * CT_WARN_ON comes here via program_check_exception, + * so avoid recursion. + */ + if (TRAP(regs) != 0x700) + CT_WARN_ON(ct_state() == CONTEXT_USER); +#ifdef CONFIG_PPC64 amr = kuap_get_and_check_amr(); +#endif - if (unlikely(*ti_flagsp & _TIF_EMULATE_STACK_STORE)) { - clear_bits(_TIF_EMULATE_STACK_STORE, ti_flagsp); + if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) { + clear_bits(_TIF_EMULATE_STACK_STORE, ¤t_thread_info()->flags); ret = 1; } local_irq_save(flags); - if (regs->softe == IRQS_ENABLED) { + if (!arch_irq_disabled_regs(regs)) { /* Returning to a kernel context with local irqs enabled. */ WARN_ON_ONCE(!(regs->msr & MSR_EE)); again: if (IS_ENABLED(CONFIG_PREEMPT)) { /* Return to preemptible kernel context */ - if (unlikely(*ti_flagsp & _TIF_NEED_RESCHED)) { + if (unlikely(current_thread_info()->flags & _TIF_NEED_RESCHED)) { if (preempt_count() == 0) preempt_schedule_irq(); } } - if (unlikely(!prep_irq_for_enabled_exit(true))) { - /* - * Can't local_irq_restore to replay if we were in - * interrupt context. Must replay directly. - */ - if (irqs_disabled_flags(flags)) { - replay_soft_interrupts(); - } else { - local_irq_restore(flags); - local_irq_save(flags); - } - /* Took an interrupt, may have more exit work to do. */ + if (unlikely(!prep_irq_for_enabled_exit(true, !irqs_disabled_flags(flags)))) goto again; - } } else { /* Returning to a kernel context with local irqs disabled. */ __hard_EE_RI_disable(); +#ifdef CONFIG_PPC64 if (regs->msr & MSR_EE) local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; +#endif } @@ -434,7 +503,9 @@ again: * which would cause Read-After-Write stalls. Hence, we take the AMR * value from the check above. */ +#ifdef CONFIG_PPC64 kuap_kernel_restore(regs, amr); +#endif return ret; } diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index 5b69a6a72a0e..c00214a4355c 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,47 @@ #define DBG(...) +#ifdef CONFIG_IOMMU_DEBUGFS +static int iommu_debugfs_weight_get(void *data, u64 *val) +{ + struct iommu_table *tbl = data; + *val = bitmap_weight(tbl->it_map, tbl->it_size); + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(iommu_debugfs_fops_weight, iommu_debugfs_weight_get, NULL, "%llu\n"); + +static void iommu_debugfs_add(struct iommu_table *tbl) +{ + char name[10]; + struct dentry *liobn_entry; + + sprintf(name, "%08lx", tbl->it_index); + liobn_entry = debugfs_create_dir(name, iommu_debugfs_dir); + + debugfs_create_file_unsafe("weight", 0400, liobn_entry, tbl, &iommu_debugfs_fops_weight); + debugfs_create_ulong("it_size", 0400, liobn_entry, &tbl->it_size); + debugfs_create_ulong("it_page_shift", 0400, liobn_entry, &tbl->it_page_shift); + debugfs_create_ulong("it_reserved_start", 0400, liobn_entry, &tbl->it_reserved_start); + debugfs_create_ulong("it_reserved_end", 0400, liobn_entry, &tbl->it_reserved_end); + debugfs_create_ulong("it_indirect_levels", 0400, liobn_entry, &tbl->it_indirect_levels); + debugfs_create_ulong("it_level_size", 0400, liobn_entry, &tbl->it_level_size); +} + +static void iommu_debugfs_del(struct iommu_table *tbl) +{ + char name[10]; + struct dentry *liobn_entry; + + sprintf(name, "%08lx", tbl->it_index); + liobn_entry = debugfs_lookup(name, iommu_debugfs_dir); + if (liobn_entry) + debugfs_remove(liobn_entry); +} +#else +static void iommu_debugfs_add(struct iommu_table *tbl){} +static void iommu_debugfs_del(struct iommu_table *tbl){} +#endif + static int novmerge; static void __iommu_free(struct iommu_table *, dma_addr_t, unsigned int); @@ -725,6 +767,8 @@ struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid, welcomed = 1; } + iommu_debugfs_add(tbl); + return tbl; } @@ -744,6 +788,8 @@ static void iommu_table_free(struct kref *kref) return; } + iommu_debugfs_del(tbl); + iommu_table_release_pages(tbl); /* verify that table contains no entries */ diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index cc7a6271b6b4..d71fd10a1dd4 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -54,6 +54,7 @@ #include #include +#include #include #include #include @@ -65,6 +66,7 @@ #include #include #include +#include #ifdef CONFIG_PPC64 #include @@ -269,6 +271,31 @@ again: } } +#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_PPC_KUAP) +static inline void replay_soft_interrupts_irqrestore(void) +{ + unsigned long kuap_state = get_kuap(); + + /* + * Check if anything calls local_irq_enable/restore() when KUAP is + * disabled (user access enabled). We handle that case here by saving + * and re-locking AMR but we shouldn't get here in the first place, + * hence the warning. + */ + kuap_check_amr(); + + if (kuap_state != AMR_KUAP_BLOCKED) + set_kuap(AMR_KUAP_BLOCKED); + + replay_soft_interrupts(); + + if (kuap_state != AMR_KUAP_BLOCKED) + set_kuap(kuap_state); +} +#else +#define replay_soft_interrupts_irqrestore() replay_soft_interrupts() +#endif + notrace void arch_local_irq_restore(unsigned long mask) { unsigned char irq_happened; @@ -332,7 +359,7 @@ notrace void arch_local_irq_restore(unsigned long mask) irq_soft_mask_set(IRQS_ALL_DISABLED); trace_hardirqs_off(); - replay_soft_interrupts(); + replay_soft_interrupts_irqrestore(); local_paca->irq_happened = 0; trace_hardirqs_on(); @@ -644,8 +671,6 @@ void __do_irq(struct pt_regs *regs) { unsigned int irq; - irq_enter(); - trace_irq_entry(regs); /* @@ -665,11 +690,9 @@ void __do_irq(struct pt_regs *regs) generic_handle_irq(irq); trace_irq_exit(regs); - - irq_exit(); } -void do_IRQ(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(do_IRQ) { struct pt_regs *old_regs = set_irq_regs(regs); void *cursp, *irqsp, *sirqsp; diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c index 9f3e133b57b7..11f0cae086ed 100644 --- a/arch/powerpc/kernel/mce.c +++ b/arch/powerpc/kernel/mce.c @@ -17,22 +17,15 @@ #include #include #include +#include +#include #include #include #include +#include -static DEFINE_PER_CPU(int, mce_nest_count); -static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], mce_event); - -/* Queue for delayed MCE events. */ -static DEFINE_PER_CPU(int, mce_queue_count); -static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], mce_event_queue); - -/* Queue for delayed MCE UE events. */ -static DEFINE_PER_CPU(int, mce_ue_count); -static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], - mce_ue_event_queue); +#include "setup.h" static void machine_check_process_queued_event(struct irq_work *work); static void machine_check_ue_irq_work(struct irq_work *work); @@ -103,9 +96,10 @@ void save_mce_event(struct pt_regs *regs, long handled, struct mce_error_info *mce_err, uint64_t nip, uint64_t addr, uint64_t phys_addr) { - int index = __this_cpu_inc_return(mce_nest_count) - 1; - struct machine_check_event *mce = this_cpu_ptr(&mce_event[index]); + int index = local_paca->mce_info->mce_nest_count++; + struct machine_check_event *mce; + mce = &local_paca->mce_info->mce_event[index]; /* * Return if we don't have enough space to log mce event. * mce_nest_count may go beyond MAX_MC_EVT but that's ok, @@ -191,7 +185,7 @@ void save_mce_event(struct pt_regs *regs, long handled, */ int get_mce_event(struct machine_check_event *mce, bool release) { - int index = __this_cpu_read(mce_nest_count) - 1; + int index = local_paca->mce_info->mce_nest_count - 1; struct machine_check_event *mc_evt; int ret = 0; @@ -201,7 +195,7 @@ int get_mce_event(struct machine_check_event *mce, bool release) /* Check if we have MCE info to process. */ if (index < MAX_MC_EVT) { - mc_evt = this_cpu_ptr(&mce_event[index]); + mc_evt = &local_paca->mce_info->mce_event[index]; /* Copy the event structure and release the original */ if (mce) *mce = *mc_evt; @@ -211,7 +205,7 @@ int get_mce_event(struct machine_check_event *mce, bool release) } /* Decrement the count to free the slot. */ if (release) - __this_cpu_dec(mce_nest_count); + local_paca->mce_info->mce_nest_count--; return ret; } @@ -233,13 +227,14 @@ static void machine_check_ue_event(struct machine_check_event *evt) { int index; - index = __this_cpu_inc_return(mce_ue_count) - 1; + index = local_paca->mce_info->mce_ue_count++; /* If queue is full, just return for now. */ if (index >= MAX_MC_EVT) { - __this_cpu_dec(mce_ue_count); + local_paca->mce_info->mce_ue_count--; return; } - memcpy(this_cpu_ptr(&mce_ue_event_queue[index]), evt, sizeof(*evt)); + memcpy(&local_paca->mce_info->mce_ue_event_queue[index], + evt, sizeof(*evt)); /* Queue work to process this event later. */ irq_work_queue(&mce_ue_event_irq_work); @@ -256,13 +251,14 @@ void machine_check_queue_event(void) if (!get_mce_event(&evt, MCE_EVENT_RELEASE)) return; - index = __this_cpu_inc_return(mce_queue_count) - 1; + index = local_paca->mce_info->mce_queue_count++; /* If queue is full, just return for now. */ if (index >= MAX_MC_EVT) { - __this_cpu_dec(mce_queue_count); + local_paca->mce_info->mce_queue_count--; return; } - memcpy(this_cpu_ptr(&mce_event_queue[index]), &evt, sizeof(evt)); + memcpy(&local_paca->mce_info->mce_event_queue[index], + &evt, sizeof(evt)); /* Queue irq work to process this event later. */ irq_work_queue(&mce_event_process_work); @@ -289,9 +285,9 @@ static void machine_process_ue_event(struct work_struct *work) int index; struct machine_check_event *evt; - while (__this_cpu_read(mce_ue_count) > 0) { - index = __this_cpu_read(mce_ue_count) - 1; - evt = this_cpu_ptr(&mce_ue_event_queue[index]); + while (local_paca->mce_info->mce_ue_count > 0) { + index = local_paca->mce_info->mce_ue_count - 1; + evt = &local_paca->mce_info->mce_ue_event_queue[index]; blocking_notifier_call_chain(&mce_notifier_list, 0, evt); #ifdef CONFIG_MEMORY_FAILURE /* @@ -304,7 +300,7 @@ static void machine_process_ue_event(struct work_struct *work) */ if (evt->error_type == MCE_ERROR_TYPE_UE) { if (evt->u.ue_error.ignore_event) { - __this_cpu_dec(mce_ue_count); + local_paca->mce_info->mce_ue_count--; continue; } @@ -320,7 +316,7 @@ static void machine_process_ue_event(struct work_struct *work) "was generated\n"); } #endif - __this_cpu_dec(mce_ue_count); + local_paca->mce_info->mce_ue_count--; } } /* @@ -338,17 +334,17 @@ static void machine_check_process_queued_event(struct irq_work *work) * For now just print it to console. * TODO: log this error event to FSP or nvram. */ - while (__this_cpu_read(mce_queue_count) > 0) { - index = __this_cpu_read(mce_queue_count) - 1; - evt = this_cpu_ptr(&mce_event_queue[index]); + while (local_paca->mce_info->mce_queue_count > 0) { + index = local_paca->mce_info->mce_queue_count - 1; + evt = &local_paca->mce_info->mce_event_queue[index]; if (evt->error_type == MCE_ERROR_TYPE_UE && evt->u.ue_error.ignore_event) { - __this_cpu_dec(mce_queue_count); + local_paca->mce_info->mce_queue_count--; continue; } machine_check_print_event_info(evt, false, false); - __this_cpu_dec(mce_queue_count); + local_paca->mce_info->mce_queue_count--; } } @@ -588,15 +584,9 @@ EXPORT_SYMBOL_GPL(machine_check_print_event_info); * * regs->nip and regs->msr contains srr0 and ssr1. */ -long notrace machine_check_early(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI(machine_check_early) { long handled = 0; - u8 ftrace_enabled = this_cpu_get_ftrace_enabled(); - - this_cpu_set_ftrace_enabled(0); - /* Do not use nmi_enter/exit for pseries hpte guest */ - if (radix_enabled() || !firmware_has_feature(FW_FEATURE_LPAR)) - nmi_enter(); hv_nmi_check_nonrecoverable(regs); @@ -606,11 +596,6 @@ long notrace machine_check_early(struct pt_regs *regs) if (ppc_md.machine_check_early) handled = ppc_md.machine_check_early(regs); - if (radix_enabled() || !firmware_has_feature(FW_FEATURE_LPAR)) - nmi_exit(); - - this_cpu_set_ftrace_enabled(ftrace_enabled); - return handled; } @@ -722,7 +707,7 @@ long hmi_handle_debugtrig(struct pt_regs *regs) /* * Return values: */ -long hmi_exception_realmode(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode) { int ret; @@ -741,3 +726,24 @@ long hmi_exception_realmode(struct pt_regs *regs) return 1; } + +void __init mce_init(void) +{ + struct mce_info *mce_info; + u64 limit; + int i; + + limit = min(ppc64_bolted_size(), ppc64_rma_size); + for_each_possible_cpu(i) { + mce_info = memblock_alloc_try_nid(sizeof(*mce_info), + __alignof__(*mce_info), + MEMBLOCK_LOW_LIMIT, + limit, cpu_to_node(i)); + if (!mce_info) + goto err; + paca_ptrs[i]->mce_info = mce_info; + } + return; +err: + panic("Failed to allocate memory for MCE event data\n"); +} diff --git a/arch/powerpc/kernel/optprobes.c b/arch/powerpc/kernel/optprobes.c index 69bfe96884e2..7f7cdbeacd1a 100644 --- a/arch/powerpc/kernel/optprobes.c +++ b/arch/powerpc/kernel/optprobes.c @@ -141,30 +141,11 @@ void arch_remove_optimized_kprobe(struct optimized_kprobe *op) } } -/* - * emulate_step() requires insn to be emulated as - * second parameter. Load register 'r4' with the - * instruction. - */ -void patch_imm32_load_insns(unsigned int val, kprobe_opcode_t *addr) -{ - /* addis r4,0,(insn)@h */ - patch_instruction((struct ppc_inst *)addr, - ppc_inst(PPC_INST_ADDIS | ___PPC_RT(4) | - ((val >> 16) & 0xffff))); - addr++; - - /* ori r4,r4,(insn)@l */ - patch_instruction((struct ppc_inst *)addr, - ppc_inst(PPC_INST_ORI | ___PPC_RA(4) | - ___PPC_RS(4) | (val & 0xffff))); -} - /* * Generate instructions to load provided immediate 64-bit value * to register 'reg' and patch these instructions at 'addr'. */ -void patch_imm64_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr) +static void patch_imm64_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr) { /* lis reg,(op)@highest */ patch_instruction((struct ppc_inst *)addr, diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index 2b555997b295..001e90cd8948 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -1699,3 +1699,13 @@ static void fixup_hide_host_resource_fsl(struct pci_dev *dev) } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MOTOROLA, PCI_ANY_ID, fixup_hide_host_resource_fsl); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_FREESCALE, PCI_ANY_ID, fixup_hide_host_resource_fsl); + + +static int __init discover_phbs(void) +{ + if (ppc_md.discover_phbs) + ppc_md.discover_phbs(); + + return 0; +} +core_initcall(discover_phbs); diff --git a/arch/powerpc/kernel/pci_dn.c b/arch/powerpc/kernel/pci_dn.c index e99b7c547d7e..61571ae23953 100644 --- a/arch/powerpc/kernel/pci_dn.c +++ b/arch/powerpc/kernel/pci_dn.c @@ -443,46 +443,6 @@ void *pci_traverse_device_nodes(struct device_node *start, } EXPORT_SYMBOL_GPL(pci_traverse_device_nodes); -static struct pci_dn *pci_dn_next_one(struct pci_dn *root, - struct pci_dn *pdn) -{ - struct list_head *next = pdn->child_list.next; - - if (next != &pdn->child_list) - return list_entry(next, struct pci_dn, list); - - while (1) { - if (pdn == root) - return NULL; - - next = pdn->list.next; - if (next != &pdn->parent->child_list) - break; - - pdn = pdn->parent; - } - - return list_entry(next, struct pci_dn, list); -} - -void *traverse_pci_dn(struct pci_dn *root, - void *(*fn)(struct pci_dn *, void *), - void *data) -{ - struct pci_dn *pdn = root; - void *ret; - - /* Only scan the child nodes */ - for (pdn = pci_dn_next_one(root, pdn); pdn; - pdn = pci_dn_next_one(root, pdn)) { - ret = fn(pdn, data); - if (ret) - return ret; - } - - return NULL; -} - static void *add_pdn(struct device_node *dn, void *data) { struct pci_controller *hose = data; @@ -521,28 +481,6 @@ void pci_devs_phb_init_dynamic(struct pci_controller *phb) pci_traverse_device_nodes(dn, add_pdn, phb); } -/** - * pci_devs_phb_init - Initialize phbs and pci devs under them. - * - * This routine walks over all phb's (pci-host bridges) on the - * system, and sets up assorted pci-related structures - * (including pci info in the device node structs) for each - * pci device found underneath. This routine runs once, - * early in the boot sequence. - */ -static int __init pci_devs_phb_init(void) -{ - struct pci_controller *phb, *tmp; - - /* This must be done first so the device nodes have valid pci info! */ - list_for_each_entry_safe(phb, tmp, &hose_list, list_node) - pci_devs_phb_init_dynamic(phb); - - return 0; -} - -core_initcall(pci_devs_phb_init); - static void pci_dev_pdn_setup(struct pci_dev *pdev) { struct pci_dn *pdn; diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index a66f435dabbf..3231c2df9e26 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -41,6 +41,7 @@ #include #include +#include #include #include #include @@ -659,11 +660,10 @@ static void do_break_handler(struct pt_regs *regs) } } -void do_break (struct pt_regs *regs, unsigned long address, - unsigned long error_code) +DEFINE_INTERRUPT_HANDLER(do_break) { current->thread.trap_nr = TRAP_HWBKPT; - if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code, + if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, regs->dsisr, 11, SIGSEGV) == NOTIFY_STOP) return; @@ -681,7 +681,7 @@ void do_break (struct pt_regs *regs, unsigned long address, do_break_handler(regs); /* Deliver the signal to userspace */ - force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)address); + force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)regs->dar); } #endif /* CONFIG_PPC_ADV_DEBUG_REGS */ @@ -1670,7 +1670,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, /* Copy registers */ sp -= sizeof(struct pt_regs); childregs = (struct pt_regs *) sp; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ memset(childregs, 0, sizeof(struct pt_regs)); childregs->gpr[1] = sp + sizeof(struct pt_regs); @@ -2047,6 +2047,9 @@ static inline int valid_emergency_stack(unsigned long sp, struct task_struct *p, unsigned long stack_page; unsigned long cpu = task_cpu(p); + if (!paca_ptrs) + return 0; + stack_page = (unsigned long)paca_ptrs[cpu]->emergency_sp - THREAD_SIZE; if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes) return 1; @@ -2176,7 +2179,7 @@ void show_stack(struct task_struct *tsk, unsigned long *stack, * See if this is an exception frame. * We look for the "regshere" marker in the current frame. */ - if (validate_sp(sp, tsk, STACK_INT_FRAME_SIZE) + if (validate_sp(sp, tsk, STACK_FRAME_WITH_PT_REGS) && stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) { struct pt_regs *regs = (struct pt_regs *) (sp + STACK_FRAME_OVERHEAD); diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index ae3c41730367..9a4797d1d40d 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -707,7 +707,7 @@ static void __init save_fscr_to_task(void) init_task.thread.fscr = mfspr(SPRN_FSCR); } #else -static inline void save_fscr_to_task(void) {}; +static inline void save_fscr_to_task(void) {} #endif diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index e9d4eb6144e1..ccf77b985c8f 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -1331,14 +1331,10 @@ static void __init prom_check_platform_support(void) if (prop_len > sizeof(vec)) prom_printf("WARNING: ibm,arch-vec-5-platform-support longer than expected (len: %d)\n", prop_len); - prom_getprop(prom.chosen, "ibm,arch-vec-5-platform-support", - &vec, sizeof(vec)); - for (i = 0; i < sizeof(vec); i += 2) { - prom_debug("%d: index = 0x%x val = 0x%x\n", i / 2 - , vec[i] - , vec[i + 1]); - prom_parse_platform_support(vec[i], vec[i + 1], - &supported); + prom_getprop(prom.chosen, "ibm,arch-vec-5-platform-support", &vec, sizeof(vec)); + for (i = 0; i < prop_len; i += 2) { + prom_debug("%d: index = 0x%x val = 0x%x\n", i / 2, vec[i], vec[i + 1]); + prom_parse_platform_support(vec[i], vec[i + 1], &supported); } } diff --git a/arch/powerpc/kernel/ptrace/ptrace.c b/arch/powerpc/kernel/ptrace/ptrace.c index 3d44b73adb83..4f3d4ff3728c 100644 --- a/arch/powerpc/kernel/ptrace/ptrace.c +++ b/arch/powerpc/kernel/ptrace/ptrace.c @@ -262,8 +262,6 @@ long do_syscall_trace_enter(struct pt_regs *regs) { u32 flags; - user_exit(); - flags = READ_ONCE(current_thread_info()->flags) & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE); @@ -340,8 +338,6 @@ void do_syscall_trace_leave(struct pt_regs *regs) step = test_thread_flag(TIF_SINGLESTEP); if (step || test_thread_flag(TIF_SYSCALL_TRACE)) tracehook_report_syscall_exit(regs, step); - - user_enter(); } void __init pt_regs_check(void); diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index 71f38e9248be..bee984b1887b 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -64,6 +64,7 @@ #include #include #include +#include #include "setup.h" @@ -237,18 +238,17 @@ static int show_cpuinfo(struct seq_file *m, void *v) maj = (pvr >> 8) & 0xFF; min = pvr & 0xFF; - seq_printf(m, "processor\t: %lu\n", cpu_id); - seq_printf(m, "cpu\t\t: "); + seq_printf(m, "processor\t: %lu\ncpu\t\t: ", cpu_id); if (cur_cpu_spec->pvr_mask && cur_cpu_spec->cpu_name) - seq_printf(m, "%s", cur_cpu_spec->cpu_name); + seq_puts(m, cur_cpu_spec->cpu_name); else seq_printf(m, "unknown (%08x)", pvr); if (cpu_has_feature(CPU_FTR_ALTIVEC)) - seq_printf(m, ", altivec supported"); + seq_puts(m, ", altivec supported"); - seq_printf(m, "\n"); + seq_putc(m, '\n'); #ifdef CONFIG_TAU if (cpu_has_feature(CPU_FTR_TAU)) { @@ -327,7 +327,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) seq_printf(m, "bogomips\t: %lu.%02lu\n", loops_per_jiffy / (500000 / HZ), (loops_per_jiffy / (5000 / HZ)) % 100); - seq_printf(m, "\n"); + seq_putc(m, '\n'); /* If this is the last cpu, print the summary */ if (cpumask_next(cpu_id, cpu_online_mask) >= nr_cpu_ids) @@ -938,6 +938,7 @@ void __init setup_arch(char **cmdline_p) exc_lvl_early_init(); emergency_stack_init(); + mce_init(); smp_release_cpus(); initmem_init(); diff --git a/arch/powerpc/kernel/setup.h b/arch/powerpc/kernel/setup.h index 2dd0d9cb5a20..84058bbc8fe9 100644 --- a/arch/powerpc/kernel/setup.h +++ b/arch/powerpc/kernel/setup.h @@ -14,31 +14,31 @@ void irqstack_early_init(void); #ifdef CONFIG_PPC32 void setup_power_save(void); #else -static inline void setup_power_save(void) { }; +static inline void setup_power_save(void) { } #endif #if defined(CONFIG_PPC64) && defined(CONFIG_SMP) void check_smt_enabled(void); #else -static inline void check_smt_enabled(void) { }; +static inline void check_smt_enabled(void) { } #endif #if defined(CONFIG_PPC_BOOK3E) && defined(CONFIG_SMP) void setup_tlb_core_data(void); #else -static inline void setup_tlb_core_data(void) { }; +static inline void setup_tlb_core_data(void) { } #endif #if defined(CONFIG_PPC_BOOK3E) || defined(CONFIG_BOOKE) || defined(CONFIG_40x) void exc_lvl_early_init(void); #else -static inline void exc_lvl_early_init(void) { }; +static inline void exc_lvl_early_init(void) { } #endif #if defined(CONFIG_PPC64) || defined(CONFIG_VMAP_STACK) void emergency_stack_init(void); #else -static inline void emergency_stack_init(void) { }; +static inline void emergency_stack_init(void) { } #endif #ifdef CONFIG_PPC64 @@ -55,7 +55,7 @@ extern unsigned long spr_default_dscr; #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE void kvm_cma_reserve(void); #else -static inline void kvm_cma_reserve(void) { }; +static inline void kvm_cma_reserve(void) { } #endif #ifdef CONFIG_TAU diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index c28e949cc222..560ed8b975e7 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -67,6 +67,7 @@ #include #include #include +#include #include "setup.h" @@ -258,7 +259,7 @@ static void cpu_ready_for_interrupts(void) unsigned long spr_default_dscr = 0; -void __init record_spr_defaults(void) +static void __init record_spr_defaults(void) { if (early_cpu_has_feature(CPU_FTR_DSCR)) spr_default_dscr = mfspr(SPRN_DSCR); @@ -1008,7 +1009,7 @@ void rfi_flush_enable(bool enable) rfi_flush = enable; } -void entry_flush_enable(bool enable) +static void entry_flush_enable(bool enable) { if (enable) { do_entry_flush_fixups(enabled_flush_types); @@ -1020,7 +1021,7 @@ void entry_flush_enable(bool enable) entry_flush = enable; } -void uaccess_flush_enable(bool enable) +static void uaccess_flush_enable(bool enable) { if (enable) { do_uaccess_flush_fixups(enabled_flush_types); diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c index 53782aa60ade..9ded046edb0e 100644 --- a/arch/powerpc/kernel/signal.c +++ b/arch/powerpc/kernel/signal.c @@ -282,8 +282,6 @@ static void do_signal(struct task_struct *tsk) void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) { - user_exit(); - if (thread_info_flags & _TIF_UPROBE) uprobe_notify_resume(regs); @@ -299,8 +297,6 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) tracehook_notify_resume(regs); rseq_handle_notify_resume(NULL, regs); } - - user_enter(); } static unsigned long get_tm_stackpointer(struct task_struct *tsk) diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c index 934cbdf6dd10..75ee918a120a 100644 --- a/arch/powerpc/kernel/signal_32.c +++ b/arch/powerpc/kernel/signal_32.c @@ -929,8 +929,9 @@ int handle_signal32(struct ksignal *ksig, sigset_t *oldset, regs->gpr[3] = ksig->sig; regs->gpr[4] = (unsigned long) sc; regs->nip = (unsigned long)ksig->ka.sa.sa_handler; - /* enter the signal handler in big-endian mode */ + /* enter the signal handler in native-endian mode */ regs->msr &= ~MSR_LE; + regs->msr |= (MSR_KERNEL & MSR_LE); return 0; failed: diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 9e2246e80efd..5a4d59a1070d 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -576,7 +577,7 @@ void tick_broadcast(const struct cpumask *mask) #endif #ifdef CONFIG_DEBUGGER -void debugger_ipi_callback(struct pt_regs *regs) +static void debugger_ipi_callback(struct pt_regs *regs) { debugger_ipi(regs); } diff --git a/arch/powerpc/kernel/sys_ppc32.c b/arch/powerpc/kernel/sys_ppc32.c index d36c6391eaf5..16ff0399a257 100644 --- a/arch/powerpc/kernel/sys_ppc32.c +++ b/arch/powerpc/kernel/sys_ppc32.c @@ -59,57 +59,64 @@ unsigned long compat_sys_mmap2(unsigned long addr, size_t len, /* * long long munging: * The 32 bit ABI passes long longs in an odd even register pair. + * High and low parts are swapped depending on endian mode, + * so define a macro (similar to mips linux32) to handle that. */ +#ifdef __LITTLE_ENDIAN__ +#define merge_64(low, high) ((u64)high << 32) | low +#else +#define merge_64(high, low) ((u64)high << 32) | low +#endif compat_ssize_t compat_sys_pread64(unsigned int fd, char __user *ubuf, compat_size_t count, - u32 reg6, u32 poshi, u32 poslo) + u32 reg6, u32 pos1, u32 pos2) { - return ksys_pread64(fd, ubuf, count, ((loff_t)poshi << 32) | poslo); + return ksys_pread64(fd, ubuf, count, merge_64(pos1, pos2)); } compat_ssize_t compat_sys_pwrite64(unsigned int fd, const char __user *ubuf, compat_size_t count, - u32 reg6, u32 poshi, u32 poslo) + u32 reg6, u32 pos1, u32 pos2) { - return ksys_pwrite64(fd, ubuf, count, ((loff_t)poshi << 32) | poslo); + return ksys_pwrite64(fd, ubuf, count, merge_64(pos1, pos2)); } -compat_ssize_t compat_sys_readahead(int fd, u32 r4, u32 offhi, u32 offlo, u32 count) +compat_ssize_t compat_sys_readahead(int fd, u32 r4, u32 offset1, u32 offset2, u32 count) { - return ksys_readahead(fd, ((loff_t)offhi << 32) | offlo, count); + return ksys_readahead(fd, merge_64(offset1, offset2), count); } asmlinkage int compat_sys_truncate64(const char __user * path, u32 reg4, - unsigned long high, unsigned long low) + unsigned long len1, unsigned long len2) { - return ksys_truncate(path, (high << 32) | low); + return ksys_truncate(path, merge_64(len1, len2)); } -asmlinkage long compat_sys_fallocate(int fd, int mode, u32 offhi, u32 offlo, - u32 lenhi, u32 lenlo) +asmlinkage long compat_sys_fallocate(int fd, int mode, u32 offset1, u32 offset2, + u32 len1, u32 len2) { - return ksys_fallocate(fd, mode, ((loff_t)offhi << 32) | offlo, - ((loff_t)lenhi << 32) | lenlo); + return ksys_fallocate(fd, mode, ((loff_t)offset1 << 32) | offset2, + merge_64(len1, len2)); } -asmlinkage int compat_sys_ftruncate64(unsigned int fd, u32 reg4, unsigned long high, - unsigned long low) +asmlinkage int compat_sys_ftruncate64(unsigned int fd, u32 reg4, unsigned long len1, + unsigned long len2) { - return ksys_ftruncate(fd, (high << 32) | low); + return ksys_ftruncate(fd, merge_64(len1, len2)); } -long ppc32_fadvise64(int fd, u32 unused, u32 offset_high, u32 offset_low, +long ppc32_fadvise64(int fd, u32 unused, u32 offset1, u32 offset2, size_t len, int advice) { - return ksys_fadvise64_64(fd, (u64)offset_high << 32 | offset_low, len, + return ksys_fadvise64_64(fd, merge_64(offset1, offset2), len, advice); } asmlinkage long compat_sys_sync_file_range2(int fd, unsigned int flags, - unsigned offset_hi, unsigned offset_lo, - unsigned nbytes_hi, unsigned nbytes_lo) + unsigned offset1, unsigned offset2, + unsigned nbytes1, unsigned nbytes2) { - loff_t offset = ((loff_t)offset_hi << 32) | offset_lo; - loff_t nbytes = ((loff_t)nbytes_hi << 32) | nbytes_lo; + loff_t offset = merge_64(offset1, offset2); + loff_t nbytes = merge_64(nbytes1, nbytes2); return ksys_sync_file_range(fd, offset, nbytes, flags); } diff --git a/arch/powerpc/kernel/syscalls/Makefile b/arch/powerpc/kernel/syscalls/Makefile index 27b48954808d..9e3be295dbba 100644 --- a/arch/powerpc/kernel/syscalls/Makefile +++ b/arch/powerpc/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -22,31 +22,31 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_offset_$(basetarget))' syshdr_abis_unistd_32 := common,nospu,32 -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abis_unistd_64 := common,nospu,64 -$(uapi)/unistd_64.h: $(syscall) $(syshdr) +$(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) systbl_abis_syscall_table_32 := common,nospu,32 systbl_abi_syscall_table_32 := 32 -$(kapi)/syscall_table_32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_64 := common,nospu,64 systbl_abi_syscall_table_64 := 64 -$(kapi)/syscall_table_64.h: $(syscall) $(systbl) +$(kapi)/syscall_table_64.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_c32 := common,nospu,32 systbl_abi_syscall_table_c32 := c32 -$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_spu := common,spu systbl_abi_syscall_table_spu := spu -$(kapi)/syscall_table_spu.h: $(syscall) $(systbl) +$(kapi)/syscall_table_spu.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h unistd_64.h @@ -55,9 +55,10 @@ kapisyshdr-y += syscall_table_32.h \ syscall_table_c32.h \ syscall_table_spu.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl index f744eb5cba88..0b2480cf3e47 100644 --- a/arch/powerpc/kernel/syscalls/syscall.tbl +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -9,9 +9,7 @@ # 0 nospu restart_syscall sys_restart_syscall 1 nospu exit sys_exit -2 32 fork ppc_fork sys_fork -2 64 fork sys_fork -2 spu fork sys_ni_syscall +2 nospu fork sys_fork 3 common read sys_read 4 common write sys_write 5 common open sys_open compat_sys_open @@ -160,9 +158,7 @@ 119 32 sigreturn sys_sigreturn compat_sys_sigreturn 119 64 sigreturn sys_ni_syscall 119 spu sigreturn sys_ni_syscall -120 32 clone ppc_clone sys_clone -120 64 clone sys_clone -120 spu clone sys_ni_syscall +120 nospu clone sys_clone 121 common setdomainname sys_setdomainname 122 common uname sys_newuname 123 common modify_ldt sys_ni_syscall @@ -244,9 +240,7 @@ 186 spu sendfile sys_sendfile64 187 common getpmsg sys_ni_syscall 188 common putpmsg sys_ni_syscall -189 32 vfork ppc_vfork sys_vfork -189 64 vfork sys_vfork -189 spu vfork sys_ni_syscall +189 nospu vfork sys_vfork 190 common ugetrlimit sys_getrlimit compat_sys_getrlimit 191 common readahead sys_readahead compat_sys_readahead 192 32 mmap2 sys_mmap2 compat_sys_mmap2 @@ -322,9 +316,7 @@ 248 32 clock_nanosleep sys_clock_nanosleep_time32 248 64 clock_nanosleep sys_clock_nanosleep 248 spu clock_nanosleep sys_clock_nanosleep -249 32 swapcontext ppc_swapcontext compat_sys_swapcontext -249 64 swapcontext sys_swapcontext -249 spu swapcontext sys_ni_syscall +249 nospu swapcontext sys_swapcontext compat_sys_swapcontext 250 common tgkill sys_tgkill 251 32 utimes sys_utimes_time32 251 64 utimes sys_utimes @@ -522,12 +514,11 @@ 432 common fsmount sys_fsmount 433 common fspick sys_fspick 434 common pidfd_open sys_pidfd_open -435 32 clone3 ppc_clone3 sys_clone3 -435 64 clone3 sys_clone3 -435 spu clone3 sys_ni_syscall +435 nospu clone3 sys_clone3 436 common close_range sys_close_range 437 common openat2 sys_openat2 438 common pidfd_getfd sys_pidfd_getfd 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/powerpc/kernel/tau_6xx.c b/arch/powerpc/kernel/tau_6xx.c index 0b4694b8d248..6c31af7f4fa8 100644 --- a/arch/powerpc/kernel/tau_6xx.c +++ b/arch/powerpc/kernel/tau_6xx.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -100,16 +101,13 @@ static void TAUupdate(int cpu) * with interrupts disabled */ -void TAUException(struct pt_regs * regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(TAUException) { int cpu = smp_processor_id(); - irq_enter(); tau[cpu].interrupts++; TAUupdate(cpu); - - irq_exit(); } #endif /* CONFIG_TAU_INT */ diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 67feb3524460..b67d93a609a2 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -53,9 +53,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -570,7 +572,7 @@ void arch_irq_work_raise(void) * timer_interrupt - gets called when the decrementer overflows, * with interrupts disabled. */ -void timer_interrupt(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt) { struct clock_event_device *evt = this_cpu_ptr(&decrementers); u64 *next_tb = this_cpu_ptr(&decrementers_next_tb); @@ -609,7 +611,7 @@ void timer_interrupt(struct pt_regs *regs) #endif old_regs = set_irq_regs(regs); - irq_enter(); + trace_timer_interrupt_entry(regs); if (test_irq_work_pending()) { @@ -634,7 +636,7 @@ void timer_interrupt(struct pt_regs *regs) } trace_timer_interrupt_exit(regs); - irq_exit(); + set_irq_regs(old_regs); } EXPORT_SYMBOL(timer_interrupt); @@ -1030,6 +1032,7 @@ void __init time_init(void) tick_setup_hrtimer_broadcast(); of_clk_init(NULL); + enable_sched_clock_irqtime(); } /* diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index 3ec7b443fe6b..1583fd1c6010 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -342,8 +343,8 @@ static bool exception_common(int signr, struct pt_regs *regs, int code, show_signal_msg(signr, regs, code, addr); - if (arch_irqs_disabled() && !arch_irq_disabled_regs(regs)) - local_irq_enable(); + if (arch_irqs_disabled()) + interrupt_cond_local_irq_enable(regs); current->thread.trap_nr = code; @@ -430,16 +431,10 @@ nonrecoverable: regs->msr &= ~MSR_RI; #endif } - -void system_reset_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI(system_reset_exception) { unsigned long hsrr0, hsrr1; bool saved_hsrrs = false; - u8 ftrace_enabled = this_cpu_get_ftrace_enabled(); - - this_cpu_set_ftrace_enabled(0); - - nmi_enter(); /* * System reset can interrupt code where HSRRs are live and MSR[RI]=1. @@ -503,19 +498,20 @@ out: die("Unrecoverable nested System Reset", regs, SIGABRT); #endif /* Must die if the interrupt is not recoverable */ - if (!(regs->msr & MSR_RI)) + if (!(regs->msr & MSR_RI)) { + /* For the reason explained in die_mce, nmi_exit before die */ + nmi_exit(); die("Unrecoverable System Reset", regs, SIGABRT); + } if (saved_hsrrs) { mtspr(SPRN_HSRR0, hsrr0); mtspr(SPRN_HSRR1, hsrr1); } - nmi_exit(); - - this_cpu_set_ftrace_enabled(ftrace_enabled); - /* What should we do here? We could issue a shutdown or hard reset. */ + + return 0; } /* @@ -788,24 +784,34 @@ int machine_check_generic(struct pt_regs *regs) } #endif /* everything else */ -void machine_check_exception(struct pt_regs *regs) +void die_mce(const char *str, struct pt_regs *regs, long err) +{ + /* + * The machine check wants to kill the interrupted context, but + * do_exit() checks for in_interrupt() and panics in that case, so + * exit the irq/nmi before calling die. + */ + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + irq_exit(); + else + nmi_exit(); + die(str, regs, err); +} + +/* + * BOOK3S_64 does not call this handler as a non-maskable interrupt + * (it uses its own early real-mode handler to handle the MCE proper + * and then raises irq_work to call this handler when interrupts are + * enabled). + */ +#ifdef CONFIG_PPC_BOOK3S_64 +DEFINE_INTERRUPT_HANDLER_ASYNC(machine_check_exception) +#else +DEFINE_INTERRUPT_HANDLER_NMI(machine_check_exception) +#endif { int recover = 0; - /* - * BOOK3S_64 does not call this handler as a non-maskable interrupt - * (it uses its own early real-mode handler to handle the MCE proper - * and then raises irq_work to call this handler when interrupts are - * enabled). - * - * This is silly. The BOOK3S_64 should just call a different function - * rather than expecting semantics to magically change. Something - * like 'non_nmi_machine_check_exception()', perhaps? - */ - const bool nmi = !IS_ENABLED(CONFIG_PPC_BOOK3S_64); - - if (nmi) nmi_enter(); - __this_cpu_inc(irq_stat.mce_exceptions); add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); @@ -830,21 +836,21 @@ void machine_check_exception(struct pt_regs *regs) if (check_io_access(regs)) goto bail; - if (nmi) nmi_exit(); - - die("Machine check", regs, SIGBUS); - - /* Must die if the interrupt is not recoverable */ - if (!(regs->msr & MSR_RI)) - die("Unrecoverable Machine check", regs, SIGBUS); - - return; + die_mce("Machine check", regs, SIGBUS); bail: - if (nmi) nmi_exit(); + /* Must die if the interrupt is not recoverable */ + if (!(regs->msr & MSR_RI)) + die_mce("Unrecoverable Machine check", regs, SIGBUS); + +#ifdef CONFIG_PPC_BOOK3S_64 + return; +#else + return 0; +#endif } -void SMIException(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(SMIException) /* async? */ { die("System Management Interrupt", regs, SIGABRT); } @@ -1030,12 +1036,11 @@ static void p9_hmi_special_emu(struct pt_regs *regs) } #endif /* CONFIG_VSX */ -void handle_hmi_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(handle_hmi_exception) { struct pt_regs *old_regs; old_regs = set_irq_regs(regs); - irq_enter(); #ifdef CONFIG_VSX /* Real mode flagged P9 special emu is needed */ @@ -1055,46 +1060,42 @@ void handle_hmi_exception(struct pt_regs *regs) if (ppc_md.handle_hmi_exception) ppc_md.handle_hmi_exception(regs); - irq_exit(); set_irq_regs(old_regs); } -void unknown_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(unknown_exception) { - enum ctx_state prev_state = exception_enter(); - printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n", regs->nip, regs->msr, regs->trap); _exception(SIGTRAP, regs, TRAP_UNK, 0); - - exception_exit(prev_state); } -void instruction_breakpoint_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception) { - enum ctx_state prev_state = exception_enter(); + printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n", + regs->nip, regs->msr, regs->trap); + _exception(SIGTRAP, regs, TRAP_UNK, 0); +} + +DEFINE_INTERRUPT_HANDLER(instruction_breakpoint_exception) +{ if (notify_die(DIE_IABR_MATCH, "iabr_match", regs, 5, 5, SIGTRAP) == NOTIFY_STOP) - goto bail; + return; if (debugger_iabr_match(regs)) - goto bail; + return; _exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip); - -bail: - exception_exit(prev_state); } -void RunModeException(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(RunModeException) { _exception(SIGTRAP, regs, TRAP_UNK, 0); } -void single_step_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(single_step_exception) { - enum ctx_state prev_state = exception_enter(); - clear_single_step(regs); clear_br_trace(regs); @@ -1103,16 +1104,12 @@ void single_step_exception(struct pt_regs *regs) if (notify_die(DIE_SSTEP, "single_step", regs, 5, 5, SIGTRAP) == NOTIFY_STOP) - goto bail; + return; if (debugger_sstep(regs)) - goto bail; + return; _exception(SIGTRAP, regs, TRAP_TRACE, regs->nip); - -bail: - exception_exit(prev_state); } -NOKPROBE_SYMBOL(single_step_exception); /* * After we have successfully emulated an instruction, we have to @@ -1436,9 +1433,8 @@ static int emulate_math(struct pt_regs *regs) static inline int emulate_math(struct pt_regs *regs) { return -1; } #endif -void program_check_exception(struct pt_regs *regs) +static void do_program_check(struct pt_regs *regs) { - enum ctx_state prev_state = exception_enter(); unsigned int reason = get_reason(regs); /* We can now get here via a FP Unavailable exception if the core @@ -1447,22 +1443,22 @@ void program_check_exception(struct pt_regs *regs) if (reason & REASON_FP) { /* IEEE FP exception */ parse_fpe(regs); - goto bail; + return; } if (reason & REASON_TRAP) { unsigned long bugaddr; /* Debugger is first in line to stop recursive faults in * rcu_lock, notify_die, or atomic_notifier_call_chain */ if (debugger_bpt(regs)) - goto bail; + return; if (kprobe_handler(regs)) - goto bail; + return; /* trap exception */ if (notify_die(DIE_BPT, "breakpoint", regs, 5, 5, SIGTRAP) == NOTIFY_STOP) - goto bail; + return; bugaddr = regs->nip; /* @@ -1474,10 +1470,10 @@ void program_check_exception(struct pt_regs *regs) if (!(regs->msr & MSR_PR) && /* not user-mode */ report_bug(bugaddr, regs) == BUG_TRAP_TYPE_WARN) { regs->nip += 4; - goto bail; + return; } _exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip); - goto bail; + return; } #ifdef CONFIG_PPC_TRANSACTIONAL_MEM if (reason & REASON_TM) { @@ -1498,7 +1494,7 @@ void program_check_exception(struct pt_regs *regs) */ if (user_mode(regs)) { _exception(SIGILL, regs, ILL_ILLOPN, regs->nip); - goto bail; + return; } else { printk(KERN_EMERG "Unexpected TM Bad Thing exception " "at %lx (msr 0x%lx) tm_scratch=%llx\n", @@ -1518,9 +1514,7 @@ void program_check_exception(struct pt_regs *regs) if (!user_mode(regs)) goto sigill; - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); /* (reason & REASON_ILLEGAL) would be the obvious thing here, * but there seems to be a hardware bug on the 405GP (RevD) @@ -1531,7 +1525,7 @@ void program_check_exception(struct pt_regs *regs) * pattern to occurrences etc. -dgibson 31/Mar/2003 */ if (!emulate_math(regs)) - goto bail; + return; /* Try to emulate it if we should. */ if (reason & (REASON_ILLEGAL | REASON_PRIVILEGED)) { @@ -1539,10 +1533,10 @@ void program_check_exception(struct pt_regs *regs) case 0: regs->nip += 4; emulate_single_step(regs); - goto bail; + return; case -EFAULT: _exception(SIGSEGV, regs, SEGV_MAPERR, regs->nip); - goto bail; + return; } } @@ -1552,34 +1546,31 @@ sigill: else _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); -bail: - exception_exit(prev_state); } -NOKPROBE_SYMBOL(program_check_exception); + +DEFINE_INTERRUPT_HANDLER(program_check_exception) +{ + do_program_check(regs); +} /* * This occurs when running in hypervisor mode on POWER6 or later * and an illegal instruction is encountered. */ -void emulation_assist_interrupt(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(emulation_assist_interrupt) { regs->msr |= REASON_ILLEGAL; - program_check_exception(regs); + do_program_check(regs); } -NOKPROBE_SYMBOL(emulation_assist_interrupt); -void alignment_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(alignment_exception) { - enum ctx_state prev_state = exception_enter(); int sig, code, fixed = 0; unsigned long reason; - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); reason = get_reason(regs); - if (reason & REASON_BOUNDARY) { sig = SIGBUS; code = BUS_ADRALN; @@ -1587,7 +1578,7 @@ void alignment_exception(struct pt_regs *regs) } if (tm_abort_check(regs, TM_CAUSE_ALIGNMENT | TM_CAUSE_PERSISTENT)) - goto bail; + return; /* we don't implement logging of alignment exceptions */ if (!(current->thread.align_ctl & PR_UNALIGN_SIGBUS)) @@ -1597,7 +1588,7 @@ void alignment_exception(struct pt_regs *regs) /* skip over emulated instruction */ regs->nip += inst_length(reason); emulate_single_step(regs); - goto bail; + return; } /* Operand address was bad */ @@ -1612,13 +1603,10 @@ bad: if (user_mode(regs)) _exception(sig, regs, code, regs->dar); else - bad_page_fault(regs, regs->dar, sig); - -bail: - exception_exit(prev_state); + bad_page_fault(regs, sig); } -void StackOverflow(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(StackOverflow) { pr_crit("Kernel stack overflow in process %s[%d], r1=%lx\n", current->comm, task_pid_nr(current), regs->gpr[1]); @@ -1627,46 +1615,33 @@ void StackOverflow(struct pt_regs *regs) panic("kernel stack overflow"); } -void stack_overflow_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(stack_overflow_exception) { - enum ctx_state prev_state = exception_enter(); - die("Kernel stack overflow", regs, SIGSEGV); - - exception_exit(prev_state); } -void kernel_fp_unavailable_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception) { - enum ctx_state prev_state = exception_enter(); - printk(KERN_EMERG "Unrecoverable FP Unavailable Exception " "%lx at %lx\n", regs->trap, regs->nip); die("Unrecoverable FP Unavailable Exception", regs, SIGABRT); - - exception_exit(prev_state); } -void altivec_unavailable_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(altivec_unavailable_exception) { - enum ctx_state prev_state = exception_enter(); - if (user_mode(regs)) { /* A user program has executed an altivec instruction, but this kernel doesn't support altivec. */ _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); - goto bail; + return; } printk(KERN_EMERG "Unrecoverable VMX/Altivec Unavailable Exception " "%lx at %lx\n", regs->trap, regs->nip); die("Unrecoverable VMX/Altivec Unavailable Exception", regs, SIGABRT); - -bail: - exception_exit(prev_state); } -void vsx_unavailable_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(vsx_unavailable_exception) { if (user_mode(regs)) { /* A user program has executed an vsx instruction, @@ -1697,7 +1672,7 @@ static void tm_unavailable(struct pt_regs *regs) die("Unrecoverable TM Unavailable Exception", regs, SIGABRT); } -void facility_unavailable_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(facility_unavailable_exception) { static char *facility_strings[] = { [FSCR_FP_LG] = "FPU", @@ -1737,9 +1712,7 @@ void facility_unavailable_exception(struct pt_regs *regs) die("Unexpected facility unavailable exception", regs, SIGABRT); } - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); if (status == FSCR_DSCR_LG) { /* @@ -1817,7 +1790,7 @@ out: #ifdef CONFIG_PPC_TRANSACTIONAL_MEM -void fp_unavailable_tm(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(fp_unavailable_tm) { /* Note: This does not handle any kind of FP laziness. */ @@ -1850,7 +1823,7 @@ void fp_unavailable_tm(struct pt_regs *regs) tm_recheckpoint(¤t->thread); } -void altivec_unavailable_tm(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(altivec_unavailable_tm) { /* See the comments in fp_unavailable_tm(). This function operates * the same way. @@ -1865,7 +1838,7 @@ void altivec_unavailable_tm(struct pt_regs *regs) current->thread.used_vr = 1; } -void vsx_unavailable_tm(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(vsx_unavailable_tm) { /* See the comments in fp_unavailable_tm(). This works similarly, * though we're loading both FP and VEC registers in here. @@ -1890,11 +1863,40 @@ void vsx_unavailable_tm(struct pt_regs *regs) } #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ -void performance_monitor_exception(struct pt_regs *regs) +#ifdef CONFIG_PPC64 +DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi); +DEFINE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi) { __this_cpu_inc(irq_stat.pmu_irqs); perf_irq(regs); + + return 0; +} +#endif + +DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async); +DEFINE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async) +{ + __this_cpu_inc(irq_stat.pmu_irqs); + + perf_irq(regs); +} + +DEFINE_INTERRUPT_HANDLER_RAW(performance_monitor_exception) +{ + /* + * On 64-bit, if perf interrupts hit in a local_irq_disable + * (soft-masked) region, we consider them as NMIs. This is required to + * prevent hash faults on user addresses when reading callchains (and + * looks better from an irq tracing perspective). + */ + if (IS_ENABLED(CONFIG_PPC64) && unlikely(arch_irq_disabled_regs(regs))) + performance_monitor_exception_nmi(regs); + else + performance_monitor_exception_async(regs); + + return 0; } #ifdef CONFIG_PPC_ADV_DEBUG_REGS @@ -1957,8 +1959,10 @@ static void handle_debug(struct pt_regs *regs, unsigned long debug_status) mtspr(SPRN_DBCR0, current->thread.debug.dbcr0); } -void DebugException(struct pt_regs *regs, unsigned long debug_status) +DEFINE_INTERRUPT_HANDLER(DebugException) { + unsigned long debug_status = regs->dsisr; + current->thread.debug.dbsr = debug_status; /* Hack alert: On BookE, Branch Taken stops on the branch itself, while @@ -2024,11 +2028,10 @@ void DebugException(struct pt_regs *regs, unsigned long debug_status) } else handle_debug(regs, debug_status); } -NOKPROBE_SYMBOL(DebugException); #endif /* CONFIG_PPC_ADV_DEBUG_REGS */ #ifdef CONFIG_ALTIVEC -void altivec_assist_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(altivec_assist_exception) { int err; @@ -2062,9 +2065,10 @@ void altivec_assist_exception(struct pt_regs *regs) #endif /* CONFIG_ALTIVEC */ #ifdef CONFIG_FSL_BOOKE -void CacheLockingException(struct pt_regs *regs, unsigned long address, - unsigned long error_code) +DEFINE_INTERRUPT_HANDLER(CacheLockingException) { + unsigned long error_code = regs->dsisr; + /* We treat cache locking instructions from the user * as priv ops, in the future we could try to do * something smarter @@ -2076,7 +2080,7 @@ void CacheLockingException(struct pt_regs *regs, unsigned long address, #endif /* CONFIG_FSL_BOOKE */ #ifdef CONFIG_SPE -void SPEFloatingPointException(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(SPEFloatingPointException) { extern int do_spe_mathemu(struct pt_regs *regs); unsigned long spefscr; @@ -2084,9 +2088,7 @@ void SPEFloatingPointException(struct pt_regs *regs) int code = FPE_FLTUNK; int err; - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); flush_spe_to_thread(current); @@ -2128,14 +2130,12 @@ void SPEFloatingPointException(struct pt_regs *regs) return; } -void SPEFloatingPointRoundException(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(SPEFloatingPointRoundException) { extern int speround_handler(struct pt_regs *regs); int err; - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); preempt_disable(); if (regs->msr & MSR_SPE) @@ -2170,13 +2170,12 @@ void SPEFloatingPointRoundException(struct pt_regs *regs) * in the MSR is 0. This indicates that SRR0/1 are live, and that * we therefore lost state by taking this exception. */ -void unrecoverable_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(unrecoverable_exception) { pr_emerg("Unrecoverable exception %lx at %lx (msr=%lx)\n", regs->trap, regs->nip, regs->msr); die("Unrecoverable exception", regs, SIGABRT); } -NOKPROBE_SYMBOL(unrecoverable_exception); #if defined(CONFIG_BOOKE_WDT) || defined(CONFIG_40x) /* @@ -2190,7 +2189,7 @@ void __attribute__ ((weak)) WatchdogHandler(struct pt_regs *regs) return; } -void WatchdogException(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(WatchdogException) /* XXX NMI? async? */ { printk (KERN_EMERG "PowerPC Book-E Watchdog Exception\n"); WatchdogHandler(regs); @@ -2201,13 +2200,12 @@ void WatchdogException(struct pt_regs *regs) * We enter here if we discover during exception entry that we are * running in supervisor mode with a userspace value in the stack pointer. */ -void kernel_bad_stack(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(kernel_bad_stack) { printk(KERN_EMERG "Bad kernel stack pointer %lx at %lx\n", regs->gpr[1], regs->nip); die("Bad kernel stack pointer", regs, SIGABRT); } -NOKPROBE_SYMBOL(kernel_bad_stack); void __init trap_init(void) { diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c index af3c15a1d41e..c9a8f4781a10 100644 --- a/arch/powerpc/kernel/watchdog.c +++ b/arch/powerpc/kernel/watchdog.c @@ -26,7 +26,9 @@ #include #include +#include #include +#include /* * The powerpc watchdog ensures that each CPU is able to service timers. @@ -247,16 +249,17 @@ static void watchdog_timer_interrupt(int cpu) watchdog_smp_panic(cpu, tb); } -void soft_nmi_interrupt(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt) { unsigned long flags; int cpu = raw_smp_processor_id(); u64 tb; - if (!cpumask_test_cpu(cpu, &wd_cpus_enabled)) - return; + /* should only arrive from kernel, with irqs disabled */ + WARN_ON_ONCE(!arch_irq_disabled_regs(regs)); - nmi_enter(); + if (!cpumask_test_cpu(cpu, &wd_cpus_enabled)) + return 0; __this_cpu_inc(irq_stat.soft_nmi_irqs); @@ -265,7 +268,7 @@ void soft_nmi_interrupt(struct pt_regs *regs) wd_smp_lock(&flags); if (cpumask_test_cpu(cpu, &wd_smp_cpus_stuck)) { wd_smp_unlock(&flags); - goto out; + return 0; } set_cpu_stuck(cpu, tb); @@ -289,8 +292,7 @@ void soft_nmi_interrupt(struct pt_regs *regs) if (wd_panic_timeout_tb < 0x7fffffff) mtspr(SPRN_DEC, wd_panic_timeout_tb); -out: - nmi_exit(); + return 0; } static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) diff --git a/arch/powerpc/kexec/elf_64.c b/arch/powerpc/kexec/elf_64.c index d0e459bb2f05..9842e33533df 100644 --- a/arch/powerpc/kexec/elf_64.c +++ b/arch/powerpc/kexec/elf_64.c @@ -102,7 +102,7 @@ static void *elf64_load(struct kimage *image, char *kernel_buf, pr_debug("Loaded initrd at 0x%lx\n", initrd_load_addr); } - fdt_size = fdt_totalsize(initial_boot_params) * 2; + fdt_size = kexec_fdt_totalsize_ppc64(image); fdt = kmalloc(fdt_size, GFP_KERNEL); if (!fdt) { pr_err("Not enough memory for the device tree.\n"); diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index c69bcf9b547a..02b9e4d0dc40 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -925,6 +926,40 @@ out: return ret; } +/** + * kexec_fdt_totalsize_ppc64 - Return the estimated size needed to setup FDT + * for kexec/kdump kernel. + * @image: kexec image being loaded. + * + * Returns the estimated size needed for kexec/kdump kernel FDT. + */ +unsigned int kexec_fdt_totalsize_ppc64(struct kimage *image) +{ + unsigned int fdt_size; + u64 usm_entries; + + /* + * The below estimate more than accounts for a typical kexec case where + * the additional space is to accommodate things like kexec cmdline, + * chosen node with properties for initrd start & end addresses and + * a property to indicate kexec boot.. + */ + fdt_size = fdt_totalsize(initial_boot_params) + (2 * COMMAND_LINE_SIZE); + if (image->type != KEXEC_TYPE_CRASH) + return fdt_size; + + /* + * For kdump kernel, also account for linux,usable-memory and + * linux,drconf-usable-memory properties. Get an approximate on the + * number of usable memory entries and use for FDT size estimation. + */ + usm_entries = ((memblock_end_of_DRAM() / drmem_lmb_size()) + + (2 * (resource_size(&crashk_res) / drmem_lmb_size()))); + fdt_size += (unsigned int)(usm_entries * sizeof(u64)); + + return fdt_size; +} + /** * setup_new_fdt_ppc64 - Update the flattend device-tree of the kernel * being loaded. diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index 549591d9aaa2..e45644657d49 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -54,6 +54,7 @@ config KVM_BOOK3S_32 select KVM select KVM_BOOK3S_32_HANDLER select KVM_BOOK3S_PR_POSSIBLE + select PPC_FPU help Support running unmodified book3s_32 guest kernels in virtual machines on book3s_32 host processors. diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index 38ea396a23d6..bb6773594cf8 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -27,6 +27,7 @@ #include #include +#include "book3s.h" #include "trace_hv.h" //#define DEBUG_RESIZE_HPT 1 @@ -590,7 +591,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_vcpu *vcpu, } else { /* Call KVM generic code to do the slow-path check */ pfn = __gfn_to_pfn_memslot(memslot, gfn, false, NULL, - writing, &write_ok); + writing, &write_ok, NULL); if (is_error_noslot_pfn(pfn)) return -EFAULT; page = NULL; diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c index bb35490400e9..e603de7ade52 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_radix.c +++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c @@ -822,7 +822,7 @@ int kvmppc_book3s_instantiate_page(struct kvm_vcpu *vcpu, /* Call KVM generic code to do the slow-path check */ pfn = __gfn_to_pfn_memslot(memslot, gfn, false, NULL, - writing, upgrade_p); + writing, upgrade_p, NULL); if (is_error_noslot_pfn(pfn)) return -EFAULT; page = NULL; diff --git a/arch/powerpc/kvm/book3s_emulate.c b/arch/powerpc/kvm/book3s_emulate.c index b08cc15f31c7..fdb57be71aa6 100644 --- a/arch/powerpc/kvm/book3s_emulate.c +++ b/arch/powerpc/kvm/book3s_emulate.c @@ -61,10 +61,6 @@ #define SPRN_GQR6 918 #define SPRN_GQR7 919 -/* Book3S_32 defines mfsrin(v) - but that messes up our abstract - * function pointers, so let's just disable the define. */ -#undef mfsrin - enum priv_level { PRIV_PROBLEM = 0, PRIV_SUPER = 1, diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index f09708da216e..13bad6bf4c95 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -3408,8 +3409,9 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) kvmppc_set_host_core(pcpu); + guest_exit_irqoff(); + local_irq_enable(); - guest_exit(); /* Let secondaries go back to the offline loop */ for (i = 0; i < controlled_threads; ++i) { @@ -4236,8 +4238,9 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, kvmppc_set_host_core(pcpu); + guest_exit_irqoff(); + local_irq_enable(); - guest_exit(); cpumask_clear_cpu(pcpu, &kvm->arch.cpu_in_guest); diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index f3d3183249fe..158d309b42a3 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include diff --git a/arch/powerpc/kvm/book3s_xive.c b/arch/powerpc/kvm/book3s_xive.c index 30dfeac731c6..e7219b6f5f9a 100644 --- a/arch/powerpc/kvm/book3s_xive.c +++ b/arch/powerpc/kvm/book3s_xive.c @@ -1813,9 +1813,9 @@ int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, return -EINVAL; if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL) - state->asserted = 1; + state->asserted = true; else if (level == 0 || level == KVM_INTERRUPT_UNSET) { - state->asserted = 0; + state->asserted = false; return 0; } diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index f38ae3e54b37..7d5fe43f85c4 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 6c083a9b3545..a2a68a958fa0 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -1522,7 +1522,7 @@ int kvmppc_handle_vmx_load(struct kvm_vcpu *vcpu, return emulated; } -int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val) +static int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val) { union kvmppc_one_reg reg; int vmx_offset = 0; @@ -1540,7 +1540,7 @@ int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val) return result; } -int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val) +static int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val) { union kvmppc_one_reg reg; int vmx_offset = 0; @@ -1558,7 +1558,7 @@ int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val) return result; } -int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val) +static int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val) { union kvmppc_one_reg reg; int vmx_offset = 0; @@ -1576,7 +1576,7 @@ int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val) return result; } -int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val) +static int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val) { union kvmppc_one_reg reg; int vmx_offset = 0; diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index 69a91b571845..d4efc182662a 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -31,7 +31,7 @@ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o # 64-bit linker creates .sfpr on demand for final link (vmlinux), # so it is only needed for modules, and only for older linkers which # do not support --save-restore-funcs -ifeq ($(call ld-ifversion, -lt, 225000000, y),y) +ifeq ($(call ld-ifversion, -lt, 22500, y),y) extra-$(CONFIG_PPC64) += crtsavres.o endif diff --git a/arch/powerpc/lib/pmem.c b/arch/powerpc/lib/pmem.c index 1550e0d2513a..eb2919ddf9b9 100644 --- a/arch/powerpc/lib/pmem.c +++ b/arch/powerpc/lib/pmem.c @@ -6,6 +6,7 @@ #include #include #include +#include #include diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c index ede093e96234..bb5c20d4ca91 100644 --- a/arch/powerpc/lib/sstep.c +++ b/arch/powerpc/lib/sstep.c @@ -1306,9 +1306,11 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, if ((word & 0xfe2) == 2) op->type = SYSCALL; else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && - (word & 0xfe3) == 1) + (word & 0xfe3) == 1) { /* scv */ op->type = SYSCALL_VECTORED_0; - else + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; + } else op->type = UNKNOWN; return 0; #endif @@ -1412,7 +1414,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #ifdef __powerpc64__ case 1: if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return -1; + goto unknown_opcode; prefix_r = GET_PREFIX_R(word); ra = GET_PREFIX_RA(suffix); @@ -1445,8 +1447,13 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #ifdef __powerpc64__ case 4: + /* + * There are very many instructions with this primary opcode + * introduced in the ISA as early as v2.03. However, the ones + * we currently emulate were all introduced with ISA 3.0 + */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; switch (word & 0x3f) { case 48: /* maddhd */ @@ -1472,7 +1479,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, * There are other instructions from ISA 3.0 with the same * primary opcode which do not have emulation support yet. */ - return -1; + goto unknown_opcode; #endif case 7: /* mulli */ @@ -1532,6 +1539,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 19: if (((word >> 1) & 0x1f) == 2) { /* addpcis */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; imm = (short) (word & 0xffc1); /* d0 + d2 fields */ imm |= (word >> 15) & 0x3e; /* d1 field */ op->val = regs->nip + (imm << 16) + 4; @@ -1844,7 +1853,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #ifdef __powerpc64__ case 265: /* modud */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; op->val = regs->gpr[ra] % regs->gpr[rb]; goto compute_done; #endif @@ -1854,7 +1863,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 267: /* moduw */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; op->val = (unsigned int) regs->gpr[ra] % (unsigned int) regs->gpr[rb]; goto compute_done; @@ -1891,7 +1900,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #endif case 755: /* darn */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; switch (ra & 0x3) { case 0: /* 32-bit conditioned */ @@ -1909,18 +1918,18 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, goto compute_done; } - return -1; + goto unknown_opcode; #ifdef __powerpc64__ case 777: /* modsd */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; op->val = (long int) regs->gpr[ra] % (long int) regs->gpr[rb]; goto compute_done; #endif case 779: /* modsw */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; op->val = (int) regs->gpr[ra] % (int) regs->gpr[rb]; goto compute_done; @@ -1997,14 +2006,14 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #endif case 538: /* cnttzw */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; val = (unsigned int) regs->gpr[rd]; op->val = (val ? __builtin_ctz(val) : 32); goto logical_done; #ifdef __powerpc64__ case 570: /* cnttzd */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; val = regs->gpr[rd]; op->val = (val ? __builtin_ctzl(val) : 64); goto logical_done; @@ -2114,7 +2123,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 890: /* extswsli with sh_5 = 0 */ case 891: /* extswsli with sh_5 = 1 */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; op->type = COMPUTE + SETREG; sh = rb | ((word & 2) << 4); val = (signed int) regs->gpr[rd]; @@ -2441,6 +2450,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 268: /* lxvx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 16); op->element_size = 16; @@ -2450,6 +2461,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 269: /* lxvl */ case 301: { /* lxvll */ int nb; + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->ea = ra ? regs->gpr[ra] : 0; nb = regs->gpr[rb] & 0xff; @@ -2470,13 +2483,15 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 333: /* lxvpx */ if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return -1; + goto unknown_opcode; op->reg = VSX_REGISTER_XTP(rd); op->type = MKOP(LOAD_VSX, 0, 32); op->element_size = 32; break; case 364: /* lxvwsx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 4); op->element_size = 4; @@ -2484,6 +2499,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 396: /* stxvx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(STORE_VSX, 0, 16); op->element_size = 16; @@ -2493,6 +2510,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 397: /* stxvl */ case 429: { /* stxvll */ int nb; + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->ea = ra ? regs->gpr[ra] : 0; nb = regs->gpr[rb] & 0xff; @@ -2506,7 +2525,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, } case 461: /* stxvpx */ if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return -1; + goto unknown_opcode; op->reg = VSX_REGISTER_XTP(rd); op->type = MKOP(STORE_VSX, 0, 32); op->element_size = 32; @@ -2544,6 +2563,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 781: /* lxsibzx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 1); op->element_size = 8; @@ -2551,6 +2572,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 812: /* lxvh8x */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 16); op->element_size = 2; @@ -2558,6 +2581,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 813: /* lxsihzx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 2); op->element_size = 8; @@ -2571,6 +2596,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 876: /* lxvb16x */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 16); op->element_size = 1; @@ -2584,6 +2611,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 909: /* stxsibx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(STORE_VSX, 0, 1); op->element_size = 8; @@ -2591,6 +2620,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 940: /* stxvh8x */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(STORE_VSX, 0, 16); op->element_size = 2; @@ -2598,6 +2629,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 941: /* stxsihx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(STORE_VSX, 0, 2); op->element_size = 8; @@ -2611,6 +2644,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 1004: /* stxvb16x */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(STORE_VSX, 0, 16); op->element_size = 1; @@ -2719,12 +2754,16 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, op->type = MKOP(LOAD_FP, 0, 16); break; case 2: /* lxsd */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd + 32; op->type = MKOP(LOAD_VSX, 0, 8); op->element_size = 8; op->vsx_flags = VSX_CHECK_VEC; break; case 3: /* lxssp */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd + 32; op->type = MKOP(LOAD_VSX, 0, 4); op->element_size = 8; @@ -2754,7 +2793,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #ifdef CONFIG_VSX case 6: if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return -1; + goto unknown_opcode; op->ea = dqform_ea(word, regs); op->reg = VSX_REGISTER_XTP(rd); op->element_size = 32; @@ -2777,6 +2816,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 1: /* lxv */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->ea = dqform_ea(word, regs); if (word & 8) op->reg = rd + 32; @@ -2787,6 +2828,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 2: /* stxsd with LSB of DS field = 0 */ case 6: /* stxsd with LSB of DS field = 1 */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->ea = dsform_ea(word, regs); op->reg = rd + 32; op->type = MKOP(STORE_VSX, 0, 8); @@ -2796,6 +2839,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 3: /* stxssp with LSB of DS field = 0 */ case 7: /* stxssp with LSB of DS field = 1 */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->ea = dsform_ea(word, regs); op->reg = rd + 32; op->type = MKOP(STORE_VSX, 0, 4); @@ -2804,6 +2849,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 5: /* stxv */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->ea = dqform_ea(word, regs); if (word & 8) op->reg = rd + 32; @@ -2833,7 +2880,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 1: /* Prefixed instructions */ if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return -1; + goto unknown_opcode; prefix_r = GET_PREFIX_R(word); ra = GET_PREFIX_RA(suffix); @@ -2972,6 +3019,20 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, } + if (OP_IS_LOAD_STORE(op->type) && (op->type & UPDATE)) { + switch (GETTYPE(op->type)) { + case LOAD: + if (ra == rd) + goto unknown_opcode; + fallthrough; + case STORE: + case LOAD_FP: + case STORE_FP: + if (ra == 0) + goto unknown_opcode; + } + } + #ifdef CONFIG_VSX if ((GETTYPE(op->type) == LOAD_VSX || GETTYPE(op->type) == STORE_VSX) && @@ -2982,6 +3043,10 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, return 0; + unknown_opcode: + op->type = UNKNOWN; + return 0; + logical_done: if (word & 1) set_cr0(regs, op); diff --git a/arch/powerpc/mm/book3s32/Makefile b/arch/powerpc/mm/book3s32/Makefile index 3f972db17761..446d9de88ce4 100644 --- a/arch/powerpc/mm/book3s32/Makefile +++ b/arch/powerpc/mm/book3s32/Makefile @@ -6,4 +6,6 @@ ifdef CONFIG_KASAN CFLAGS_mmu.o += -DDISABLE_BRANCH_PROFILING endif -obj-y += mmu.o hash_low.o mmu_context.o tlb.o nohash_low.o +obj-y += mmu.o mmu_context.o +obj-$(CONFIG_PPC_BOOK3S_603) += nohash_low.o +obj-$(CONFIG_PPC_BOOK3S_604) += hash_low.o tlb.o diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c index 859e5bd603ac..d7eb266a3f7a 100644 --- a/arch/powerpc/mm/book3s32/mmu.c +++ b/arch/powerpc/mm/book3s32/mmu.c @@ -234,7 +234,7 @@ void mmu_mark_initmem_nx(void) if (is_module_segment(i << 28)) continue; - mtsrin(mfsrin(i << 28) | 0x10000000, i << 28); + mtsr(mfsr(i << 28) | 0x10000000, i << 28); } } diff --git a/arch/powerpc/mm/book3s64/hash_hugetlbpage.c b/arch/powerpc/mm/book3s64/hash_hugetlbpage.c index b5e9fff8c217..a688e1324ae5 100644 --- a/arch/powerpc/mm/book3s64/hash_hugetlbpage.c +++ b/arch/powerpc/mm/book3s64/hash_hugetlbpage.c @@ -16,10 +16,6 @@ unsigned int hpage_shift; EXPORT_SYMBOL(hpage_shift); -extern long hpte_insert_repeating(unsigned long hash, unsigned long vpn, - unsigned long pa, unsigned long rlags, - unsigned long vflags, int psize, int ssize); - int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize, unsigned int shift, unsigned int mmu_psize) diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c index 73b06adb6eeb..581b20a2feaf 100644 --- a/arch/powerpc/mm/book3s64/hash_utils.c +++ b/arch/powerpc/mm/book3s64/hash_utils.c @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -1143,10 +1144,10 @@ unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap) page = pte_page(pte); /* page is dirty */ - if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) { + if (!test_bit(PG_dcache_clean, &page->flags) && !PageReserved(page)) { if (trap == 0x400) { flush_dcache_icache_page(page); - set_bit(PG_arch_1, &page->flags); + set_bit(PG_dcache_clean, &page->flags); } else pp |= HPTE_R_N; } @@ -1288,7 +1289,6 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea, unsigned long flags) { bool is_thp; - enum ctx_state prev_state = exception_enter(); pgd_t *pgdir; unsigned long vsid; pte_t *ptep; @@ -1490,7 +1490,6 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea, DBG_LOW(" -> rc=%d\n", rc); bail: - exception_exit(prev_state); return rc; } EXPORT_SYMBOL_GPL(hash_page_mm); @@ -1512,16 +1511,22 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap, } EXPORT_SYMBOL_GPL(hash_page); -int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, - unsigned long msr) +DECLARE_INTERRUPT_HANDLER_RET(__do_hash_fault); +DEFINE_INTERRUPT_HANDLER_RET(__do_hash_fault) { + unsigned long ea = regs->dar; + unsigned long dsisr = regs->dsisr; unsigned long access = _PAGE_PRESENT | _PAGE_READ; unsigned long flags = 0; - struct mm_struct *mm = current->mm; - unsigned int region_id = get_region_id(ea); + struct mm_struct *mm; + unsigned int region_id; + long err; + region_id = get_region_id(ea); if ((region_id == VMALLOC_REGION_ID) || (region_id == IO_REGION_ID)) mm = &init_mm; + else + mm = current->mm; if (dsisr & DSISR_NOHPTE) flags |= HPTE_NOHPTE_UPDATE; @@ -1537,13 +1542,66 @@ int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, * 2) user space access kernel space. */ access |= _PAGE_PRIVILEGED; - if ((msr & MSR_PR) || (region_id == USER_REGION_ID)) + if (user_mode(regs) || (region_id == USER_REGION_ID)) access &= ~_PAGE_PRIVILEGED; - if (trap == 0x400) + if (regs->trap == 0x400) access |= _PAGE_EXEC; - return hash_page_mm(mm, ea, access, trap, flags); + err = hash_page_mm(mm, ea, access, regs->trap, flags); + if (unlikely(err < 0)) { + // failed to instert a hash PTE due to an hypervisor error + if (user_mode(regs)) { + if (IS_ENABLED(CONFIG_PPC_SUBPAGE_PROT) && err == -2) + _exception(SIGSEGV, regs, SEGV_ACCERR, ea); + else + _exception(SIGBUS, regs, BUS_ADRERR, ea); + } else { + bad_page_fault(regs, SIGBUS); + } + err = 0; + } + + return err; +} + +/* + * The _RAW interrupt entry checks for the in_nmi() case before + * running the full handler. + */ +DEFINE_INTERRUPT_HANDLER_RAW(do_hash_fault) +{ + unsigned long dsisr = regs->dsisr; + long err; + + if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_KEYFAULT))) + goto page_fault; + + /* + * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then + * don't call hash_page, just fail the fault. This is required to + * prevent re-entrancy problems in the hash code, namely perf + * interrupts hitting while something holds H_PAGE_BUSY, and taking a + * hash fault. See the comment in hash_preload(). + * + * We come here as a result of a DSI at a point where we don't want + * to call hash_page, such as when we are accessing memory (possibly + * user memory) inside a PMU interrupt that occurred while interrupts + * were soft-disabled. We want to invoke the exception handler for + * the access, or panic if there isn't a handler. + */ + if (unlikely(in_nmi())) { + do_bad_page_fault_segv(regs); + return 0; + } + + err = __do_hash_fault(regs); + if (err) { +page_fault: + err = hash__do_page_fault(regs); + } + + return err; } #ifdef CONFIG_PPC_MM_SLICES @@ -1843,27 +1901,6 @@ void flush_hash_range(unsigned long number, int local) } } -/* - * low_hash_fault is called when we the low level hash code failed - * to instert a PTE due to an hypervisor error - */ -void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc) -{ - enum ctx_state prev_state = exception_enter(); - - if (user_mode(regs)) { -#ifdef CONFIG_PPC_SUBPAGE_PROT - if (rc == -2) - _exception(SIGSEGV, regs, SEGV_ACCERR, address); - else -#endif - _exception(SIGBUS, regs, BUS_ADRERR, address); - } else - bad_page_fault(regs, address, SIGBUS); - - exception_exit(prev_state); -} - long hpte_insert_repeating(unsigned long hash, unsigned long vpn, unsigned long pa, unsigned long rflags, unsigned long vflags, int psize, int ssize) diff --git a/arch/powerpc/mm/book3s64/internal.h b/arch/powerpc/mm/book3s64/internal.h index c12d78ee42f5..5045048ce244 100644 --- a/arch/powerpc/mm/book3s64/internal.h +++ b/arch/powerpc/mm/book3s64/internal.h @@ -15,4 +15,6 @@ static inline bool stress_slb(void) void slb_setup_new_exec(void); +void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush); + #endif /* ARCH_POWERPC_MM_BOOK3S64_INTERNAL_H */ diff --git a/arch/powerpc/mm/book3s64/iommu_api.c b/arch/powerpc/mm/book3s64/iommu_api.c index 685d7bb3d26f..cd18e94d0843 100644 --- a/arch/powerpc/mm/book3s64/iommu_api.c +++ b/arch/powerpc/mm/book3s64/iommu_api.c @@ -129,7 +129,8 @@ good_exit: mutex_lock(&mem_list_mutex); - list_for_each_entry_rcu(mem2, &mm->context.iommu_group_mem_list, next) { + list_for_each_entry_rcu(mem2, &mm->context.iommu_group_mem_list, next, + lockdep_is_held(&mem_list_mutex)) { /* Overlap? */ if ((mem2->ua < (ua + (entries << PAGE_SHIFT))) && (ua < (mem2->ua + @@ -289,6 +290,7 @@ struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm, { struct mm_iommu_table_group_mem_t *mem, *ret = NULL; + rcu_read_lock(); list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { if ((mem->ua <= ua) && (ua + size <= mem->ua + @@ -297,6 +299,7 @@ struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm, break; } } + rcu_read_unlock(); return ret; } @@ -327,7 +330,8 @@ struct mm_iommu_table_group_mem_t *mm_iommu_get(struct mm_struct *mm, mutex_lock(&mem_list_mutex); - list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { + list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next, + lockdep_is_held(&mem_list_mutex)) { if ((mem->ua == ua) && (mem->entries == entries)) { ret = mem; ++mem->used; @@ -421,6 +425,7 @@ bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa, struct mm_iommu_table_group_mem_t *mem; unsigned long end; + rcu_read_lock(); list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { if (mem->dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) continue; @@ -437,6 +442,7 @@ bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa, return true; } } + rcu_read_unlock(); return false; } diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index 5b3a3bae21aa..9ffa65074cb0 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -20,6 +20,8 @@ #include #include +#include "internal.h" + unsigned long __pmd_frag_nr; EXPORT_SYMBOL(__pmd_frag_nr); unsigned long __pmd_frag_size_shift; @@ -79,10 +81,15 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd)); } -static void do_nothing(void *unused) +static void do_serialize(void *arg) { - + /* We've taken the IPI, so try to trim the mask while here */ + if (radix_enabled()) { + struct mm_struct *mm = arg; + exit_lazy_flush_tlb(mm, false); + } } + /* * Serialize against find_current_mm_pte which does lock-less * lookup in page tables with local interrupts disabled. For huge pages @@ -96,7 +103,7 @@ static void do_nothing(void *unused) void serialize_against_pte_lookup(struct mm_struct *mm) { smp_mb(); - smp_call_function_many(mm_cpumask(mm), do_nothing, NULL, 1); + smp_call_function_many(mm_cpumask(mm), do_serialize, mm, 1); } /* diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index fb66d154b26c..409e61210789 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -18,6 +18,8 @@ #include #include +#include "internal.h" + #define RIC_FLUSH_TLB 0 #define RIC_FLUSH_PWC 1 #define RIC_FLUSH_ALL 2 @@ -627,15 +629,6 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd } EXPORT_SYMBOL(radix__local_flush_tlb_page); -static bool mm_is_singlethreaded(struct mm_struct *mm) -{ - if (atomic_read(&mm->context.copros) > 0) - return false; - if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) - return true; - return false; -} - static bool mm_needs_flush_escalation(struct mm_struct *mm) { /* @@ -648,21 +641,24 @@ static bool mm_needs_flush_escalation(struct mm_struct *mm) return false; } -#ifdef CONFIG_SMP -static void do_exit_flush_lazy_tlb(void *arg) +/* + * If always_flush is true, then flush even if this CPU can't be removed + * from mm_cpumask. + */ +void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush) { - struct mm_struct *mm = arg; unsigned long pid = mm->context.id; + int cpu = smp_processor_id(); /* * A kthread could have done a mmget_not_zero() after the flushing CPU - * checked mm_is_singlethreaded, and be in the process of - * kthread_use_mm when interrupted here. In that case, current->mm will - * be set to mm, because kthread_use_mm() setting ->mm and switching to - * the mm is done with interrupts off. + * checked mm_cpumask, and be in the process of kthread_use_mm when + * interrupted here. In that case, current->mm will be set to mm, + * because kthread_use_mm() setting ->mm and switching to the mm is + * done with interrupts off. */ if (current->mm == mm) - goto out_flush; + goto out; if (current->active_mm == mm) { WARN_ON_ONCE(current->mm != NULL); @@ -673,11 +669,30 @@ static void do_exit_flush_lazy_tlb(void *arg) mmdrop(mm); } - atomic_dec(&mm->context.active_cpus); - cpumask_clear_cpu(smp_processor_id(), mm_cpumask(mm)); + /* + * This IPI may be initiated from any source including those not + * running the mm, so there may be a racing IPI that comes after + * this one which finds the cpumask already clear. Check and avoid + * underflowing the active_cpus count in that case. The race should + * not otherwise be a problem, but the TLB must be flushed because + * that's what the caller expects. + */ + if (cpumask_test_cpu(cpu, mm_cpumask(mm))) { + atomic_dec(&mm->context.active_cpus); + cpumask_clear_cpu(cpu, mm_cpumask(mm)); + always_flush = true; + } -out_flush: - _tlbiel_pid(pid, RIC_FLUSH_ALL); +out: + if (always_flush) + _tlbiel_pid(pid, RIC_FLUSH_ALL); +} + +#ifdef CONFIG_SMP +static void do_exit_flush_lazy_tlb(void *arg) +{ + struct mm_struct *mm = arg; + exit_lazy_flush_tlb(mm, true); } static void exit_flush_lazy_tlbs(struct mm_struct *mm) @@ -693,9 +708,110 @@ static void exit_flush_lazy_tlbs(struct mm_struct *mm) (void *)mm, 1); } +#else /* CONFIG_SMP */ +static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { } +#endif /* CONFIG_SMP */ + +static DEFINE_PER_CPU(unsigned int, mm_cpumask_trim_clock); + +/* + * Interval between flushes at which we send out IPIs to check whether the + * mm_cpumask can be trimmed for the case where it's not a single-threaded + * process flushing its own mm. The intent is to reduce the cost of later + * flushes. Don't want this to be so low that it adds noticable cost to TLB + * flushing, or so high that it doesn't help reduce global TLBIEs. + */ +static unsigned long tlb_mm_cpumask_trim_timer = 1073; + +static bool tick_and_test_trim_clock(void) +{ + if (__this_cpu_inc_return(mm_cpumask_trim_clock) == + tlb_mm_cpumask_trim_timer) { + __this_cpu_write(mm_cpumask_trim_clock, 0); + return true; + } + return false; +} + +enum tlb_flush_type { + FLUSH_TYPE_NONE, + FLUSH_TYPE_LOCAL, + FLUSH_TYPE_GLOBAL, +}; + +static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm) +{ + int active_cpus = atomic_read(&mm->context.active_cpus); + int cpu = smp_processor_id(); + + if (active_cpus == 0) + return FLUSH_TYPE_NONE; + if (active_cpus == 1 && cpumask_test_cpu(cpu, mm_cpumask(mm))) { + if (current->mm != mm) { + /* + * Asynchronous flush sources may trim down to nothing + * if the process is not running, so occasionally try + * to trim. + */ + if (tick_and_test_trim_clock()) { + exit_lazy_flush_tlb(mm, true); + return FLUSH_TYPE_NONE; + } + } + return FLUSH_TYPE_LOCAL; + } + + /* Coprocessors require TLBIE to invalidate nMMU. */ + if (atomic_read(&mm->context.copros) > 0) + return FLUSH_TYPE_GLOBAL; + + /* + * In the fullmm case there's no point doing the exit_flush_lazy_tlbs + * because the mm is being taken down anyway, and a TLBIE tends to + * be faster than an IPI+TLBIEL. + */ + if (fullmm) + return FLUSH_TYPE_GLOBAL; + + /* + * If we are running the only thread of a single-threaded process, + * then we should almost always be able to trim off the rest of the + * CPU mask (except in the case of use_mm() races), so always try + * trimming the mask. + */ + if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) { + exit_flush_lazy_tlbs(mm); + /* + * use_mm() race could prevent IPIs from being able to clear + * the cpumask here, however those users are established + * after our first check (and so after the PTEs are removed), + * and the TLB still gets flushed by the IPI, so this CPU + * will only require a local flush. + */ + return FLUSH_TYPE_LOCAL; + } + + /* + * Occasionally try to trim down the cpumask. It's possible this can + * bring the mask to zero, which results in no flush. + */ + if (tick_and_test_trim_clock()) { + exit_flush_lazy_tlbs(mm); + if (current->mm == mm) + return FLUSH_TYPE_LOCAL; + if (cpumask_test_cpu(cpu, mm_cpumask(mm))) + exit_lazy_flush_tlb(mm, true); + return FLUSH_TYPE_NONE; + } + + return FLUSH_TYPE_GLOBAL; +} + +#ifdef CONFIG_SMP void radix__flush_tlb_mm(struct mm_struct *mm) { unsigned long pid; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) @@ -703,16 +819,15 @@ void radix__flush_tlb_mm(struct mm_struct *mm) preempt_disable(); /* - * Order loads of mm_cpumask vs previous stores to clear ptes before - * the invalidate. See barrier in switch_mm_irqs_off + * Order loads of mm_cpumask (in flush_type_needed) vs previous + * stores to clear ptes before the invalidate. See barrier in + * switch_mm_irqs_off */ smp_mb(); - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - exit_flush_lazy_tlbs(mm); - goto local; - } - + type = flush_type_needed(mm, false); + if (type == FLUSH_TYPE_LOCAL) { + _tlbiel_pid(pid, RIC_FLUSH_TLB); + } else if (type == FLUSH_TYPE_GLOBAL) { if (!mmu_has_feature(MMU_FTR_GTSE)) { unsigned long tgt = H_RPTI_TARGET_CMMU; @@ -728,9 +843,6 @@ void radix__flush_tlb_mm(struct mm_struct *mm) } else { _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB); } - } else { -local: - _tlbiel_pid(pid, RIC_FLUSH_TLB); } preempt_enable(); } @@ -739,6 +851,7 @@ EXPORT_SYMBOL(radix__flush_tlb_mm); static void __flush_all_mm(struct mm_struct *mm, bool fullmm) { unsigned long pid; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) @@ -746,13 +859,10 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm) preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - if (!fullmm) { - exit_flush_lazy_tlbs(mm); - goto local; - } - } + type = flush_type_needed(mm, fullmm); + if (type == FLUSH_TYPE_LOCAL) { + _tlbiel_pid(pid, RIC_FLUSH_ALL); + } else if (type == FLUSH_TYPE_GLOBAL) { if (!mmu_has_feature(MMU_FTR_GTSE)) { unsigned long tgt = H_RPTI_TARGET_CMMU; unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | @@ -766,9 +876,6 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm) _tlbie_pid(pid, RIC_FLUSH_ALL); else _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL); - } else { -local: - _tlbiel_pid(pid, RIC_FLUSH_ALL); } preempt_enable(); } @@ -783,6 +890,7 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, int psize) { unsigned long pid; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) @@ -790,11 +898,10 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - exit_flush_lazy_tlbs(mm); - goto local; - } + type = flush_type_needed(mm, false); + if (type == FLUSH_TYPE_LOCAL) { + _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); + } else if (type == FLUSH_TYPE_GLOBAL) { if (!mmu_has_feature(MMU_FTR_GTSE)) { unsigned long tgt, pg_sizes, size; @@ -811,9 +918,6 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB); else _tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB); - } else { -local: - _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); } preempt_enable(); } @@ -828,8 +932,6 @@ void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) } EXPORT_SYMBOL(radix__flush_tlb_page); -#else /* CONFIG_SMP */ -static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { } #endif /* CONFIG_SMP */ static void do_tlbiel_kernel(void *info) @@ -893,7 +995,9 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift; unsigned long page_size = 1UL << page_shift; unsigned long nr_pages = (end - start) >> page_shift; - bool local, full; + bool fullmm = (end == TLB_FLUSH_ALL); + bool flush_pid; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) @@ -901,24 +1005,18 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - if (end != TLB_FLUSH_ALL) { - exit_flush_lazy_tlbs(mm); - goto is_local; - } - } - local = false; - full = (end == TLB_FLUSH_ALL || - nr_pages > tlb_single_page_flush_ceiling); - } else { -is_local: - local = true; - full = (end == TLB_FLUSH_ALL || - nr_pages > tlb_local_single_page_flush_ceiling); - } + type = flush_type_needed(mm, fullmm); + if (type == FLUSH_TYPE_NONE) + goto out; - if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { + if (fullmm) + flush_pid = true; + else if (type == FLUSH_TYPE_GLOBAL) + flush_pid = nr_pages > tlb_single_page_flush_ceiling; + else + flush_pid = nr_pages > tlb_local_single_page_flush_ceiling; + + if (!mmu_has_feature(MMU_FTR_GTSE) && type == FLUSH_TYPE_GLOBAL) { unsigned long tgt = H_RPTI_TARGET_CMMU; unsigned long pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize); @@ -928,8 +1026,8 @@ is_local: tgt |= H_RPTI_TARGET_NMMU; pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, pg_sizes, start, end); - } else if (full) { - if (local) { + } else if (flush_pid) { + if (type == FLUSH_TYPE_LOCAL) { _tlbiel_pid(pid, RIC_FLUSH_TLB); } else { if (cputlb_use_tlbie()) { @@ -952,7 +1050,7 @@ is_local: hflush = true; } - if (local) { + if (type == FLUSH_TYPE_LOCAL) { asm volatile("ptesync": : :"memory"); __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize); if (hflush) @@ -974,6 +1072,7 @@ is_local: hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, false); } } +out: preempt_enable(); } @@ -1085,32 +1184,30 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned int page_shift = mmu_psize_defs[psize].shift; unsigned long page_size = 1UL << page_shift; unsigned long nr_pages = (end - start) >> page_shift; - bool local, full; + bool fullmm = (end == TLB_FLUSH_ALL); + bool flush_pid; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) return; + fullmm = (end == TLB_FLUSH_ALL); + preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - if (end != TLB_FLUSH_ALL) { - exit_flush_lazy_tlbs(mm); - goto is_local; - } - } - local = false; - full = (end == TLB_FLUSH_ALL || - nr_pages > tlb_single_page_flush_ceiling); - } else { -is_local: - local = true; - full = (end == TLB_FLUSH_ALL || - nr_pages > tlb_local_single_page_flush_ceiling); - } + type = flush_type_needed(mm, fullmm); + if (type == FLUSH_TYPE_NONE) + goto out; - if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { + if (fullmm) + flush_pid = true; + else if (type == FLUSH_TYPE_GLOBAL) + flush_pid = nr_pages > tlb_single_page_flush_ceiling; + else + flush_pid = nr_pages > tlb_local_single_page_flush_ceiling; + + if (!mmu_has_feature(MMU_FTR_GTSE) && type == FLUSH_TYPE_GLOBAL) { unsigned long tgt = H_RPTI_TARGET_CMMU; unsigned long type = H_RPTI_TYPE_TLB; unsigned long pg_sizes = psize_to_rpti_pgsize(psize); @@ -1120,8 +1217,8 @@ is_local: if (atomic_read(&mm->context.copros) > 0) tgt |= H_RPTI_TARGET_NMMU; pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end); - } else if (full) { - if (local) { + } else if (flush_pid) { + if (type == FLUSH_TYPE_LOCAL) { _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); } else { if (cputlb_use_tlbie()) { @@ -1137,7 +1234,7 @@ is_local: } } else { - if (local) + if (type == FLUSH_TYPE_LOCAL) _tlbiel_va_range(start, end, pid, page_size, psize, also_pwc); else if (cputlb_use_tlbie()) _tlbie_va_range(start, end, pid, page_size, psize, also_pwc); @@ -1145,6 +1242,7 @@ is_local: _tlbiel_va_range_multicast(mm, start, end, pid, page_size, psize, also_pwc); } +out: preempt_enable(); } @@ -1164,6 +1262,7 @@ static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) { unsigned long pid, end; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) @@ -1180,11 +1279,10 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) /* Otherwise first do the PWC, then iterate the pages. */ preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - exit_flush_lazy_tlbs(mm); - goto local; - } + type = flush_type_needed(mm, false); + if (type == FLUSH_TYPE_LOCAL) { + _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); + } else if (type == FLUSH_TYPE_GLOBAL) { if (!mmu_has_feature(MMU_FTR_GTSE)) { unsigned long tgt, type, pg_sizes; @@ -1202,9 +1300,6 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) else _tlbiel_va_range_multicast(mm, addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); - } else { -local: - _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); } preempt_enable(); diff --git a/arch/powerpc/mm/book3s64/slb.c b/arch/powerpc/mm/book3s64/slb.c index 584567970c11..c91bd85eb90e 100644 --- a/arch/powerpc/mm/book3s64/slb.c +++ b/arch/powerpc/mm/book3s64/slb.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -813,8 +814,9 @@ static long slb_allocate_user(struct mm_struct *mm, unsigned long ea) return slb_insert_entry(ea, context, flags, ssize, false); } -long do_slb_fault(struct pt_regs *regs, unsigned long ea) +DEFINE_INTERRUPT_HANDLER_RAW(do_slb_fault) { + unsigned long ea = regs->dar; unsigned long id = get_region_id(ea); /* IRQs are not reconciled here, so can't check irqs_disabled */ @@ -824,19 +826,21 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea) return -EINVAL; /* - * SLB kernel faults must be very careful not to touch anything - * that is not bolted. E.g., PACA and global variables are okay, - * mm->context stuff is not. - * - * SLB user faults can access all of kernel memory, but must be - * careful not to touch things like IRQ state because it is not - * "reconciled" here. The difficulty is that we must use - * fast_exception_return to return from kernel SLB faults without - * looking at possible non-bolted memory. We could test user vs - * kernel faults in the interrupt handler asm and do a full fault, - * reconcile, ret_from_except for user faults which would make them - * first class kernel code. But for performance it's probably nicer - * if they go via fast_exception_return too. + * SLB kernel faults must be very careful not to touch anything that is + * not bolted. E.g., PACA and global variables are okay, mm->context + * stuff is not. SLB user faults may access all of memory (and induce + * one recursive SLB kernel fault), so the kernel fault must not + * trample on the user fault state at those points. + */ + + /* + * This is a raw interrupt handler, for performance, so that + * fast_interrupt_return can be used. The handler must not touch local + * irq state, or schedule. We could test for usermode and upgrade to a + * normal process context (synchronous) interrupt for those, which + * would make them first-class kernel code and able to be traced and + * instrumented, although performance would suffer a bit, it would + * probably be a good tradeoff. */ if (id >= LINEAR_MAP_REGION_ID) { long err; @@ -865,13 +869,15 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea) } } -void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err) +DEFINE_INTERRUPT_HANDLER(do_bad_slb_fault) { + int err = regs->result; + if (err == -EFAULT) { if (user_mode(regs)) - _exception(SIGSEGV, regs, SEGV_BNDERR, ea); + _exception(SIGSEGV, regs, SEGV_BNDERR, regs->dar); else - bad_page_fault(regs, ea, SIGSEGV); + bad_page_fault(regs, SIGSEGV); } else if (err == -EINVAL) { unrecoverable_exception(regs); } else { diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 8961b44f350c..bb368257b55c 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -377,18 +378,16 @@ static void sanity_check_fault(bool is_write, bool is_user, /* * For 600- and 800-family processors, the error_code parameter is DSISR - * for a data fault, SRR1 for an instruction fault. For 400-family processors - * the error_code parameter is ESR for a data fault, 0 for an instruction - * fault. - * For 64-bit processors, the error_code parameter is - * - DSISR for a non-SLB data access fault, - * - SRR1 & 0x08000000 for a non-SLB instruction access fault - * - 0 any SLB fault. + * for a data fault, SRR1 for an instruction fault. + * For 400-family processors the error_code parameter is ESR for a data fault, + * 0 for an instruction fault. + * For 64-bit processors, the error_code parameter is DSISR for a data access + * fault, SRR1 & 0x08000000 for an instruction access fault. * * The return value is 0 if the fault was handled, or the signal * number if this is a kernel fault that can't be handled here. */ -static int __do_page_fault(struct pt_regs *regs, unsigned long address, +static int ___do_page_fault(struct pt_regs *regs, unsigned long address, unsigned long error_code) { struct vm_area_struct * vma; @@ -435,9 +434,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, return bad_area_nosemaphore(regs, address); } - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); @@ -540,34 +537,51 @@ retry: return 0; } -NOKPROBE_SYMBOL(__do_page_fault); +NOKPROBE_SYMBOL(___do_page_fault); -int do_page_fault(struct pt_regs *regs, unsigned long address, - unsigned long error_code) +static long __do_page_fault(struct pt_regs *regs) { const struct exception_table_entry *entry; - enum ctx_state prev_state = exception_enter(); - int rc = __do_page_fault(regs, address, error_code); - exception_exit(prev_state); - if (likely(!rc)) - return 0; + long err; + + err = ___do_page_fault(regs, regs->dar, regs->dsisr); + if (likely(!err)) + return err; entry = search_exception_tables(regs->nip); - if (unlikely(!entry)) - return rc; - - instruction_pointer_set(regs, extable_fixup(entry)); - - return 0; + if (likely(entry)) { + instruction_pointer_set(regs, extable_fixup(entry)); + return 0; + } else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) { + __bad_page_fault(regs, err); + return 0; + } else { + /* 32 and 64e handle the bad page fault in asm */ + return err; + } } -NOKPROBE_SYMBOL(do_page_fault); +NOKPROBE_SYMBOL(__do_page_fault); + +DEFINE_INTERRUPT_HANDLER_RET(do_page_fault) +{ + return __do_page_fault(regs); +} + +#ifdef CONFIG_PPC_BOOK3S_64 +/* Same as do_page_fault but interrupt entry has already run in do_hash_fault */ +long hash__do_page_fault(struct pt_regs *regs) +{ + return __do_page_fault(regs); +} +NOKPROBE_SYMBOL(hash__do_page_fault); +#endif /* * bad_page_fault is called when we have a bad access from the kernel. * It is called from the DSI and ISI handlers in head.S and from some * of the procedures in traps.c. */ -void __bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) +void __bad_page_fault(struct pt_regs *regs, int sig) { int is_write = page_fault_is_write(regs->dsisr); @@ -605,7 +619,7 @@ void __bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) die("Kernel access of bad area", regs, sig); } -void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) +void bad_page_fault(struct pt_regs *regs, int sig) { const struct exception_table_entry *entry; @@ -614,5 +628,12 @@ void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) if (entry) instruction_pointer_set(regs, extable_fixup(entry)); else - __bad_page_fault(regs, address, sig); + __bad_page_fault(regs, sig); } + +#ifdef CONFIG_PPC_BOOK3S_64 +DEFINE_INTERRUPT_HANDLER(do_bad_page_fault_segv) +{ + bad_page_fault(regs, SIGSEGV); +} +#endif diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 8b3cc4d688e8..d142b76d507d 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -217,7 +217,7 @@ void __init pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_p } } -int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) +static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) { struct huge_bootmem_page *m; if (nr_gpages == 0) @@ -663,24 +663,6 @@ static int __init hugetlbpage_init(void) arch_initcall(hugetlbpage_init); -void flush_dcache_icache_hugepage(struct page *page) -{ - int i; - void *start; - - BUG_ON(!PageCompound(page)); - - for (i = 0; i < compound_nr(page); i++) { - if (!PageHighMem(page)) { - __flush_dcache_icache(page_address(page+i)); - } else { - start = kmap_atomic(page+i); - __flush_dcache_icache(start); - kunmap_atomic(start); - } - } -} - void __init gigantic_hugetlb_cma_reserve(void) { unsigned long order = 0; diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index afab328d0887..4e8ce6d85232 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -91,27 +91,6 @@ int __weak remove_section_mapping(unsigned long start, unsigned long end) return -ENODEV; } -#define FLUSH_CHUNK_SIZE SZ_1G -/** - * flush_dcache_range_chunked(): Write any modified data cache blocks out to - * memory and invalidate them, in chunks of up to FLUSH_CHUNK_SIZE - * Does not invalidate the corresponding instruction cache blocks. - * - * @start: the start address - * @stop: the stop address (exclusive) - * @chunk: the max size of the chunks - */ -static void flush_dcache_range_chunked(unsigned long start, unsigned long stop, - unsigned long chunk) -{ - unsigned long i; - - for (i = start; i < stop; i += chunk) { - flush_dcache_range(i, min(stop, i + chunk)); - cond_resched(); - } -} - int __ref arch_create_linear_mapping(int nid, u64 start, u64 size, struct mhp_params *params) { @@ -136,7 +115,6 @@ void __ref arch_remove_linear_mapping(u64 start, u64 size) /* Remove htab bolted mappings for this section of memory */ start = (unsigned long)__va(start); - flush_dcache_range_chunked(start, start + size, FLUSH_CHUNK_SIZE); mutex_lock(&linear_mapping_mutex); ret = remove_section_mapping(start, start + size); @@ -489,19 +467,35 @@ void flush_dcache_page(struct page *page) if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) return; /* avoid an atomic op if possible */ - if (test_bit(PG_arch_1, &page->flags)) - clear_bit(PG_arch_1, &page->flags); + if (test_bit(PG_dcache_clean, &page->flags)) + clear_bit(PG_dcache_clean, &page->flags); } EXPORT_SYMBOL(flush_dcache_page); +static void flush_dcache_icache_hugepage(struct page *page) +{ + int i; + void *start; + + BUG_ON(!PageCompound(page)); + + for (i = 0; i < compound_nr(page); i++) { + if (!PageHighMem(page)) { + __flush_dcache_icache(page_address(page+i)); + } else { + start = kmap_atomic(page+i); + __flush_dcache_icache(start); + kunmap_atomic(start); + } + } +} + void flush_dcache_icache_page(struct page *page) { -#ifdef CONFIG_HUGETLB_PAGE - if (PageCompound(page)) { - flush_dcache_icache_hugepage(page); - return; - } -#endif + + if (PageCompound(page)) + return flush_dcache_icache_hugepage(page); + #if defined(CONFIG_PPC_8xx) || defined(CONFIG_PPC64) /* On 8xx there is no need to kmap since highmem is not supported */ __flush_dcache_icache(page_address(page)); diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index 15555c95cebc..354611940118 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -26,6 +26,7 @@ #include #include #include +#include static inline int is_exec_fault(void) { @@ -81,9 +82,9 @@ static pte_t set_pte_filter_hash(pte_t pte) struct page *pg = maybe_pte_to_page(pte); if (!pg) return pte; - if (!test_bit(PG_arch_1, &pg->flags)) { + if (!test_bit(PG_dcache_clean, &pg->flags)) { flush_dcache_icache_page(pg); - set_bit(PG_arch_1, &pg->flags); + set_bit(PG_dcache_clean, &pg->flags); } } return pte; @@ -116,13 +117,13 @@ static inline pte_t set_pte_filter(pte_t pte) return pte; /* If the page clean, we move on */ - if (test_bit(PG_arch_1, &pg->flags)) + if (test_bit(PG_dcache_clean, &pg->flags)) return pte; /* If it's an exec fault, we flush the cache and make it clean */ if (is_exec_fault()) { flush_dcache_icache_page(pg); - set_bit(PG_arch_1, &pg->flags); + set_bit(PG_dcache_clean, &pg->flags); return pte; } @@ -161,12 +162,12 @@ static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma, goto bail; /* If the page is already clean, we move on */ - if (test_bit(PG_arch_1, &pg->flags)) + if (test_bit(PG_dcache_clean, &pg->flags)) goto bail; - /* Clean the page and set PG_arch_1 */ + /* Clean the page and set PG_dcache_clean */ flush_dcache_icache_page(pg); - set_bit(PG_arch_1, &pg->flags); + set_bit(PG_dcache_clean, &pg->flags); bail: return pte_mkexec(pte); diff --git a/arch/powerpc/mm/ptdump/segment_regs.c b/arch/powerpc/mm/ptdump/segment_regs.c index dde2fe8de4b2..565048a0c9be 100644 --- a/arch/powerpc/mm/ptdump/segment_regs.c +++ b/arch/powerpc/mm/ptdump/segment_regs.c @@ -10,7 +10,7 @@ static void seg_show(struct seq_file *m, int i) { - u32 val = mfsrin(i << 28); + u32 val = mfsr(i << 28); seq_printf(m, "0x%01x0000000-0x%01xfffffff ", i, i); seq_printf(m, "Kern key %d ", (val >> 30) & 1); diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 869d999a836e..6817331e22ff 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -54,6 +54,9 @@ struct cpu_hw_events { struct perf_branch_stack bhrb_stack; struct perf_branch_entry bhrb_entries[BHRB_MAX_ENTRIES]; u64 ic_init; + + /* Store the PMC values */ + unsigned long pmcs[MAX_HWEVENTS]; }; static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); @@ -110,10 +113,6 @@ static inline void perf_read_regs(struct pt_regs *regs) { regs->result = 0; } -static inline int perf_intr_is_nmi(struct pt_regs *regs) -{ - return 0; -} static inline int siar_valid(struct pt_regs *regs) { @@ -147,6 +146,17 @@ bool is_sier_available(void) return false; } +/* + * Return PMC value corresponding to the + * index passed. + */ +unsigned long get_pmcs_ext_regs(int idx) +{ + struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); + + return cpuhw->pmcs[idx]; +} + static bool regs_use_siar(struct pt_regs *regs) { /* @@ -353,15 +363,6 @@ static inline void perf_read_regs(struct pt_regs *regs) regs->result = use_siar; } -/* - * If interrupts were soft-disabled when a PMU interrupt occurs, treat - * it as an NMI. - */ -static inline int perf_intr_is_nmi(struct pt_regs *regs) -{ - return (regs->softe & IRQS_DISABLED); -} - /* * On processors like P7+ that have the SIAR-Valid bit, marked instructions * must be sampled only if the SIAR-valid bit is set. @@ -915,7 +916,7 @@ void perf_event_print_debug(void) */ static int power_check_constraints(struct cpu_hw_events *cpuhw, u64 event_id[], unsigned int cflags[], - int n_ev) + int n_ev, struct perf_event **event) { unsigned long mask, value, nv; unsigned long smasks[MAX_HWEVENTS], svalues[MAX_HWEVENTS]; @@ -938,7 +939,7 @@ static int power_check_constraints(struct cpu_hw_events *cpuhw, event_id[i] = cpuhw->alternatives[i][0]; } if (ppmu->get_constraint(event_id[i], &cpuhw->amasks[i][0], - &cpuhw->avalues[i][0])) + &cpuhw->avalues[i][0], event[i]->attr.config1)) return -1; } value = mask = 0; @@ -973,7 +974,8 @@ static int power_check_constraints(struct cpu_hw_events *cpuhw, for (j = 1; j < n_alt[i]; ++j) ppmu->get_constraint(cpuhw->alternatives[i][j], &cpuhw->amasks[i][j], - &cpuhw->avalues[i][j]); + &cpuhw->avalues[i][j], + event[i]->attr.config1); } /* enumerate all possibilities and see if any will work */ @@ -1391,7 +1393,7 @@ static void power_pmu_enable(struct pmu *pmu) memset(&cpuhw->mmcr, 0, sizeof(cpuhw->mmcr)); if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index, - &cpuhw->mmcr, cpuhw->event)) { + &cpuhw->mmcr, cpuhw->event, ppmu->flags)) { /* shouldn't ever get here */ printk(KERN_ERR "oops compute_mmcr failed\n"); goto out; @@ -1579,7 +1581,7 @@ static int power_pmu_add(struct perf_event *event, int ef_flags) if (check_excludes(cpuhw->event, cpuhw->flags, n0, 1)) goto out; - if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1)) + if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1, cpuhw->event)) goto out; event->hw.config = cpuhw->events[n0]; @@ -1789,7 +1791,7 @@ static int power_pmu_commit_txn(struct pmu *pmu) n = cpuhw->n_events; if (check_excludes(cpuhw->event, cpuhw->flags, 0, n)) return -EAGAIN; - i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n); + i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n, cpuhw->event); if (i < 0) return -EAGAIN; @@ -2027,7 +2029,7 @@ static int power_pmu_event_init(struct perf_event *event) local_irq_save(irq_flags); cpuhw = this_cpu_ptr(&cpu_hw_events); - err = power_check_constraints(cpuhw, events, cflags, n + 1); + err = power_check_constraints(cpuhw, events, cflags, n + 1, ctrs); if (has_branch_stack(event)) { u64 bhrb_filter = -1; @@ -2149,7 +2151,17 @@ static void record_and_restart(struct perf_event *event, unsigned long val, left += period; if (left <= 0) left = period; - record = siar_valid(regs); + + /* + * If address is not requested in the sample via + * PERF_SAMPLE_IP, just record that sample irrespective + * of SIAR valid check. + */ + if (event->attr.sample_type & PERF_SAMPLE_IP) + record = siar_valid(regs); + else + record = 1; + event->hw.last_period = event->hw.sample_period; } if (left < 0x80000000LL) @@ -2167,9 +2179,10 @@ static void record_and_restart(struct perf_event *event, unsigned long val, * MMCR2. Check attr.exclude_kernel and address to drop the sample in * these cases. */ - if (event->attr.exclude_kernel && record) - if (is_kernel_addr(mfspr(SPRN_SIAR))) - record = 0; + if (event->attr.exclude_kernel && + (event->attr.sample_type & PERF_SAMPLE_IP) && + is_kernel_addr(mfspr(SPRN_SIAR))) + record = 0; /* * Finally record data if requested. @@ -2277,9 +2290,7 @@ static void __perf_event_interrupt(struct pt_regs *regs) int i, j; struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); struct perf_event *event; - unsigned long val[8]; int found, active; - int nmi; if (cpuhw->n_limited) freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5), @@ -2287,26 +2298,14 @@ static void __perf_event_interrupt(struct pt_regs *regs) perf_read_regs(regs); - /* - * If perf interrupts hit in a local_irq_disable (soft-masked) region, - * we consider them as NMIs. This is required to prevent hash faults on - * user addresses when reading callchains. See the NMI test in - * do_hash_page. - */ - nmi = perf_intr_is_nmi(regs); - if (nmi) - nmi_enter(); - else - irq_enter(); - /* Read all the PMCs since we'll need them a bunch of times */ for (i = 0; i < ppmu->n_counter; ++i) - val[i] = read_pmc(i + 1); + cpuhw->pmcs[i] = read_pmc(i + 1); /* Try to find what caused the IRQ */ found = 0; for (i = 0; i < ppmu->n_counter; ++i) { - if (!pmc_overflow(val[i])) + if (!pmc_overflow(cpuhw->pmcs[i])) continue; if (is_limited_pmc(i + 1)) continue; /* these won't generate IRQs */ @@ -2321,7 +2320,7 @@ static void __perf_event_interrupt(struct pt_regs *regs) event = cpuhw->event[j]; if (event->hw.idx == (i + 1)) { active = 1; - record_and_restart(event, val[i], regs); + record_and_restart(event, cpuhw->pmcs[i], regs); break; } } @@ -2335,17 +2334,17 @@ static void __perf_event_interrupt(struct pt_regs *regs) event = cpuhw->event[i]; if (!event->hw.idx || is_limited_pmc(event->hw.idx)) continue; - if (pmc_overflow_power7(val[event->hw.idx - 1])) { + if (pmc_overflow_power7(cpuhw->pmcs[event->hw.idx - 1])) { /* event has overflowed in a buggy way*/ found = 1; record_and_restart(event, - val[event->hw.idx - 1], + cpuhw->pmcs[event->hw.idx - 1], regs); } } } - if (!found && !nmi && printk_ratelimit()) - printk(KERN_WARNING "Can't find PMC that caused IRQ\n"); + if (unlikely(!found) && !arch_irq_disabled_regs(regs)) + printk_ratelimited(KERN_WARNING "Can't find PMC that caused IRQ\n"); /* * Reset MMCR0 to its normal value. This will set PMXE and @@ -2356,10 +2355,9 @@ static void __perf_event_interrupt(struct pt_regs *regs) */ write_mmcr0(cpuhw, cpuhw->mmcr.mmcr0); - if (nmi) - nmi_exit(); - else - irq_exit(); + /* Clear the cpuhw->pmcs */ + memset(&cpuhw->pmcs, 0, sizeof(cpuhw->pmcs)); + } static void perf_event_interrupt(struct pt_regs *regs) diff --git a/arch/powerpc/perf/core-fsl-emb.c b/arch/powerpc/perf/core-fsl-emb.c index e0e7e276bfd2..ee721f420a7b 100644 --- a/arch/powerpc/perf/core-fsl-emb.c +++ b/arch/powerpc/perf/core-fsl-emb.c @@ -31,19 +31,6 @@ static atomic_t num_events; /* Used to avoid races in calling reserve/release_pmc_hardware */ static DEFINE_MUTEX(pmc_reserve_mutex); -/* - * If interrupts were soft-disabled when a PMU interrupt occurs, treat - * it as an NMI. - */ -static inline int perf_intr_is_nmi(struct pt_regs *regs) -{ -#ifdef __powerpc64__ - return (regs->softe & IRQS_DISABLED); -#else - return 0; -#endif -} - static void perf_event_interrupt(struct pt_regs *regs); /* @@ -659,13 +646,6 @@ static void perf_event_interrupt(struct pt_regs *regs) struct perf_event *event; unsigned long val; int found = 0; - int nmi; - - nmi = perf_intr_is_nmi(regs); - if (nmi) - nmi_enter(); - else - irq_enter(); for (i = 0; i < ppmu->n_counter; ++i) { event = cpuhw->event[i]; @@ -690,11 +670,6 @@ static void perf_event_interrupt(struct pt_regs *regs) mtmsr(mfmsr() | MSR_PMM); mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE); isync(); - - if (nmi) - nmi_exit(); - else - irq_exit(); } void hw_perf_event_setup(int cpu) diff --git a/arch/powerpc/perf/hv-24x7.c b/arch/powerpc/perf/hv-24x7.c index 6e7e820508df..e5eb33255066 100644 --- a/arch/powerpc/perf/hv-24x7.c +++ b/arch/powerpc/perf/hv-24x7.c @@ -764,6 +764,14 @@ static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event, return ev_len; } +/* + * Return true incase of invalid or dummy events with names like RESERVED* + */ +static bool ignore_event(const char *name) +{ + return strncmp(name, "RESERVED", 8) == 0; +} + #define MAX_4K (SIZE_MAX / 4096) static int create_events_from_catalog(struct attribute ***events_, @@ -894,6 +902,10 @@ static int create_events_from_catalog(struct attribute ***events_, name = event_name(event, &nl); + if (ignore_event(name)) { + junk_events++; + continue; + } if (event->event_group_record_len == 0) { pr_devel("invalid event %zu (%.*s): group_record_len == 0, skipping\n", event_idx, nl, name); @@ -955,6 +967,9 @@ static int create_events_from_catalog(struct attribute ***events_, continue; name = event_name(event, &nl); + if (ignore_event(name)) + continue; + nonce = event_uniq_add(&ev_uniq, name, nl, event->domain); ct = event_data_to_attrs(event_idx, events + event_attr_ct, event, nonce); diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c index 6ab5b272090a..e4f577da33d8 100644 --- a/arch/powerpc/perf/isa207-common.c +++ b/arch/powerpc/perf/isa207-common.c @@ -108,12 +108,57 @@ static void mmcra_sdar_mode(u64 event, unsigned long *mmcra) *mmcra |= MMCRA_SDAR_MODE_TLB; } +static u64 p10_thresh_cmp_val(u64 value) +{ + int exp = 0; + u64 result = value; + + if (!value) + return value; + + /* + * Incase of P10, thresh_cmp value is not part of raw event code + * and provided via attr.config1 parameter. To program threshold in MMCRA, + * take a 18 bit number N and shift right 2 places and increment + * the exponent E by 1 until the upper 10 bits of N are zero. + * Write E to the threshold exponent and write the lower 8 bits of N + * to the threshold mantissa. + * The max threshold that can be written is 261120. + */ + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + if (value > 261120) + value = 261120; + while ((64 - __builtin_clzl(value)) > 8) { + exp++; + value >>= 2; + } + + /* + * Note that it is invalid to write a mantissa with the + * upper 2 bits of mantissa being zero, unless the + * exponent is also zero. + */ + if (!(value & 0xC0) && exp) + result = 0; + else + result = (exp << 8) | value; + } + return result; +} + static u64 thresh_cmp_val(u64 value) { + if (cpu_has_feature(CPU_FTR_ARCH_31)) + value = p10_thresh_cmp_val(value); + + /* + * Since location of threshold compare bits in MMCRA + * is different for p8, using different shift value. + */ if (cpu_has_feature(CPU_FTR_ARCH_300)) return value << p9_MMCRA_THR_CMP_SHIFT; - - return value << MMCRA_THR_CMP_SHIFT; + else + return value << MMCRA_THR_CMP_SHIFT; } static unsigned long combine_from_event(u64 event) @@ -141,13 +186,13 @@ static bool is_thresh_cmp_valid(u64 event) { unsigned int cmp, exp; + if (cpu_has_feature(CPU_FTR_ARCH_31)) + return p10_thresh_cmp_val(event) != 0; + /* * Check the mantissa upper two bits are not zero, unless the * exponent is also zero. See the THRESH_CMP_MANTISSA doc. - * Power10: thresh_cmp is replaced by l2_l3 event select. */ - if (cpu_has_feature(CPU_FTR_ARCH_31)) - return false; cmp = (event >> EVENT_THR_CMP_SHIFT) & EVENT_THR_CMP_MASK; exp = cmp >> 7; @@ -256,7 +301,7 @@ void isa207_get_mem_weight(u64 *weight) *weight = mantissa << (2 * exp); } -int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp) +int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp, u64 event_config1) { unsigned int unit, pmc, cache, ebb; unsigned long mask, value; @@ -355,9 +400,11 @@ int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp) } if (cpu_has_feature(CPU_FTR_ARCH_31)) { - if (event_is_threshold(event)) { + if (event_is_threshold(event) && is_thresh_cmp_valid(event_config1)) { mask |= CNST_THRESH_CTL_SEL_MASK; value |= CNST_THRESH_CTL_SEL_VAL(event >> EVENT_THRESH_SHIFT); + mask |= p10_CNST_THRESH_CMP_MASK; + value |= p10_CNST_THRESH_CMP_VAL(p10_thresh_cmp_val(event_config1)); } } else if (cpu_has_feature(CPU_FTR_ARCH_300)) { if (event_is_threshold(event) && is_thresh_cmp_valid(event)) { @@ -411,7 +458,7 @@ ebb_bhrb: int isa207_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], u32 flags) { unsigned long mmcra, mmcr1, mmcr2, unit, combine, psel, cache, val; unsigned long mmcr3; @@ -504,6 +551,10 @@ int isa207_compute_mmcr(u64 event[], int n_ev, val = (event[i] >> EVENT_THR_CMP_SHIFT) & EVENT_THR_CMP_MASK; mmcra |= thresh_cmp_val(val); + } else if (flags & PPMU_HAS_ATTR_CONFIG1) { + val = (pevents[i]->attr.config1 >> p10_EVENT_THR_CMP_SHIFT) & + p10_EVENT_THR_CMP_MASK; + mmcra |= thresh_cmp_val(val); } } diff --git a/arch/powerpc/perf/isa207-common.h b/arch/powerpc/perf/isa207-common.h index 454b32c31440..1af0e8c97ac7 100644 --- a/arch/powerpc/perf/isa207-common.h +++ b/arch/powerpc/perf/isa207-common.h @@ -105,6 +105,10 @@ #define p10_EVENT_RADIX_SCOPE_QUAL_MASK 0x1 #define p10_MMCR1_RADIX_SCOPE_QUAL_SHIFT 45 +/* Event Threshold Compare bit constant for power10 in config1 attribute */ +#define p10_EVENT_THR_CMP_SHIFT 0 +#define p10_EVENT_THR_CMP_MASK 0x3FFFFull + #define p10_EVENT_VALID_MASK \ ((p10_SDAR_MODE_MASK << p10_SDAR_MODE_SHIFT | \ (p10_EVENT_THRESH_MASK << EVENT_THRESH_SHIFT) | \ @@ -124,8 +128,8 @@ * 60 56 52 48 44 40 36 32 * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | * [ fab_match ] [ thresh_cmp ] [ thresh_ctl ] [ ] - * | - * thresh_sel -* + * | | + * [ thresh_cmp bits for p10] thresh_sel -* * * 28 24 20 16 12 8 4 0 * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | @@ -152,6 +156,9 @@ #define CNST_THRESH_CTL_SEL_VAL(v) (((v) & 0x7ffull) << 32) #define CNST_THRESH_CTL_SEL_MASK CNST_THRESH_CTL_SEL_VAL(0x7ff) +#define p10_CNST_THRESH_CMP_VAL(v) (((v) & 0x7ffull) << 43) +#define p10_CNST_THRESH_CMP_MASK p10_CNST_THRESH_CMP_VAL(0x7ff) + #define CNST_EBB_VAL(v) (((v) & EVENT_EBB_MASK) << 24) #define CNST_EBB_MASK CNST_EBB_VAL(EVENT_EBB_MASK) @@ -262,10 +269,10 @@ #define PH(a, b) (P(LVL, HIT) | P(a, b)) #define PM(a, b) (P(LVL, MISS) | P(a, b)) -int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp); +int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp, u64 event_config1); int isa207_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]); + struct perf_event *pevents[], u32 flags); void isa207_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr); int isa207_get_alternatives(u64 event, u64 alt[], int size, unsigned int flags, const unsigned int ev_alt[][MAX_ALT]); diff --git a/arch/powerpc/perf/mpc7450-pmu.c b/arch/powerpc/perf/mpc7450-pmu.c index 1919e9df9165..e39b15b79a83 100644 --- a/arch/powerpc/perf/mpc7450-pmu.c +++ b/arch/powerpc/perf/mpc7450-pmu.c @@ -148,7 +148,7 @@ static u32 classbits[N_CLASSES - 1][2] = { }; static int mpc7450_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, class; u32 mask, value; @@ -258,7 +258,8 @@ static const u32 pmcsel_mask[N_COUNTER] = { */ static int mpc7450_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], + u32 flags __maybe_unused) { u8 event_index[N_CLASSES][N_COUNTER]; int n_classevent[N_CLASSES]; diff --git a/arch/powerpc/perf/perf_regs.c b/arch/powerpc/perf/perf_regs.c index 6f681b105eec..b931eed482c9 100644 --- a/arch/powerpc/perf/perf_regs.c +++ b/arch/powerpc/perf/perf_regs.c @@ -75,6 +75,8 @@ static unsigned int pt_regs_offset[PERF_REG_POWERPC_MAX] = { static u64 get_ext_regs_value(int idx) { switch (idx) { + case PERF_REG_POWERPC_PMC1 ... PERF_REG_POWERPC_PMC6: + return get_pmcs_ext_regs(idx - PERF_REG_POWERPC_PMC1); case PERF_REG_POWERPC_MMCR0: return mfspr(SPRN_MMCR0); case PERF_REG_POWERPC_MMCR1: @@ -95,13 +97,6 @@ static u64 get_ext_regs_value(int idx) u64 perf_reg_value(struct pt_regs *regs, int idx) { - u64 perf_reg_extended_max = PERF_REG_POWERPC_MAX; - - if (cpu_has_feature(CPU_FTR_ARCH_31)) - perf_reg_extended_max = PERF_REG_MAX_ISA_31; - else if (cpu_has_feature(CPU_FTR_ARCH_300)) - perf_reg_extended_max = PERF_REG_MAX_ISA_300; - if (idx == PERF_REG_POWERPC_SIER && (IS_ENABLED(CONFIG_FSL_EMB_PERF_EVENT) || IS_ENABLED(CONFIG_PPC32) || @@ -113,14 +108,14 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) IS_ENABLED(CONFIG_PPC32))) return 0; - if (idx >= PERF_REG_POWERPC_MAX && idx < perf_reg_extended_max) + if (idx >= PERF_REG_POWERPC_MAX && idx < PERF_REG_EXTENDED_MAX) return get_ext_regs_value(idx); /* * If the idx is referring to value beyond the * supported registers, return 0 with a warning */ - if (WARN_ON_ONCE(idx >= perf_reg_extended_max)) + if (WARN_ON_ONCE(idx >= PERF_REG_EXTENDED_MAX)) return 0; return regs_get_register(regs, pt_regs_offset[idx]); diff --git a/arch/powerpc/perf/power10-pmu.c b/arch/powerpc/perf/power10-pmu.c index 79e0206ca454..a901c1348cad 100644 --- a/arch/powerpc/perf/power10-pmu.c +++ b/arch/powerpc/perf/power10-pmu.c @@ -216,6 +216,7 @@ PMU_FORMAT_ATTR(invert_bit, "config:47"); PMU_FORMAT_ATTR(src_mask, "config:48-53"); PMU_FORMAT_ATTR(src_match, "config:54-59"); PMU_FORMAT_ATTR(radix_scope, "config:9"); +PMU_FORMAT_ATTR(thresh_cmp, "config1:0-17"); static struct attribute *power10_pmu_format_attr[] = { &format_attr_event.attr, @@ -236,6 +237,7 @@ static struct attribute *power10_pmu_format_attr[] = { &format_attr_src_mask.attr, &format_attr_src_match.attr, &format_attr_radix_scope.attr, + &format_attr_thresh_cmp.attr, NULL, }; @@ -550,7 +552,7 @@ static struct power_pmu power10_pmu = { .get_mem_weight = isa207_get_mem_weight, .disable_pmc = isa207_disable_pmc, .flags = PPMU_HAS_SIER | PPMU_ARCH_207S | - PPMU_ARCH_31, + PPMU_ARCH_31 | PPMU_HAS_ATTR_CONFIG1, .n_generic = ARRAY_SIZE(power10_generic_events), .generic_events = power10_generic_events, .cache_events = &power10_cache_events, diff --git a/arch/powerpc/perf/power5+-pmu.c b/arch/powerpc/perf/power5+-pmu.c index 3e64b4a1511f..18732267993a 100644 --- a/arch/powerpc/perf/power5+-pmu.c +++ b/arch/powerpc/perf/power5+-pmu.c @@ -132,7 +132,7 @@ static unsigned long unit_cons[PM_LASTUNIT+1][2] = { }; static int power5p_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, byte, unit, sh; int bit, fmask; @@ -451,7 +451,8 @@ static int power5p_marked_instr_event(u64 event) static int power5p_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], + u32 flags __maybe_unused) { unsigned long mmcr1 = 0; unsigned long mmcra = 0; diff --git a/arch/powerpc/perf/power5-pmu.c b/arch/powerpc/perf/power5-pmu.c index 017bb19b73fb..cb611c1e7abe 100644 --- a/arch/powerpc/perf/power5-pmu.c +++ b/arch/powerpc/perf/power5-pmu.c @@ -136,7 +136,7 @@ static unsigned long unit_cons[PM_LASTUNIT+1][2] = { }; static int power5_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, byte, unit, sh; int bit, fmask; @@ -382,7 +382,8 @@ static int power5_marked_instr_event(u64 event) static int power5_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], + u32 flags __maybe_unused) { unsigned long mmcr1 = 0; unsigned long mmcra = MMCRA_SDAR_DCACHE_MISS | MMCRA_SDAR_ERAT_MISS; diff --git a/arch/powerpc/perf/power6-pmu.c b/arch/powerpc/perf/power6-pmu.c index 189974478e9f..69ef38216418 100644 --- a/arch/powerpc/perf/power6-pmu.c +++ b/arch/powerpc/perf/power6-pmu.c @@ -173,7 +173,8 @@ static int power6_marked_instr_event(u64 event) * Assign PMC numbers and compute MMCR1 value for a set of events */ static int p6_compute_mmcr(u64 event[], int n_ev, - unsigned int hwc[], struct mmcr_regs *mmcr, struct perf_event *pevents[]) + unsigned int hwc[], struct mmcr_regs *mmcr, struct perf_event *pevents[], + u32 flags __maybe_unused) { unsigned long mmcr1 = 0; unsigned long mmcra = MMCRA_SDAR_DCACHE_MISS | MMCRA_SDAR_ERAT_MISS; @@ -266,7 +267,7 @@ static int p6_compute_mmcr(u64 event[], int n_ev, * 32-34 select field: nest (subunit) event selector */ static int p6_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, byte, sh, subunit; unsigned long mask = 0, value = 0; diff --git a/arch/powerpc/perf/power7-pmu.c b/arch/powerpc/perf/power7-pmu.c index bacfab104a1a..894c17f9a762 100644 --- a/arch/powerpc/perf/power7-pmu.c +++ b/arch/powerpc/perf/power7-pmu.c @@ -81,7 +81,7 @@ enum { */ static int power7_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, sh, unit; unsigned long mask = 0, value = 0; @@ -245,7 +245,8 @@ static int power7_marked_instr_event(u64 event) static int power7_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], + u32 flags __maybe_unused) { unsigned long mmcr1 = 0; unsigned long mmcra = MMCRA_SDAR_DCACHE_MISS | MMCRA_SDAR_ERAT_MISS; diff --git a/arch/powerpc/perf/ppc970-pmu.c b/arch/powerpc/perf/ppc970-pmu.c index 7d78df97f272..1f8263785286 100644 --- a/arch/powerpc/perf/ppc970-pmu.c +++ b/arch/powerpc/perf/ppc970-pmu.c @@ -190,7 +190,7 @@ static unsigned long unit_cons[PM_LASTUNIT+1][2] = { }; static int p970_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, byte, unit, sh, spcsel; unsigned long mask = 0, value = 0; @@ -256,7 +256,8 @@ static int p970_get_alternatives(u64 event, unsigned int flags, u64 alt[]) static int p970_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], + u32 flags __maybe_unused) { unsigned long mmcr0 = 0, mmcr1 = 0, mmcra = 0; unsigned int pmc, unit, byte, psel; diff --git a/arch/powerpc/platforms/44x/Kconfig b/arch/powerpc/platforms/44x/Kconfig index 78ac6d67a935..7d41e9264510 100644 --- a/arch/powerpc/platforms/44x/Kconfig +++ b/arch/powerpc/platforms/44x/Kconfig @@ -206,17 +206,10 @@ config AKEBONO select PPC4xx_HSTA_MSI select I2C select I2C_IBM_IIC - select NETDEVICES - select ETHERNET - select NET_VENDOR_IBM select IBM_EMAC_EMAC4 if IBM_EMAC select USB if USB_SUPPORT select USB_OHCI_HCD_PLATFORM if USB_OHCI_HCD select USB_EHCI_HCD_PLATFORM if USB_EHCI_HCD - select MMC_SDHCI - select MMC_SDHCI_PLTFM - select ATA - select SATA_AHCI_PLATFORM help This option enables support for the IBM Akebono (476gtr) evaluation board diff --git a/arch/powerpc/platforms/512x/mpc5121_ads.c b/arch/powerpc/platforms/512x/mpc5121_ads.c index 6303fbfc4e4f..9d030c2e0004 100644 --- a/arch/powerpc/platforms/512x/mpc5121_ads.c +++ b/arch/powerpc/platforms/512x/mpc5121_ads.c @@ -24,21 +24,23 @@ static void __init mpc5121_ads_setup_arch(void) { -#ifdef CONFIG_PCI - struct device_node *np; -#endif printk(KERN_INFO "MPC5121 ADS board from Freescale Semiconductor\n"); /* * cpld regs are needed early */ mpc5121_ads_cpld_map(); + mpc512x_setup_arch(); +} + +static void __init mpc5121_ads_setup_pci(void) +{ #ifdef CONFIG_PCI + struct device_node *np; + for_each_compatible_node(np, "pci", "fsl,mpc5121-pci") mpc83xx_add_bridge(np); #endif - - mpc512x_setup_arch(); } static void __init mpc5121_ads_init_IRQ(void) @@ -64,6 +66,7 @@ define_machine(mpc5121_ads) { .name = "MPC5121 ADS", .probe = mpc5121_ads_probe, .setup_arch = mpc5121_ads_setup_arch, + .discover_phbs = mpc5121_ads_setup_pci, .init = mpc512x_init, .init_IRQ = mpc5121_ads_init_IRQ, .get_irq = ipic_get_irq, diff --git a/arch/powerpc/platforms/52xx/efika.c b/arch/powerpc/platforms/52xx/efika.c index 4514a6f7458a..3b7d70d71692 100644 --- a/arch/powerpc/platforms/52xx/efika.c +++ b/arch/powerpc/platforms/52xx/efika.c @@ -185,8 +185,6 @@ static void __init efika_setup_arch(void) /* Map important registers from the internal memory map */ mpc52xx_map_common_devices(); - efika_pcisetup(); - #ifdef CONFIG_PM mpc52xx_suspend.board_suspend_prepare = efika_suspend_prepare; mpc52xx_pm_init(); @@ -218,6 +216,7 @@ define_machine(efika) .name = EFIKA_PLATFORM_NAME, .probe = efika_probe, .setup_arch = efika_setup_arch, + .discover_phbs = efika_pcisetup, .init = mpc52xx_declare_of_platform_devices, .show_cpuinfo = efika_show_cpuinfo, .init_IRQ = mpc52xx_init_irq, diff --git a/arch/powerpc/platforms/52xx/lite5200.c b/arch/powerpc/platforms/52xx/lite5200.c index 3181aac08225..04cc97397095 100644 --- a/arch/powerpc/platforms/52xx/lite5200.c +++ b/arch/powerpc/platforms/52xx/lite5200.c @@ -165,8 +165,6 @@ static void __init lite5200_setup_arch(void) mpc52xx_suspend.board_resume_finish = lite5200_resume_finish; lite5200_pm_init(); #endif - - mpc52xx_setup_pci(); } static const char * const board[] __initconst = { @@ -187,6 +185,7 @@ define_machine(lite5200) { .name = "lite5200", .probe = lite5200_probe, .setup_arch = lite5200_setup_arch, + .discover_phbs = mpc52xx_setup_pci, .init = mpc52xx_declare_of_platform_devices, .init_IRQ = mpc52xx_init_irq, .get_irq = mpc52xx_get_irq, diff --git a/arch/powerpc/platforms/52xx/media5200.c b/arch/powerpc/platforms/52xx/media5200.c index 07c5bc4ed0b5..efb8bdecbcc7 100644 --- a/arch/powerpc/platforms/52xx/media5200.c +++ b/arch/powerpc/platforms/52xx/media5200.c @@ -202,8 +202,6 @@ static void __init media5200_setup_arch(void) /* Some mpc5200 & mpc5200b related configuration */ mpc5200_setup_xlb_arbiter(); - mpc52xx_setup_pci(); - np = of_find_matching_node(NULL, mpc5200_gpio_ids); gpio = of_iomap(np, 0); of_node_put(np); @@ -244,6 +242,7 @@ define_machine(media5200_platform) { .name = "media5200-platform", .probe = media5200_probe, .setup_arch = media5200_setup_arch, + .discover_phbs = mpc52xx_setup_pci, .init = mpc52xx_declare_of_platform_devices, .init_IRQ = media5200_init_irq, .get_irq = mpc52xx_get_irq, diff --git a/arch/powerpc/platforms/52xx/mpc5200_simple.c b/arch/powerpc/platforms/52xx/mpc5200_simple.c index 2d01e9b2e779..b9f5675b0a1d 100644 --- a/arch/powerpc/platforms/52xx/mpc5200_simple.c +++ b/arch/powerpc/platforms/52xx/mpc5200_simple.c @@ -40,8 +40,6 @@ static void __init mpc5200_simple_setup_arch(void) /* Some mpc5200 & mpc5200b related configuration */ mpc5200_setup_xlb_arbiter(); - - mpc52xx_setup_pci(); } /* list of the supported boards */ @@ -73,6 +71,7 @@ define_machine(mpc5200_simple_platform) { .name = "mpc5200-simple-platform", .probe = mpc5200_simple_probe, .setup_arch = mpc5200_simple_setup_arch, + .discover_phbs = mpc52xx_setup_pci, .init = mpc52xx_declare_of_platform_devices, .init_IRQ = mpc52xx_init_irq, .get_irq = mpc52xx_get_irq, diff --git a/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c b/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c index 05e19470d523..b91ebebd9ff2 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c @@ -229,7 +229,7 @@ static irqreturn_t mpc52xx_lpbfifo_irq(int irq, void *dev_id) int dma, write, poll_dma; spin_lock_irqsave(&lpbfifo.lock, flags); - ts = get_tbl(); + ts = mftb(); req = lpbfifo.req; if (!req) { @@ -307,7 +307,7 @@ static irqreturn_t mpc52xx_lpbfifo_irq(int irq, void *dev_id) if (irq != 0) /* don't increment on polled case */ req->irq_count++; - req->irq_ticks += get_tbl() - ts; + req->irq_ticks += mftb() - ts; spin_unlock_irqrestore(&lpbfifo.lock, flags); /* Spinlock is released; it is now safe to call the callback */ @@ -330,7 +330,7 @@ static irqreturn_t mpc52xx_lpbfifo_bcom_irq(int irq, void *dev_id) u32 ts; spin_lock_irqsave(&lpbfifo.lock, flags); - ts = get_tbl(); + ts = mftb(); req = lpbfifo.req; if (!req || (req->flags & MPC52XX_LPBFIFO_FLAG_NO_DMA)) { @@ -361,7 +361,7 @@ static irqreturn_t mpc52xx_lpbfifo_bcom_irq(int irq, void *dev_id) lpbfifo.req = NULL; /* Release the lock before calling out to the callback. */ - req->irq_ticks += get_tbl() - ts; + req->irq_ticks += mftb() - ts; spin_unlock_irqrestore(&lpbfifo.lock, flags); if (req->callback) diff --git a/arch/powerpc/platforms/82xx/mpc8272_ads.c b/arch/powerpc/platforms/82xx/mpc8272_ads.c index 3fe1a6593280..0b5b9dec16d5 100644 --- a/arch/powerpc/platforms/82xx/mpc8272_ads.c +++ b/arch/powerpc/platforms/82xx/mpc8272_ads.c @@ -171,7 +171,6 @@ static void __init mpc8272_ads_setup_arch(void) iounmap(bcsr); init_ioports(); - pq2_init_pci(); if (ppc_md.progress) ppc_md.progress("mpc8272_ads_setup_arch(), finish", 0); @@ -205,6 +204,7 @@ define_machine(mpc8272_ads) .name = "Freescale MPC8272 ADS", .probe = mpc8272_ads_probe, .setup_arch = mpc8272_ads_setup_arch, + .discover_phbs = pq2_init_pci, .init_IRQ = mpc8272_ads_pic_init, .get_irq = cpm2_get_irq, .calibrate_decr = generic_calibrate_decr, diff --git a/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c b/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c index 096cc0d59fd8..f82f75a6085c 100644 --- a/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c +++ b/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c @@ -123,20 +123,17 @@ int __init pq2ads_pci_init_irq(void) np = of_find_compatible_node(NULL, NULL, "fsl,pq2ads-pci-pic"); if (!np) { printk(KERN_ERR "No pci pic node in device tree.\n"); - of_node_put(np); goto out; } irq = irq_of_parse_and_map(np, 0); if (!irq) { printk(KERN_ERR "No interrupt in pci pic node.\n"); - of_node_put(np); - goto out; + goto out_put_node; } priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) { - of_node_put(np); ret = -ENOMEM; goto out_unmap_irq; } @@ -161,17 +158,17 @@ int __init pq2ads_pci_init_irq(void) priv->host = host; irq_set_handler_data(irq, priv); irq_set_chained_handler(irq, pq2ads_pci_irq_demux); - - of_node_put(np); - return 0; + ret = 0; + goto out_put_node; out_unmap_regs: iounmap(priv->regs); out_free_kmalloc: kfree(priv); - of_node_put(np); out_unmap_irq: irq_dispose_mapping(irq); +out_put_node: + of_node_put(np); out: return ret; } diff --git a/arch/powerpc/platforms/82xx/pq2fads.c b/arch/powerpc/platforms/82xx/pq2fads.c index a74082140718..ac9113d524af 100644 --- a/arch/powerpc/platforms/82xx/pq2fads.c +++ b/arch/powerpc/platforms/82xx/pq2fads.c @@ -150,8 +150,6 @@ static void __init pq2fads_setup_arch(void) /* Enable external IRQs */ clrbits32(&cpm2_immr->im_siu_conf.siu_82xx.sc_siumcr, 0x0c000000); - pq2_init_pci(); - if (ppc_md.progress) ppc_md.progress("pq2fads_setup_arch(), finish", 0); } @@ -184,6 +182,7 @@ define_machine(pq2fads) .name = "Freescale PQ2FADS", .probe = pq2fads_probe, .setup_arch = pq2fads_setup_arch, + .discover_phbs = pq2_init_pci, .init_IRQ = pq2fads_pic_init, .get_irq = cpm2_get_irq, .calibrate_decr = generic_calibrate_decr, diff --git a/arch/powerpc/platforms/83xx/asp834x.c b/arch/powerpc/platforms/83xx/asp834x.c index 28474876f41b..68061c2a57c1 100644 --- a/arch/powerpc/platforms/83xx/asp834x.c +++ b/arch/powerpc/platforms/83xx/asp834x.c @@ -44,6 +44,7 @@ define_machine(asp834x) { .name = "ASP8347E", .probe = asp834x_probe, .setup_arch = asp834x_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/km83xx.c b/arch/powerpc/platforms/83xx/km83xx.c index bcdc2c203ec9..108e1e4d2683 100644 --- a/arch/powerpc/platforms/83xx/km83xx.c +++ b/arch/powerpc/platforms/83xx/km83xx.c @@ -180,6 +180,7 @@ define_machine(mpc83xx_km) { .name = "mpc83xx-km-platform", .probe = mpc83xx_km_probe, .setup_arch = mpc83xx_km_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/misc.c b/arch/powerpc/platforms/83xx/misc.c index a952e91db3ee..3285dabcf923 100644 --- a/arch/powerpc/platforms/83xx/misc.c +++ b/arch/powerpc/platforms/83xx/misc.c @@ -132,8 +132,6 @@ void __init mpc83xx_setup_arch(void) setbat(-1, va, immrbase, immrsize, PAGE_KERNEL_NCG); update_bats(); } - - mpc83xx_setup_pci(); } int machine_check_83xx(struct pt_regs *regs) diff --git a/arch/powerpc/platforms/83xx/mpc830x_rdb.c b/arch/powerpc/platforms/83xx/mpc830x_rdb.c index 51426e88ec67..956d4389effa 100644 --- a/arch/powerpc/platforms/83xx/mpc830x_rdb.c +++ b/arch/powerpc/platforms/83xx/mpc830x_rdb.c @@ -48,6 +48,7 @@ define_machine(mpc830x_rdb) { .name = "MPC830x RDB", .probe = mpc830x_rdb_probe, .setup_arch = mpc830x_rdb_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc831x_rdb.c b/arch/powerpc/platforms/83xx/mpc831x_rdb.c index 5ccd57a48492..3b578f080e3b 100644 --- a/arch/powerpc/platforms/83xx/mpc831x_rdb.c +++ b/arch/powerpc/platforms/83xx/mpc831x_rdb.c @@ -48,6 +48,7 @@ define_machine(mpc831x_rdb) { .name = "MPC831x RDB", .probe = mpc831x_rdb_probe, .setup_arch = mpc831x_rdb_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc832x_mds.c b/arch/powerpc/platforms/83xx/mpc832x_mds.c index 6fa5402ebf20..850d566ef900 100644 --- a/arch/powerpc/platforms/83xx/mpc832x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc832x_mds.c @@ -101,6 +101,7 @@ define_machine(mpc832x_mds) { .name = "MPC832x MDS", .probe = mpc832x_sys_probe, .setup_arch = mpc832x_sys_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc832x_rdb.c b/arch/powerpc/platforms/83xx/mpc832x_rdb.c index 622c625d5ce4..b6133a237a70 100644 --- a/arch/powerpc/platforms/83xx/mpc832x_rdb.c +++ b/arch/powerpc/platforms/83xx/mpc832x_rdb.c @@ -219,6 +219,7 @@ define_machine(mpc832x_rdb) { .name = "MPC832x RDB", .probe = mpc832x_rdb_probe, .setup_arch = mpc832x_rdb_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc834x_itx.c b/arch/powerpc/platforms/83xx/mpc834x_itx.c index ebfd139bca20..9630f3aa4d9c 100644 --- a/arch/powerpc/platforms/83xx/mpc834x_itx.c +++ b/arch/powerpc/platforms/83xx/mpc834x_itx.c @@ -70,6 +70,7 @@ define_machine(mpc834x_itx) { .name = "MPC834x ITX", .probe = mpc834x_itx_probe, .setup_arch = mpc834x_itx_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc834x_mds.c b/arch/powerpc/platforms/83xx/mpc834x_mds.c index 356228e35279..6d91bdce0a18 100644 --- a/arch/powerpc/platforms/83xx/mpc834x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc834x_mds.c @@ -91,6 +91,7 @@ define_machine(mpc834x_mds) { .name = "MPC834x MDS", .probe = mpc834x_mds_probe, .setup_arch = mpc834x_mds_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc836x_mds.c b/arch/powerpc/platforms/83xx/mpc836x_mds.c index 90d9cbfae659..da4cf52cb55b 100644 --- a/arch/powerpc/platforms/83xx/mpc836x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc836x_mds.c @@ -201,6 +201,7 @@ define_machine(mpc836x_mds) { .name = "MPC836x MDS", .probe = mpc836x_mds_probe, .setup_arch = mpc836x_mds_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc836x_rdk.c b/arch/powerpc/platforms/83xx/mpc836x_rdk.c index b4aac2cde849..3427ad0d9d38 100644 --- a/arch/powerpc/platforms/83xx/mpc836x_rdk.c +++ b/arch/powerpc/platforms/83xx/mpc836x_rdk.c @@ -41,6 +41,7 @@ define_machine(mpc836x_rdk) { .name = "MPC836x RDK", .probe = mpc836x_rdk_probe, .setup_arch = mpc836x_rdk_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc837x_mds.c b/arch/powerpc/platforms/83xx/mpc837x_mds.c index 9d3721c965be..f28d166ea7db 100644 --- a/arch/powerpc/platforms/83xx/mpc837x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc837x_mds.c @@ -93,6 +93,7 @@ define_machine(mpc837x_mds) { .name = "MPC837x MDS", .probe = mpc837x_mds_probe, .setup_arch = mpc837x_mds_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc837x_rdb.c b/arch/powerpc/platforms/83xx/mpc837x_rdb.c index 7c45f7ac2607..7fb7684c256b 100644 --- a/arch/powerpc/platforms/83xx/mpc837x_rdb.c +++ b/arch/powerpc/platforms/83xx/mpc837x_rdb.c @@ -73,6 +73,7 @@ define_machine(mpc837x_rdb) { .name = "MPC837x RDB/WLAN", .probe = mpc837x_rdb_probe, .setup_arch = mpc837x_rdb_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc83xx.h b/arch/powerpc/platforms/83xx/mpc83xx.h index f37d04332fc7..a30d30588cf6 100644 --- a/arch/powerpc/platforms/83xx/mpc83xx.h +++ b/arch/powerpc/platforms/83xx/mpc83xx.h @@ -76,7 +76,7 @@ extern void mpc83xx_ipic_init_IRQ(void); #ifdef CONFIG_PCI extern void mpc83xx_setup_pci(void); #else -#define mpc83xx_setup_pci() do {} while (0) +#define mpc83xx_setup_pci NULL #endif extern int mpc83xx_declare_of_platform_devices(void); diff --git a/arch/powerpc/platforms/8xx/machine_check.c b/arch/powerpc/platforms/8xx/machine_check.c index 88dedf38eccd..656365975895 100644 --- a/arch/powerpc/platforms/8xx/machine_check.c +++ b/arch/powerpc/platforms/8xx/machine_check.c @@ -26,7 +26,7 @@ int machine_check_8xx(struct pt_regs *regs) * to deal with that than having a wart in the mcheck handler. * -- BenH */ - bad_page_fault(regs, regs->dar, SIGBUS); + bad_page_fault(regs, SIGBUS); return 1; #else return 0; diff --git a/arch/powerpc/platforms/amigaone/setup.c b/arch/powerpc/platforms/amigaone/setup.c index f5d0bf999759..9d252c554f7f 100644 --- a/arch/powerpc/platforms/amigaone/setup.c +++ b/arch/powerpc/platforms/amigaone/setup.c @@ -65,6 +65,12 @@ static int __init amigaone_add_bridge(struct device_node *dev) } void __init amigaone_setup_arch(void) +{ + if (ppc_md.progress) + ppc_md.progress("Linux/PPC "UTS_RELEASE"\n", 0); +} + +static void __init amigaone_discover_phbs(void) { struct device_node *np; int phb = -ENODEV; @@ -74,9 +80,6 @@ void __init amigaone_setup_arch(void) phb = amigaone_add_bridge(np); BUG_ON(phb != 0); - - if (ppc_md.progress) - ppc_md.progress("Linux/PPC "UTS_RELEASE"\n", 0); } void __init amigaone_init_IRQ(void) @@ -159,6 +162,7 @@ define_machine(amigaone) { .name = "AmigaOne", .probe = amigaone_probe, .setup_arch = amigaone_setup_arch, + .discover_phbs = amigaone_discover_phbs, .show_cpuinfo = amigaone_show_cpuinfo, .init_IRQ = amigaone_init_IRQ, .restart = amigaone_restart, diff --git a/arch/powerpc/platforms/cell/pervasive.c b/arch/powerpc/platforms/cell/pervasive.c index 9068edef71f7..5b9a7e9f144b 100644 --- a/arch/powerpc/platforms/cell/pervasive.c +++ b/arch/powerpc/platforms/cell/pervasive.c @@ -25,6 +25,7 @@ #include #include "pervasive.h" +#include "ras.h" static void cbe_power_save(void) { diff --git a/arch/powerpc/platforms/cell/pervasive.h b/arch/powerpc/platforms/cell/pervasive.h index c6fccad6caee..0da74ab10716 100644 --- a/arch/powerpc/platforms/cell/pervasive.h +++ b/arch/powerpc/platforms/cell/pervasive.h @@ -13,9 +13,6 @@ #define PERVASIVE_H extern void cbe_pervasive_init(void); -extern void cbe_system_error_exception(struct pt_regs *regs); -extern void cbe_maintenance_exception(struct pt_regs *regs); -extern void cbe_thermal_exception(struct pt_regs *regs); #ifdef CONFIG_PPC_IBM_CELL_RESETBUTTON extern int cbe_sysreset_hack(void); diff --git a/arch/powerpc/platforms/cell/ras.c b/arch/powerpc/platforms/cell/ras.c index 6ea480539419..4325c05bedd9 100644 --- a/arch/powerpc/platforms/cell/ras.c +++ b/arch/powerpc/platforms/cell/ras.c @@ -49,7 +49,7 @@ static void dump_fir(int cpu) } -void cbe_system_error_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(cbe_system_error_exception) { int cpu = smp_processor_id(); @@ -58,7 +58,7 @@ void cbe_system_error_exception(struct pt_regs *regs) dump_stack(); } -void cbe_maintenance_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(cbe_maintenance_exception) { int cpu = smp_processor_id(); @@ -70,7 +70,7 @@ void cbe_maintenance_exception(struct pt_regs *regs) dump_stack(); } -void cbe_thermal_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(cbe_thermal_exception) { int cpu = smp_processor_id(); diff --git a/arch/powerpc/platforms/cell/ras.h b/arch/powerpc/platforms/cell/ras.h index 6c2e6bc0062e..226dbd48efad 100644 --- a/arch/powerpc/platforms/cell/ras.h +++ b/arch/powerpc/platforms/cell/ras.h @@ -2,9 +2,12 @@ #ifndef RAS_H #define RAS_H -extern void cbe_system_error_exception(struct pt_regs *regs); -extern void cbe_maintenance_exception(struct pt_regs *regs); -extern void cbe_thermal_exception(struct pt_regs *regs); +#include + +DECLARE_INTERRUPT_HANDLER(cbe_system_error_exception); +DECLARE_INTERRUPT_HANDLER(cbe_maintenance_exception); +DECLARE_INTERRUPT_HANDLER(cbe_thermal_exception); + extern void cbe_ras_init(void); #endif /* RAS_H */ diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c index 25390569e24c..b83a3670bd74 100644 --- a/arch/powerpc/platforms/cell/spufs/inode.c +++ b/arch/powerpc/platforms/cell/spufs/inode.c @@ -91,14 +91,15 @@ out: } static int -spufs_setattr(struct dentry *dentry, struct iattr *attr) +spufs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); if ((attr->ia_valid & ATTR_SIZE) && (attr->ia_size != inode->i_size)) return -EINVAL; - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/arch/powerpc/platforms/chrp/pci.c b/arch/powerpc/platforms/chrp/pci.c index b2c2bf35b76c..8c421dc78b28 100644 --- a/arch/powerpc/platforms/chrp/pci.c +++ b/arch/powerpc/platforms/chrp/pci.c @@ -314,6 +314,14 @@ chrp_find_bridges(void) } } of_node_put(root); + + /* + * "Temporary" fixes for PCI devices. + * -- Geert + */ + hydra_init(); /* Mac I/O */ + + pci_create_OF_bus_map(); } /* SL82C105 IDE Control/Status Register */ diff --git a/arch/powerpc/platforms/chrp/setup.c b/arch/powerpc/platforms/chrp/setup.c index c45435aa5e36..3cfc382841e5 100644 --- a/arch/powerpc/platforms/chrp/setup.c +++ b/arch/powerpc/platforms/chrp/setup.c @@ -334,22 +334,11 @@ static void __init chrp_setup_arch(void) /* On pegasos, enable the L2 cache if not already done by OF */ pegasos_set_l2cr(); - /* Lookup PCI host bridges */ - chrp_find_bridges(); - - /* - * Temporary fixes for PCI devices. - * -- Geert - */ - hydra_init(); /* Mac I/O */ - /* * Fix the Super I/O configuration */ sio_init(); - pci_create_OF_bus_map(); - /* * Print the banner, then scroll down so boot progress * can be printed. -- Cort @@ -582,6 +571,7 @@ define_machine(chrp) { .name = "CHRP", .probe = chrp_probe, .setup_arch = chrp_setup_arch, + .discover_phbs = chrp_find_bridges, .init = chrp_init2, .show_cpuinfo = chrp_show_cpuinfo, .init_IRQ = chrp_init_IRQ, diff --git a/arch/powerpc/platforms/embedded6xx/holly.c b/arch/powerpc/platforms/embedded6xx/holly.c index d8f2e2c737bb..53065d564161 100644 --- a/arch/powerpc/platforms/embedded6xx/holly.c +++ b/arch/powerpc/platforms/embedded6xx/holly.c @@ -108,15 +108,13 @@ static void holly_remap_bridge(void) tsi108_write_reg(TSI108_PCI_P2O_BAR2, 0x0); } -static void __init holly_setup_arch(void) +static void __init holly_init_pci(void) { struct device_node *np; if (ppc_md.progress) ppc_md.progress("holly_setup_arch():set_bridge", 0); - tsi108_csr_vir_base = get_vir_csrbase(); - /* setup PCI host bridge */ holly_remap_bridge(); @@ -127,6 +125,11 @@ static void __init holly_setup_arch(void) ppc_md.pci_exclude_device = holly_exclude_device; if (ppc_md.progress) ppc_md.progress("tsi108: resources set", 0x100); +} + +static void __init holly_setup_arch(void) +{ + tsi108_csr_vir_base = get_vir_csrbase(); printk(KERN_INFO "PPC750GX/CL Platform\n"); } @@ -259,6 +262,7 @@ define_machine(holly){ .name = "PPC750 GX/CL TSI", .probe = holly_probe, .setup_arch = holly_setup_arch, + .discover_phbs = holly_init_pci, .init_IRQ = holly_init_IRQ, .show_cpuinfo = holly_show_cpuinfo, .get_irq = mpic_get_irq, diff --git a/arch/powerpc/platforms/embedded6xx/linkstation.c b/arch/powerpc/platforms/embedded6xx/linkstation.c index f514d5d28cd4..eb8342e7f84e 100644 --- a/arch/powerpc/platforms/embedded6xx/linkstation.c +++ b/arch/powerpc/platforms/embedded6xx/linkstation.c @@ -63,15 +63,18 @@ static int __init linkstation_add_bridge(struct device_node *dev) } static void __init linkstation_setup_arch(void) +{ + printk(KERN_INFO "BUFFALO Network Attached Storage Series\n"); + printk(KERN_INFO "(C) 2002-2005 BUFFALO INC.\n"); +} + +static void __init linkstation_setup_pci(void) { struct device_node *np; /* Lookup PCI host bridges */ for_each_compatible_node(np, "pci", "mpc10x-pci") linkstation_add_bridge(np); - - printk(KERN_INFO "BUFFALO Network Attached Storage Series\n"); - printk(KERN_INFO "(C) 2002-2005 BUFFALO INC.\n"); } /* @@ -153,6 +156,7 @@ define_machine(linkstation){ .name = "Buffalo Linkstation", .probe = linkstation_probe, .setup_arch = linkstation_setup_arch, + .discover_phbs = linkstation_setup_pci, .init_IRQ = linkstation_init_IRQ, .show_cpuinfo = linkstation_show_cpuinfo, .get_irq = mpic_get_irq, diff --git a/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c b/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c index b95c3380d2b5..5565647dc879 100644 --- a/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c +++ b/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c @@ -58,16 +58,14 @@ int mpc7448_hpc2_exclude_device(struct pci_controller *hose, return PCIBIOS_SUCCESSFUL; } -static void __init mpc7448_hpc2_setup_arch(void) +static void __init mpc7448_hpc2_setup_pci(void) { +#ifdef CONFIG_PCI struct device_node *np; if (ppc_md.progress) - ppc_md.progress("mpc7448_hpc2_setup_arch():set_bridge", 0); - - tsi108_csr_vir_base = get_vir_csrbase(); + ppc_md.progress("mpc7448_hpc2_setup_pci():set_bridge", 0); /* setup PCI host bridge */ -#ifdef CONFIG_PCI for_each_compatible_node(np, "pci", "tsi108-pci") tsi108_setup_pci(np, MPC7448HPC2_PCI_CFG_PHYS, 0); @@ -75,6 +73,11 @@ static void __init mpc7448_hpc2_setup_arch(void) if (ppc_md.progress) ppc_md.progress("tsi108: resources set", 0x100); #endif +} + +static void __init mpc7448_hpc2_setup_arch(void) +{ + tsi108_csr_vir_base = get_vir_csrbase(); printk(KERN_INFO "MPC7448HPC2 (TAIGA) Platform\n"); printk(KERN_INFO @@ -181,6 +184,7 @@ define_machine(mpc7448_hpc2){ .name = "MPC7448 HPC2", .probe = mpc7448_hpc2_probe, .setup_arch = mpc7448_hpc2_setup_arch, + .discover_phbs = mpc7448_hpc2_setup_pci, .init_IRQ = mpc7448_hpc2_init_IRQ, .show_cpuinfo = mpc7448_hpc2_show_cpuinfo, .get_irq = mpic_get_irq, diff --git a/arch/powerpc/platforms/embedded6xx/mvme5100.c b/arch/powerpc/platforms/embedded6xx/mvme5100.c index 1cd488daa0bf..c06a0490d157 100644 --- a/arch/powerpc/platforms/embedded6xx/mvme5100.c +++ b/arch/powerpc/platforms/embedded6xx/mvme5100.c @@ -154,17 +154,19 @@ static const struct of_device_id mvme5100_of_bus_ids[] __initconst = { */ static void __init mvme5100_setup_arch(void) { - struct device_node *np; - if (ppc_md.progress) ppc_md.progress("mvme5100_setup_arch()", 0); - for_each_compatible_node(np, "pci", "hawk-pci") - mvme5100_add_bridge(np); - restart = ioremap(BOARD_MODRST_REG, 4); } +static void __init mvme5100_setup_pci(void) +{ + struct device_node *np; + + for_each_compatible_node(np, "pci", "hawk-pci") + mvme5100_add_bridge(np); +} static void mvme5100_show_cpuinfo(struct seq_file *m) { @@ -205,6 +207,7 @@ define_machine(mvme5100) { .name = "MVME5100", .probe = mvme5100_probe, .setup_arch = mvme5100_setup_arch, + .discover_phbs = mvme5100_setup_pci, .init_IRQ = mvme5100_pic_init, .show_cpuinfo = mvme5100_show_cpuinfo, .get_irq = mpic_get_irq, diff --git a/arch/powerpc/platforms/embedded6xx/storcenter.c b/arch/powerpc/platforms/embedded6xx/storcenter.c index e346ddcef45e..e188b90f7016 100644 --- a/arch/powerpc/platforms/embedded6xx/storcenter.c +++ b/arch/powerpc/platforms/embedded6xx/storcenter.c @@ -65,14 +65,17 @@ static int __init storcenter_add_bridge(struct device_node *dev) } static void __init storcenter_setup_arch(void) +{ + printk(KERN_INFO "IOMEGA StorCenter\n"); +} + +static void __init storcenter_setup_pci(void) { struct device_node *np; /* Lookup PCI host bridges */ for_each_compatible_node(np, "pci", "mpc10x-pci") storcenter_add_bridge(np); - - printk(KERN_INFO "IOMEGA StorCenter\n"); } /* @@ -117,6 +120,7 @@ define_machine(storcenter){ .name = "IOMEGA StorCenter", .probe = storcenter_probe, .setup_arch = storcenter_setup_arch, + .discover_phbs = storcenter_setup_pci, .init_IRQ = storcenter_init_IRQ, .get_irq = mpic_get_irq, .restart = storcenter_restart, diff --git a/arch/powerpc/platforms/maple/pci.c b/arch/powerpc/platforms/maple/pci.c index c86a66d5e998..a20b9576de22 100644 --- a/arch/powerpc/platforms/maple/pci.c +++ b/arch/powerpc/platforms/maple/pci.c @@ -536,6 +536,9 @@ static int __init maple_add_bridge(struct device_node *dev) /* Check for legacy IOs */ isa_bridge_find_early(hose); + /* create pci_dn's for DT nodes under this PHB */ + pci_devs_phb_init_dynamic(hose); + return 0; } diff --git a/arch/powerpc/platforms/maple/setup.c b/arch/powerpc/platforms/maple/setup.c index f7e66a2005b4..4e9ad5bf3efb 100644 --- a/arch/powerpc/platforms/maple/setup.c +++ b/arch/powerpc/platforms/maple/setup.c @@ -179,9 +179,6 @@ static void __init maple_setup_arch(void) #ifdef CONFIG_SMP smp_ops = &maple_smp_ops; #endif - /* Lookup PCI hosts */ - maple_pci_init(); - maple_use_rtas_reboot_and_halt_if_present(); printk(KERN_DEBUG "Using native/NAP idle loop\n"); @@ -351,6 +348,7 @@ define_machine(maple) { .name = "Maple", .probe = maple_probe, .setup_arch = maple_setup_arch, + .discover_phbs = maple_pci_init, .init_IRQ = maple_init_IRQ, .pci_irq_fixup = maple_pci_irq_fixup, .pci_get_legacy_ide_irq = maple_pci_get_legacy_ide_irq, diff --git a/arch/powerpc/platforms/pasemi/setup.c b/arch/powerpc/platforms/pasemi/setup.c index b612474f8f8e..376797eb7894 100644 --- a/arch/powerpc/platforms/pasemi/setup.c +++ b/arch/powerpc/platforms/pasemi/setup.c @@ -144,8 +144,6 @@ static void __init pas_setup_arch(void) /* Setup SMP callback */ smp_ops = &pas_smp_ops; #endif - /* Lookup PCI hosts */ - pas_pci_init(); /* Remap SDC register for doing reset */ /* XXXOJN This should maybe come out of the device tree */ @@ -446,6 +444,7 @@ define_machine(pasemi) { .name = "PA Semi PWRficient", .probe = pas_probe, .setup_arch = pas_setup_arch, + .discover_phbs = pas_pci_init, .init_IRQ = pas_init_IRQ, .get_irq = mpic_get_irq, .restart = pas_restart, diff --git a/arch/powerpc/platforms/powermac/pci.c b/arch/powerpc/platforms/powermac/pci.c index e35eaa9cf938..e9abe0f2e7f0 100644 --- a/arch/powerpc/platforms/powermac/pci.c +++ b/arch/powerpc/platforms/powermac/pci.c @@ -850,6 +850,10 @@ static int __init pmac_add_bridge(struct device_node *dev) /* Fixup "bus-range" OF property */ fixup_bus_range(dev); + /* create pci_dn's for DT nodes under this PHB */ + if (IS_ENABLED(CONFIG_PPC64)) + pci_devs_phb_init_dynamic(hose); + return 0; } diff --git a/arch/powerpc/platforms/powermac/setup.c b/arch/powerpc/platforms/powermac/setup.c index 2e2cc0c75d87..86aee3f2483f 100644 --- a/arch/powerpc/platforms/powermac/setup.c +++ b/arch/powerpc/platforms/powermac/setup.c @@ -298,9 +298,6 @@ static void __init pmac_setup_arch(void) of_node_put(ic); } - /* Lookup PCI hosts */ - pmac_pci_init(); - #ifdef CONFIG_PPC32 ohare_init(); l2cr_init(); @@ -600,6 +597,7 @@ define_machine(powermac) { .name = "PowerMac", .probe = pmac_probe, .setup_arch = pmac_setup_arch, + .discover_phbs = pmac_pci_init, .show_cpuinfo = pmac_show_cpuinfo, .init_IRQ = pmac_pic_init, .get_irq = NULL, /* changed later */ diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c index e6f461812856..999997d9e9a9 100644 --- a/arch/powerpc/platforms/powernv/idle.c +++ b/arch/powerpc/platforms/powernv/idle.c @@ -14,6 +14,7 @@ #include #include +#include #include #include #include diff --git a/arch/powerpc/platforms/powernv/memtrace.c b/arch/powerpc/platforms/powernv/memtrace.c index 5fc9408bb0b3..019669eb21d2 100644 --- a/arch/powerpc/platforms/powernv/memtrace.c +++ b/arch/powerpc/platforms/powernv/memtrace.c @@ -19,6 +19,7 @@ #include #include #include +#include /* This enables us to keep track of the memory removed from each node. */ struct memtrace_entry { @@ -51,6 +52,27 @@ static const struct file_operations memtrace_fops = { .open = simple_open, }; +#define FLUSH_CHUNK_SIZE SZ_1G +/** + * flush_dcache_range_chunked(): Write any modified data cache blocks out to + * memory and invalidate them, in chunks of up to FLUSH_CHUNK_SIZE + * Does not invalidate the corresponding instruction cache blocks. + * + * @start: the start address + * @stop: the stop address (exclusive) + * @chunk: the max size of the chunks + */ +static void flush_dcache_range_chunked(unsigned long start, unsigned long stop, + unsigned long chunk) +{ + unsigned long i; + + for (i = start; i < stop; i += chunk) { + flush_dcache_range(i, min(stop, i + chunk)); + cond_resched(); + } +} + static void memtrace_clear_range(unsigned long start_pfn, unsigned long nr_pages) { @@ -62,6 +84,13 @@ static void memtrace_clear_range(unsigned long start_pfn, cond_resched(); clear_page(__va(PFN_PHYS(pfn))); } + /* + * Before we go ahead and use this range as cache inhibited range + * flush the cache. + */ + flush_dcache_range_chunked(PFN_PHYS(start_pfn), + PFN_PHYS(start_pfn + nr_pages), + FLUSH_CHUNK_SIZE); } static u64 memtrace_alloc_node(u32 nid, u64 size) diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c index c61c3b62c8c6..303d7c775740 100644 --- a/arch/powerpc/platforms/powernv/opal.c +++ b/arch/powerpc/platforms/powernv/opal.c @@ -624,7 +624,7 @@ static int opal_recover_mce(struct pt_regs *regs, */ recovered = 0; } else { - die("Machine check", regs, SIGBUS); + die_mce("Machine check", regs, SIGBUS); recovered = 1; } } diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c index 8c739c94ed28..53172862d23b 100644 --- a/arch/powerpc/platforms/powernv/pci-cxl.c +++ b/arch/powerpc/platforms/powernv/pci-cxl.c @@ -150,25 +150,3 @@ int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq, return 0; } EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup); - -#if IS_MODULE(CONFIG_CXL) -static inline int get_cxl_module(void) -{ - struct module *cxl_module; - - mutex_lock(&module_mutex); - - cxl_module = find_module("cxl"); - if (cxl_module) - __module_get(cxl_module); - - mutex_unlock(&module_mutex); - - if (!cxl_module) - return -ENODEV; - - return 0; -} -#else -static inline int get_cxl_module(void) { return 0; } -#endif diff --git a/arch/powerpc/platforms/powernv/pci-ioda-tce.c b/arch/powerpc/platforms/powernv/pci-ioda-tce.c index 5218f5da2737..30551bbd7988 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda-tce.c +++ b/arch/powerpc/platforms/powernv/pci-ioda-tce.c @@ -380,6 +380,8 @@ void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, /* Remove link to a group from table's list of attached groups */ found = false; + + rcu_read_lock(); list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) { if (tgl->table_group == table_group) { list_del_rcu(&tgl->next); @@ -388,6 +390,8 @@ void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, break; } } + rcu_read_unlock(); + if (WARN_ON(!found)) return; diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index c4f72cdc9b51..f0f901683a2f 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -2402,9 +2402,6 @@ static void pnv_pci_ioda_create_dbgfs(void) list_for_each_entry_safe(hose, tmp, &hose_list, list_node) { phb = hose->private_data; - /* Notify initialization of PHB done */ - phb->initialized = 1; - sprintf(name, "PCI%04x", hose->global_number); phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root); @@ -2601,17 +2598,8 @@ static resource_size_t pnv_pci_default_alignment(void) */ static bool pnv_pci_enable_device_hook(struct pci_dev *dev) { - struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus); struct pci_dn *pdn; - /* The function is probably called while the PEs have - * not be created yet. For example, resource reassignment - * during PCI probe period. We just skip the check if - * PEs isn't ready. - */ - if (!phb->initialized) - return true; - pdn = pci_get_pdn(dev); if (!pdn || pdn->pe_number == IODA_INVALID_PE) { pci_err(dev, "pci_enable_device() blocked, no PE assigned.\n"); @@ -2623,14 +2611,9 @@ static bool pnv_pci_enable_device_hook(struct pci_dev *dev) static bool pnv_ocapi_enable_device_hook(struct pci_dev *dev) { - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; struct pci_dn *pdn; struct pnv_ioda_pe *pe; - if (!phb->initialized) - return true; - pdn = pci_get_pdn(dev); if (!pdn) return false; @@ -2938,7 +2921,7 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, phb_id = be64_to_cpup(prop64); pr_debug(" PHB-ID : 0x%016llx\n", phb_id); - phb = memblock_alloc(sizeof(*phb), SMP_CACHE_BYTES); + phb = kzalloc(sizeof(*phb), GFP_KERNEL); if (!phb) panic("%s: Failed to allocate %zu bytes\n", __func__, sizeof(*phb)); @@ -2987,7 +2970,7 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, else phb->diag_data_size = PNV_PCI_DIAG_BUF_SIZE; - phb->diag_data = memblock_alloc(phb->diag_data_size, SMP_CACHE_BYTES); + phb->diag_data = kzalloc(phb->diag_data_size, GFP_KERNEL); if (!phb->diag_data) panic("%s: Failed to allocate %u bytes\n", __func__, phb->diag_data_size); @@ -3049,9 +3032,10 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, } pemap_off = size; size += phb->ioda.total_pe_num * sizeof(struct pnv_ioda_pe); - aux = memblock_alloc(size, SMP_CACHE_BYTES); + aux = kzalloc(size, GFP_KERNEL); if (!aux) panic("%s: Failed to allocate %lu bytes\n", __func__, size); + phb->ioda.pe_alloc = aux; phb->ioda.m64_segmap = aux + m64map_off; phb->ioda.m32_segmap = aux + m32map_off; @@ -3178,6 +3162,9 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, /* Remove M64 resource if we can't configure it successfully */ if (!phb->init_m64 || phb->init_m64(phb)) hose->mem_resources[1].flags = 0; + + /* create pci_dn's for DT nodes under this PHB */ + pci_devs_phb_init_dynamic(hose); } void __init pnv_pci_init_ioda2_phb(struct device_node *np) diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h index 739a0b3b72e1..36d22920f5a3 100644 --- a/arch/powerpc/platforms/powernv/pci.h +++ b/arch/powerpc/platforms/powernv/pci.h @@ -119,7 +119,6 @@ struct pnv_phb { int flags; void __iomem *regs; u64 regs_phys; - int initialized; spinlock_t lock; #ifdef CONFIG_DEBUG_FS diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c index 4426a109ec2f..aadf932c4e61 100644 --- a/arch/powerpc/platforms/powernv/setup.c +++ b/arch/powerpc/platforms/powernv/setup.c @@ -180,9 +180,6 @@ static void __init pnv_setup_arch(void) /* Initialize SMP */ pnv_smp_init(); - /* Setup PCI */ - pnv_pci_init(); - /* Setup RTC and NVRAM callbacks */ if (firmware_has_feature(FW_FEATURE_OPAL)) opal_nvram_init(); @@ -547,6 +544,7 @@ define_machine(powernv) { .init_IRQ = pnv_init_IRQ, .show_cpuinfo = pnv_show_cpuinfo, .get_proc_freq = pnv_get_proc_freq, + .discover_phbs = pnv_pci_init, .progress = pnv_progress, .machine_shutdown = pnv_shutdown, .power_save = NULL, diff --git a/arch/powerpc/platforms/powernv/subcore.h b/arch/powerpc/platforms/powernv/subcore.h index c8f574d1c04a..77feee8436d4 100644 --- a/arch/powerpc/platforms/powernv/subcore.h +++ b/arch/powerpc/platforms/powernv/subcore.h @@ -15,7 +15,7 @@ void split_core_secondary_loop(u8 *state); extern void update_subcore_sibling_mask(void); #else -static inline void update_subcore_sibling_mask(void) { }; +static inline void update_subcore_sibling_mask(void) { } #endif /* CONFIG_SMP */ #endif /* __ASSEMBLY__ */ diff --git a/arch/powerpc/platforms/powernv/vas.c b/arch/powerpc/platforms/powernv/vas.c index 598e4cd563fb..b65256a63e87 100644 --- a/arch/powerpc/platforms/powernv/vas.c +++ b/arch/powerpc/platforms/powernv/vas.c @@ -28,12 +28,10 @@ static DEFINE_PER_CPU(int, cpu_vas_id); static int vas_irq_fault_window_setup(struct vas_instance *vinst) { - char devname[64]; int rc = 0; - snprintf(devname, sizeof(devname), "vas-%d", vinst->vas_id); rc = request_threaded_irq(vinst->virq, vas_fault_handler, - vas_fault_thread_fn, 0, devname, vinst); + vas_fault_thread_fn, 0, vinst->name, vinst); if (rc) { pr_err("VAS[%d]: Request IRQ(%d) failed with %d\n", @@ -80,6 +78,12 @@ static int init_vas_instance(struct platform_device *pdev) if (!vinst) return -ENOMEM; + vinst->name = kasprintf(GFP_KERNEL, "vas-%d", vasid); + if (!vinst->name) { + kfree(vinst); + return -ENOMEM; + } + INIT_LIST_HEAD(&vinst->node); ida_init(&vinst->ida); mutex_init(&vinst->mutex); @@ -162,6 +166,7 @@ static int init_vas_instance(struct platform_device *pdev) return 0; free_vinst: + kfree(vinst->name); kfree(vinst); return -ENODEV; diff --git a/arch/powerpc/platforms/powernv/vas.h b/arch/powerpc/platforms/powernv/vas.h index 70f793e8f6cc..c7db3190baca 100644 --- a/arch/powerpc/platforms/powernv/vas.h +++ b/arch/powerpc/platforms/powernv/vas.h @@ -340,6 +340,7 @@ struct vas_instance { struct vas_window *rxwin[VAS_COP_TYPE_MAX]; struct vas_window *windows[VAS_WINDOWS_PER_CHIP]; + char *name; char *dbgname; struct dentry *dbgdir; }; diff --git a/arch/powerpc/platforms/pseries/dlpar.c b/arch/powerpc/platforms/pseries/dlpar.c index 16e86ba8aa20..233503fcf8f0 100644 --- a/arch/powerpc/platforms/pseries/dlpar.c +++ b/arch/powerpc/platforms/pseries/dlpar.c @@ -127,7 +127,6 @@ void dlpar_free_cc_nodes(struct device_node *dn) #define NEXT_PROPERTY 3 #define PREV_PARENT 4 #define MORE_MEMORY 5 -#define CALL_AGAIN -2 #define ERR_CFG_USE -9003 struct device_node *dlpar_configure_connector(__be32 drc_index, @@ -168,6 +167,9 @@ struct device_node *dlpar_configure_connector(__be32 drc_index, spin_unlock(&rtas_data_buf_lock); + if (rtas_busy_delay(rc)) + continue; + switch (rc) { case COMPLETE: break; @@ -216,9 +218,6 @@ struct device_node *dlpar_configure_connector(__be32 drc_index, last_dn = last_dn->parent; break; - case CALL_AGAIN: - break; - case MORE_MEMORY: case ERR_CFG_USE: default: @@ -521,11 +520,8 @@ static ssize_t dlpar_store(struct class *class, struct class_attribute *attr, int rc; args = argbuf = kstrdup(buf, GFP_KERNEL); - if (!argbuf) { - pr_info("Could not allocate resources for DLPAR operation\n"); - kfree(argbuf); + if (!argbuf) return -ENOMEM; - } /* * Parse out the request from the user, this will be in the form: diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c index cf024fa37bda..bc15200852b7 100644 --- a/arch/powerpc/platforms/pseries/eeh_pseries.c +++ b/arch/powerpc/platforms/pseries/eeh_pseries.c @@ -43,7 +43,7 @@ static int ibm_get_config_addr_info; static int ibm_get_config_addr_info2; static int ibm_configure_pe; -void pseries_pcibios_bus_add_device(struct pci_dev *pdev) +static void pseries_pcibios_bus_add_device(struct pci_dev *pdev) { struct pci_dn *pdn = pci_get_pdn(pdev); @@ -694,8 +694,7 @@ static int pseries_eeh_write_config(struct eeh_dev *edev, int where, int size, u } #ifdef CONFIG_PCI_IOV -int pseries_send_allow_unfreeze(struct pci_dn *pdn, - u16 *vf_pe_array, int cur_vfs) +static int pseries_send_allow_unfreeze(struct pci_dn *pdn, u16 *vf_pe_array, int cur_vfs) { int rc; int ibm_allow_unfreeze = rtas_token("ibm,open-sriov-allow-unfreeze"); diff --git a/arch/powerpc/platforms/pseries/ibmebus.c b/arch/powerpc/platforms/pseries/ibmebus.c index 8c6e509f6967..a15ab33646b3 100644 --- a/arch/powerpc/platforms/pseries/ibmebus.c +++ b/arch/powerpc/platforms/pseries/ibmebus.c @@ -355,12 +355,12 @@ static int ibmebus_bus_device_probe(struct device *dev) if (!drv->probe) return error; - of_dev_get(of_dev); + get_device(dev); if (of_driver_match_device(dev, dev->driver)) error = drv->probe(of_dev); if (error) - of_dev_put(of_dev); + put_device(dev); return error; } diff --git a/arch/powerpc/platforms/pseries/pci.c b/arch/powerpc/platforms/pseries/pci.c index 72a4d4167849..1bffbd1c9a94 100644 --- a/arch/powerpc/platforms/pseries/pci.c +++ b/arch/powerpc/platforms/pseries/pci.c @@ -55,9 +55,8 @@ struct pe_map_bar_entry { __be32 reserved; /* Reserved Space */ }; -int pseries_send_map_pe(struct pci_dev *pdev, - u16 num_vfs, - struct pe_map_bar_entry *vf_pe_array) +static int pseries_send_map_pe(struct pci_dev *pdev, u16 num_vfs, + struct pe_map_bar_entry *vf_pe_array) { struct pci_dn *pdn; int rc; @@ -88,7 +87,7 @@ int pseries_send_map_pe(struct pci_dev *pdev, return rc; } -void pseries_set_pe_num(struct pci_dev *pdev, u16 vf_index, __be16 pe_num) +static void pseries_set_pe_num(struct pci_dev *pdev, u16 vf_index, __be16 pe_num) { struct pci_dn *pdn; @@ -102,7 +101,7 @@ void pseries_set_pe_num(struct pci_dev *pdev, u16 vf_index, __be16 pe_num) pdn->pe_num_map[vf_index]); } -int pseries_associate_pes(struct pci_dev *pdev, u16 num_vfs) +static int pseries_associate_pes(struct pci_dev *pdev, u16 num_vfs) { struct pci_dn *pdn; int i, rc, vf_index; @@ -146,7 +145,7 @@ int pseries_associate_pes(struct pci_dev *pdev, u16 num_vfs) return rc; } -int pseries_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs) +static int pseries_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs) { struct pci_dn *pdn; int rc; @@ -189,14 +188,14 @@ int pseries_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs) return rc; } -int pseries_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) +static int pseries_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) { /* Allocate PCI data */ add_sriov_vf_pdns(pdev); return pseries_pci_sriov_enable(pdev, num_vfs); } -int pseries_pcibios_sriov_disable(struct pci_dev *pdev) +static int pseries_pcibios_sriov_disable(struct pci_dev *pdev) { struct pci_dn *pdn; diff --git a/arch/powerpc/platforms/pseries/pseries.h b/arch/powerpc/platforms/pseries/pseries.h index 593840847cd3..4fe48c04c6c2 100644 --- a/arch/powerpc/platforms/pseries/pseries.h +++ b/arch/powerpc/platforms/pseries/pseries.h @@ -33,7 +33,7 @@ int smp_query_cpu_stopped(unsigned int pcpu); #define QCSS_HARDWARE_ERROR -1 #define QCSS_HARDWARE_BUSY -2 #else -static inline void smp_init_pseries(void) { }; +static inline void smp_init_pseries(void) { } #endif extern void pseries_kexec_cpu_down(int crash_shutdown, int secondary); diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c index 149cec2212e6..f8b390a9d9fb 100644 --- a/arch/powerpc/platforms/pseries/ras.c +++ b/arch/powerpc/platforms/pseries/ras.c @@ -122,7 +122,7 @@ static inline u8 rtas_mc_error_sub_type(const struct pseries_mc_errorlog *mlog) * devices or systems (e.g. hugepages) that have not been initialized at the * subsys stage. */ -int __init init_ras_hotplug_IRQ(void) +static int __init init_ras_hotplug_IRQ(void) { struct device_node *np; @@ -315,12 +315,10 @@ static irqreturn_t ras_hotplug_interrupt(int irq, void *dev_id) /* Handle environmental and power warning (EPOW) interrupts. */ static irqreturn_t ras_epow_interrupt(int irq, void *dev_id) { - int status; int state; int critical; - status = rtas_get_sensor_fast(EPOW_SENSOR_TOKEN, EPOW_SENSOR_INDEX, - &state); + rtas_get_sensor_fast(EPOW_SENSOR_TOKEN, EPOW_SENSOR_INDEX, &state); if (state > 3) critical = 1; /* Time Critical */ @@ -329,12 +327,9 @@ static irqreturn_t ras_epow_interrupt(int irq, void *dev_id) spin_lock(&ras_log_buf_lock); - status = rtas_call(ras_check_exception_token, 6, 1, NULL, - RTAS_VECTOR_EXTERNAL_INTERRUPT, - virq_to_hw(irq), - RTAS_EPOW_WARNING, - critical, __pa(&ras_log_buf), - rtas_get_error_log_max()); + rtas_call(ras_check_exception_token, 6, 1, NULL, RTAS_VECTOR_EXTERNAL_INTERRUPT, + virq_to_hw(irq), RTAS_EPOW_WARNING, critical, __pa(&ras_log_buf), + rtas_get_error_log_max()); log_error(ras_log_buf, ERR_TYPE_RTAS_LOG, 0); @@ -722,6 +717,7 @@ static int mce_handle_error(struct pt_regs *regs, struct rtas_error_log *errp) struct pseries_errorlog *pseries_log; struct pseries_mc_errorlog *mce_log = NULL; int disposition = rtas_error_disposition(errp); + unsigned long msr; u8 error_type; if (!rtas_error_extended(errp)) @@ -747,9 +743,21 @@ static int mce_handle_error(struct pt_regs *regs, struct rtas_error_log *errp) * SLB multihit is done by now. */ out: - mtmsr(mfmsr() | MSR_IR | MSR_DR); + msr = mfmsr(); + mtmsr(msr | MSR_IR | MSR_DR); + disposition = mce_handle_err_virtmode(regs, errp, mce_log, disposition); + + /* + * Queue irq work to log this rtas event later. + * irq_work_queue uses per-cpu variables, so do this in virt + * mode as well. + */ + irq_work_queue(&mce_errlog_process_work); + + mtmsr(msr); + return disposition; } @@ -813,7 +821,7 @@ static int recover_mce(struct pt_regs *regs, struct machine_check_event *evt) */ recovered = 0; } else { - die("Machine check", regs, SIGBUS); + die_mce("Machine check", regs, SIGBUS); recovered = 1; } } @@ -865,10 +873,8 @@ long pseries_machine_check_realmode(struct pt_regs *regs) * virtual mode. */ disposition = mce_handle_error(regs, errp); - fwnmi_release_errinfo(); - /* Queue irq work to log this rtas event later. */ - irq_work_queue(&mce_errlog_process_work); + fwnmi_release_errinfo(); if (disposition == RTAS_DISP_FULLY_RECOVERED) return 1; diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index 090c13f6c881..46e1540abc22 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -463,7 +463,7 @@ void pseries_little_endian_exceptions(void) } #endif -static void __init find_and_init_phbs(void) +static void __init pSeries_discover_phbs(void) { struct device_node *node; struct pci_controller *phb; @@ -481,6 +481,9 @@ static void __init find_and_init_phbs(void) pci_process_bridge_OF_ranges(phb, node, 0); isa_bridge_find_early(phb); phb->controller_ops = pseries_pci_controller_ops; + + /* create pci_dn's for DT nodes under this PHB */ + pci_devs_phb_init_dynamic(phb); } of_node_put(root); @@ -607,8 +610,8 @@ enum get_iov_fw_value_index { WDW_SIZE = 3 /* Get Window Size */ }; -resource_size_t pseries_get_iov_fw_value(struct pci_dev *dev, int resno, - enum get_iov_fw_value_index value) +static resource_size_t pseries_get_iov_fw_value(struct pci_dev *dev, int resno, + enum get_iov_fw_value_index value) { const int *indexes; struct device_node *dn = pci_device_to_OF_node(dev); @@ -643,7 +646,7 @@ resource_size_t pseries_get_iov_fw_value(struct pci_dev *dev, int resno, return ret; } -void of_pci_set_vf_bar_size(struct pci_dev *dev, const int *indexes) +static void of_pci_set_vf_bar_size(struct pci_dev *dev, const int *indexes) { struct resource *res; resource_size_t base, size; @@ -665,7 +668,7 @@ void of_pci_set_vf_bar_size(struct pci_dev *dev, const int *indexes) } } -void of_pci_parse_iov_addrs(struct pci_dev *dev, const int *indexes) +static void of_pci_parse_iov_addrs(struct pci_dev *dev, const int *indexes) { struct resource *res, *root, *conflict; resource_size_t base, size; @@ -786,7 +789,6 @@ static void __init pSeries_setup_arch(void) /* Find and initialize PCI host bridges */ init_pci_config_tokens(); - find_and_init_phbs(); of_reconfig_notifier_register(&pci_dn_reconfig_nb); pSeries_nvram_init(); @@ -1050,6 +1052,7 @@ define_machine(pseries) { .init_IRQ = pseries_init_irq, .show_cpuinfo = pSeries_show_cpuinfo, .log_error = pSeries_log_error, + .discover_phbs = pSeries_discover_phbs, .pcibios_fixup = pSeries_final_fixup, .restart = rtas_restart, .halt = rtas_halt, diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index dcd817ca2edf..3fe37495f63d 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -1383,7 +1383,6 @@ static long check_bp_loc(unsigned long addr) return 1; } -#ifndef CONFIG_PPC_8xx static int find_free_data_bpt(void) { int i; @@ -1395,7 +1394,6 @@ static int find_free_data_bpt(void) printf("Couldn't find free breakpoint register\n"); return -1; } -#endif static void print_data_bpts(void) { @@ -1435,7 +1433,6 @@ bpt_cmds(void) cmd = inchar(); switch (cmd) { -#ifndef CONFIG_PPC_8xx static const char badaddr[] = "Only kernel addresses are permitted for breakpoints\n"; int mode; case 'd': /* bd - hardware data breakpoint */ @@ -1497,7 +1494,6 @@ bpt_cmds(void) force_enable_xmon(); } break; -#endif case 'c': if (!scanhex(&a)) { @@ -3723,7 +3719,7 @@ void dump_segments(void) printf("sr0-15 ="); for (i = 0; i < 16; ++i) - printf(" %x", mfsrin(i << 28)); + printf(" %x", mfsr(i << 28)); printf("\n"); } #endif diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index e0a34eb5ed3b..85d626b8ce5e 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -57,6 +57,7 @@ config RISCV select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN if MMU && 64BIT + select HAVE_ARCH_KASAN_VMALLOC if MMU && 64BIT select HAVE_ARCH_KGDB select HAVE_ARCH_KGDB_QXFER_PKT select HAVE_ARCH_MMAP_RND_BITS if MMU @@ -67,14 +68,19 @@ config RISCV select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU select HAVE_EBPF_JIT if MMU + select HAVE_FUNCTION_ERROR_INJECTION select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_GCC_PLUGINS select HAVE_GENERIC_VDSO if MMU && 64BIT select HAVE_IRQ_TIME_ACCOUNTING + select HAVE_KPROBES + select HAVE_KPROBES_ON_FTRACE + select HAVE_KRETPROBES select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN @@ -143,7 +149,7 @@ config PAGE_OFFSET default 0xffffffe000000000 if 64BIT && MAXPHYSMEM_128GB config ARCH_FLATMEM_ENABLE - def_bool y + def_bool !NUMA config ARCH_SPARSEMEM_ENABLE def_bool y @@ -156,6 +162,9 @@ config ARCH_SELECT_MEMORY_MODEL config ARCH_WANT_GENERAL_HUGETLB def_bool y +config ARCH_SUPPORTS_UPROBES + def_bool y + config SYS_SUPPORTS_HUGETLBFS depends on MMU def_bool y @@ -302,6 +311,36 @@ config TUNE_GENERIC endchoice +# Common NUMA Features +config NUMA + bool "NUMA Memory Allocation and Scheduler Support" + depends on SMP + select GENERIC_ARCH_NUMA + select OF_NUMA + select ARCH_SUPPORTS_NUMA_BALANCING + help + Enable NUMA (Non-Uniform Memory Access) support. + + The kernel will try to allocate memory used by a CPU on the + local memory of the CPU and add some more NUMA awareness to the kernel. + +config NODES_SHIFT + int "Maximum NUMA Nodes (as a power of 2)" + range 1 10 + default "2" + depends on NEED_MULTIPLE_NODES + help + Specify the maximum number of NUMA Nodes available on the target + system. Increases memory reserved to accommodate various tables. + +config USE_PERCPU_NUMA_NODE_ID + def_bool y + depends on NUMA + +config NEED_PER_CPU_EMBED_FIRST_CHUNK + def_bool y + depends on NUMA + config RISCV_ISA_C bool "Emit compressed instructions when building Linux" default y @@ -416,11 +455,17 @@ config EFI allow the kernel to be booted as an EFI application. This is only useful on systems that have UEFI firmware. +config CC_HAVE_STACKPROTECTOR_TLS + def_bool $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=tp -mstack-protector-guard-offset=0) + +config STACKPROTECTOR_PER_TASK + def_bool y + depends on STACKPROTECTOR && CC_HAVE_STACKPROTECTOR_TLS + endmenu config BUILTIN_DTB def_bool n - depends on RISCV_M_MODE depends on OF menu "Power management options" diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index 3284d5c291be..7efcece8896c 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -22,30 +22,41 @@ config SOC_VIRT help This enables support for QEMU Virt Machine. -config SOC_KENDRYTE - bool "Kendryte K210 SoC" +config SOC_CANAAN + bool "Canaan Kendryte K210 SoC" depends on !MMU select CLINT_TIMER if RISCV_M_MODE select SERIAL_SIFIVE if TTY select SERIAL_SIFIVE_CONSOLE if TTY select SIFIVE_PLIC + select ARCH_HAS_RESET_CONTROLLER + select PINCTRL help - This enables support for Kendryte K210 SoC platform hardware. + This enables support for Canaan Kendryte K210 SoC platform hardware. -config SOC_KENDRYTE_K210_DTB - def_bool y - depends on SOC_KENDRYTE_K210_DTB_BUILTIN +if SOC_CANAAN -config SOC_KENDRYTE_K210_DTB_BUILTIN - bool "Builtin device tree for the Kendryte K210" - depends on SOC_KENDRYTE +config SOC_CANAAN_K210_DTB_BUILTIN + bool "Builtin device tree for the Canaan Kendryte K210" + depends on SOC_CANAAN default y select OF select BUILTIN_DTB - select SOC_KENDRYTE_K210_DTB help - Builds a device tree for the Kendryte K210 into the Linux image. + Build a device tree for the Kendryte K210 into the Linux image. This option should be selected if no bootloader is being used. If unsure, say Y. +config SOC_CANAAN_K210_DTB_SOURCE + string "Source file for the Canaan Kendryte K210 builtin DTB" + depends on SOC_CANAAN + depends on SOC_CANAAN_K210_DTB_BUILTIN + default "k210_generic" + help + Base name (without suffix, relative to arch/riscv/boot/dts/canaan) + for the DTS file that will be used to produce the DTB linked into the + kernel. + +endif + endmenu diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 8c29e553ef7f..1368d943f1f3 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -12,6 +12,8 @@ OBJCOPYFLAGS := -O binary LDFLAGS_vmlinux := ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax + KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY + CC_FLAGS_FTRACE := -fpatchable-function-entry=8 endif ifeq ($(CONFIG_64BIT)$(CONFIG_CMODEL_MEDLOW),yy) @@ -65,6 +67,16 @@ KBUILD_CFLAGS_MODULE += $(call cc-option,-mno-relax) # architectures. It's faster to have GCC emit only aligned accesses. KBUILD_CFLAGS += $(call cc-option,-mstrict-align) +ifeq ($(CONFIG_STACKPROTECTOR_PER_TASK),y) +prepare: stack_protector_prepare +stack_protector_prepare: prepare0 + $(eval KBUILD_CFLAGS += -mstack-protector-guard=tls \ + -mstack-protector-guard-reg=tp \ + -mstack-protector-guard-offset=$(shell \ + awk '{if ($$2 == "TSK_STACK_CANARY") print $$3;}' \ + include/generated/asm-offsets.h)) +endif + # arch specific predefines for sparse CHECKFLAGS += -D__riscv -D__riscv_xlen=$(BITS) @@ -83,7 +95,7 @@ PHONY += vdso_install vdso_install: $(Q)$(MAKE) $(build)=arch/riscv/kernel/vdso $@ -ifeq ($(CONFIG_RISCV_M_MODE)$(CONFIG_SOC_KENDRYTE),yy) +ifeq ($(CONFIG_RISCV_M_MODE)$(CONFIG_SOC_CANAAN),yy) KBUILD_IMAGE := $(boot)/loader.bin else KBUILD_IMAGE := $(boot)/Image.gz diff --git a/arch/riscv/boot/dts/Makefile b/arch/riscv/boot/dts/Makefile index ca1f8cbd78c0..7ffd502e3e7b 100644 --- a/arch/riscv/boot/dts/Makefile +++ b/arch/riscv/boot/dts/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 subdir-y += sifive -subdir-y += kendryte +subdir-$(CONFIG_SOC_CANAAN_K210_DTB_BUILTIN) += canaan obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix /, $(subdir-y)) diff --git a/arch/riscv/boot/dts/canaan/Makefile b/arch/riscv/boot/dts/canaan/Makefile new file mode 100644 index 000000000000..9ee7156c0c31 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +ifneq ($(CONFIG_SOC_CANAAN_K210_DTB_SOURCE),"") +dtb-y += $(strip $(shell echo $(CONFIG_SOC_CANAAN_K210_DTB_SOURCE))).dtb +obj-$(CONFIG_SOC_CANAAN_K210_DTB_BUILTIN) += $(addsuffix .o, $(dtb-y)) +endif diff --git a/arch/riscv/boot/dts/canaan/canaan_kd233.dts b/arch/riscv/boot/dts/canaan/canaan_kd233.dts new file mode 100644 index 000000000000..039b92abf046 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/canaan_kd233.dts @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include + +/ { + model = "Kendryte KD233"; + compatible = "canaan,kendryte-kd233", "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; + + gpio-leds { + compatible = "gpio-leds"; + + led0 { + gpios = <&gpio0 8 GPIO_ACTIVE_LOW>; + }; + + led1 { + gpios = <&gpio0 9 GPIO_ACTIVE_LOW>; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + + key0 { + label = "KEY0"; + linux,code = ; + gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&fpioa { + pinctrl-0 = <&jtag_pinctrl>; + pinctrl-names = "default"; + status = "okay"; + + jtag_pinctrl: jtag-pinmux { + pinmux = , + , + , + ; + }; + + uarths_pinctrl: uarths-pinmux { + pinmux = , + ; + }; + + spi0_pinctrl: spi0-pinmux { + pinmux = , /* cs */ + , /* wr */ + ; /* dc */ + }; + + dvp_pinctrl: dvp-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + gpiohs_pinctrl: gpiohs-pinmux { + pinmux = , + , /* Rot. dip sw line 8 */ + , /* Rot. dip sw line 4 */ + , /* Rot. dip sw line 2 */ + , /* Rot. dip sw line 1 */ + , + , + ; + }; + + spi1_pinctrl: spi1-pinmux { + pinmux = , + , + , + ; /* cs */ + }; + + i2s0_pinctrl: i2s0-pinmux { + pinmux = , + , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio0 { + pinctrl-0 = <&gpiohs_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0 { + #sound-dai-cells = <1>; + pinctrl-0 = <&i2s0_pinctrl>; + pinctrl-names = "default"; +}; + +&spi0 { + pinctrl-0 = <&spi0_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + + panel@0 { + compatible = "ilitek,ili9341"; + reg = <0>; + dc-gpios = <&gpio0 21 GPIO_ACTIVE_HIGH>; + spi-max-frequency = <15000000>; + status = "disabled"; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 16 GPIO_ACTIVE_LOW>; + status = "okay"; + + slot@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <25000000>; + broken-cd; + }; +}; diff --git a/arch/riscv/boot/dts/canaan/k210.dtsi b/arch/riscv/boot/dts/canaan/k210.dtsi new file mode 100644 index 000000000000..5e8ca8142482 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/k210.dtsi @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ +#include +#include +#include + +/ { + /* + * Although the K210 is a 64-bit CPU, the address bus is only 32-bits + * wide, and the upper half of all addresses is ignored. + */ + #address-cells = <1>; + #size-cells = <1>; + compatible = "canaan,kendryte-k210"; + + aliases { + serial0 = &uarths0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + }; + + /* + * The K210 has an sv39 MMU following the privileged specification v1.9. + * Since this is a non-ratified draft specification, the kernel does not + * support it and the K210 support enabled only for the !MMU case. + * Be consistent with this by setting the CPUs MMU type to "none". + */ + cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <7800000>; + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "canaan,k210", "riscv"; + reg = <0>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,none"; + i-cache-block-size = <64>; + i-cache-size = <0x8000>; + d-cache-block-size = <64>; + d-cache-size = <0x8000>; + cpu0_intc: interrupt-controller { + #interrupt-cells = <1>; + interrupt-controller; + compatible = "riscv,cpu-intc"; + }; + }; + cpu1: cpu@1 { + device_type = "cpu"; + compatible = "canaan,k210", "riscv"; + reg = <1>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,none"; + i-cache-block-size = <64>; + i-cache-size = <0x8000>; + d-cache-block-size = <64>; + d-cache-size = <0x8000>; + cpu1_intc: interrupt-controller { + #interrupt-cells = <1>; + interrupt-controller; + compatible = "riscv,cpu-intc"; + }; + }; + }; + + sram: memory@80000000 { + device_type = "memory"; + compatible = "canaan,k210-sram"; + reg = <0x80000000 0x400000>, + <0x80400000 0x200000>, + <0x80600000 0x200000>; + reg-names = "sram0", "sram1", "aisram"; + clocks = <&sysclk K210_CLK_SRAM0>, + <&sysclk K210_CLK_SRAM1>, + <&sysclk K210_CLK_AI>; + clock-names = "sram0", "sram1", "aisram"; + }; + + clocks { + in0: oscillator { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <26000000>; + }; + }; + + soc { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-bus"; + ranges; + interrupt-parent = <&plic0>; + + rom0: nvmem@1000 { + reg = <0x1000 0x1000>; + read-only; + }; + + clint0: timer@2000000 { + compatible = "canaan,k210-clint", "sifive,clint0"; + reg = <0x2000000 0xC000>; + interrupts-extended = <&cpu0_intc 3 &cpu0_intc 7 + &cpu1_intc 3 &cpu1_intc 7>; + }; + + plic0: interrupt-controller@c000000 { + #interrupt-cells = <1>; + #address-cells = <0>; + compatible = "canaan,k210-plic", "sifive,plic-1.0.0"; + reg = <0xC000000 0x4000000>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 11 &cpu1_intc 11>; + riscv,ndev = <65>; + }; + + uarths0: serial@38000000 { + compatible = "canaan,k210-uarths", "sifive,uart0"; + reg = <0x38000000 0x1000>; + interrupts = <33>; + clocks = <&sysclk K210_CLK_CPU>; + }; + + gpio0: gpio-controller@38001000 { + #interrupt-cells = <2>; + #gpio-cells = <2>; + compatible = "canaan,k210-gpiohs", "sifive,gpio0"; + reg = <0x38001000 0x1000>; + interrupt-controller; + interrupts = <34 35 36 37 38 39 40 41 + 42 43 44 45 46 47 48 49 + 50 51 52 53 54 55 56 57 + 58 59 60 61 62 63 64 65>; + gpio-controller; + ngpios = <32>; + }; + + dmac0: dma-controller@50000000 { + compatible = "snps,axi-dma-1.01a"; + reg = <0x50000000 0x1000>; + interrupts = <27 28 29 30 31 32>; + #dma-cells = <1>; + clocks = <&sysclk K210_CLK_DMA>, <&sysclk K210_CLK_DMA>; + clock-names = "core-clk", "cfgr-clk"; + resets = <&sysrst K210_RST_DMA>; + dma-channels = <6>; + snps,dma-masters = <2>; + snps,priority = <0 1 2 3 4 5>; + snps,data-width = <5>; + snps,block-size = <0x200000 0x200000 0x200000 + 0x200000 0x200000 0x200000>; + snps,axi-max-burst-len = <256>; + }; + + apb0: bus@50200000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-pm-bus"; + ranges; + clocks = <&sysclk K210_CLK_APB0>; + + gpio1: gpio@50200000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dw-apb-gpio"; + reg = <0x50200000 0x80>; + clocks = <&sysclk K210_CLK_APB0>, + <&sysclk K210_CLK_GPIO>; + clock-names = "bus", "db"; + resets = <&sysrst K210_RST_GPIO>; + + gpio1_0: gpio-port@0 { + #gpio-cells = <2>; + #interrupt-cells = <2>; + compatible = "snps,dw-apb-gpio-port"; + reg = <0>; + interrupt-controller; + interrupts = <23>; + gpio-controller; + ngpios = <8>; + }; + }; + + uart1: serial@50210000 { + compatible = "snps,dw-apb-uart"; + reg = <0x50210000 0x100>; + interrupts = <11>; + clocks = <&sysclk K210_CLK_UART1>, + <&sysclk K210_CLK_APB0>; + clock-names = "baudclk", "apb_pclk"; + resets = <&sysrst K210_RST_UART1>; + reg-io-width = <4>; + reg-shift = <2>; + dcd-override; + dsr-override; + cts-override; + ri-override; + }; + + uart2: serial@50220000 { + compatible = "snps,dw-apb-uart"; + reg = <0x50220000 0x100>; + interrupts = <12>; + clocks = <&sysclk K210_CLK_UART2>, + <&sysclk K210_CLK_APB0>; + clock-names = "baudclk", "apb_pclk"; + resets = <&sysrst K210_RST_UART2>; + reg-io-width = <4>; + reg-shift = <2>; + dcd-override; + dsr-override; + cts-override; + ri-override; + }; + + uart3: serial@50230000 { + compatible = "snps,dw-apb-uart"; + reg = <0x50230000 0x100>; + interrupts = <13>; + clocks = <&sysclk K210_CLK_UART3>, + <&sysclk K210_CLK_APB0>; + clock-names = "baudclk", "apb_pclk"; + resets = <&sysrst K210_RST_UART3>; + reg-io-width = <4>; + reg-shift = <2>; + dcd-override; + dsr-override; + cts-override; + ri-override; + }; + + spi2: spi@50240000 { + compatible = "canaan,k210-spi"; + spi-slave; + reg = <0x50240000 0x100>; + #address-cells = <0>; + #size-cells = <0>; + interrupts = <3>; + clocks = <&sysclk K210_CLK_SPI2>, + <&sysclk K210_CLK_APB0>; + clock-names = "ssi_clk", "pclk"; + resets = <&sysrst K210_RST_SPI2>; + spi-max-frequency = <25000000>; + }; + + i2s0: i2s@50250000 { + compatible = "snps,designware-i2s"; + reg = <0x50250000 0x200>; + interrupts = <5>; + clocks = <&sysclk K210_CLK_I2S0>; + clock-names = "i2sclk"; + resets = <&sysrst K210_RST_I2S0>; + }; + + i2s1: i2s@50260000 { + compatible = "snps,designware-i2s"; + reg = <0x50260000 0x200>; + interrupts = <6>; + clocks = <&sysclk K210_CLK_I2S1>; + clock-names = "i2sclk"; + resets = <&sysrst K210_RST_I2S1>; + }; + + i2s2: i2s@50270000 { + compatible = "snps,designware-i2s"; + reg = <0x50270000 0x200>; + interrupts = <7>; + clocks = <&sysclk K210_CLK_I2S2>; + clock-names = "i2sclk"; + resets = <&sysrst K210_RST_I2S2>; + }; + + i2c0: i2c@50280000 { + compatible = "snps,designware-i2c"; + reg = <0x50280000 0x100>; + interrupts = <8>; + clocks = <&sysclk K210_CLK_I2C0>, + <&sysclk K210_CLK_APB0>; + clock-names = "ref", "pclk"; + resets = <&sysrst K210_RST_I2C0>; + }; + + i2c1: i2c@50290000 { + compatible = "snps,designware-i2c"; + reg = <0x50290000 0x100>; + interrupts = <9>; + clocks = <&sysclk K210_CLK_I2C1>, + <&sysclk K210_CLK_APB0>; + clock-names = "ref", "pclk"; + resets = <&sysrst K210_RST_I2C1>; + }; + + i2c2: i2c@502a0000 { + compatible = "snps,designware-i2c"; + reg = <0x502A0000 0x100>; + interrupts = <10>; + clocks = <&sysclk K210_CLK_I2C2>, + <&sysclk K210_CLK_APB0>; + clock-names = "ref", "pclk"; + resets = <&sysrst K210_RST_I2C2>; + }; + + fpioa: pinmux@502b0000 { + compatible = "canaan,k210-fpioa"; + reg = <0x502B0000 0x100>; + clocks = <&sysclk K210_CLK_FPIOA>, + <&sysclk K210_CLK_APB0>; + clock-names = "ref", "pclk"; + resets = <&sysrst K210_RST_FPIOA>; + canaan,k210-sysctl-power = <&sysctl 108>; + }; + + timer0: timer@502d0000 { + compatible = "snps,dw-apb-timer"; + reg = <0x502D0000 0x100>; + interrupts = <14 15>; + clocks = <&sysclk K210_CLK_TIMER0>, + <&sysclk K210_CLK_APB0>; + clock-names = "timer", "pclk"; + resets = <&sysrst K210_RST_TIMER0>; + }; + + timer1: timer@502e0000 { + compatible = "snps,dw-apb-timer"; + reg = <0x502E0000 0x100>; + interrupts = <16 17>; + clocks = <&sysclk K210_CLK_TIMER1>, + <&sysclk K210_CLK_APB0>; + clock-names = "timer", "pclk"; + resets = <&sysrst K210_RST_TIMER1>; + }; + + timer2: timer@502f0000 { + compatible = "snps,dw-apb-timer"; + reg = <0x502F0000 0x100>; + interrupts = <18 19>; + clocks = <&sysclk K210_CLK_TIMER2>, + <&sysclk K210_CLK_APB0>; + clock-names = "timer", "pclk"; + resets = <&sysrst K210_RST_TIMER2>; + }; + }; + + apb1: bus@50400000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-pm-bus"; + ranges; + clocks = <&sysclk K210_CLK_APB1>; + + wdt0: watchdog@50400000 { + compatible = "snps,dw-wdt"; + reg = <0x50400000 0x100>; + interrupts = <21>; + clocks = <&sysclk K210_CLK_WDT0>, + <&sysclk K210_CLK_APB1>; + clock-names = "tclk", "pclk"; + resets = <&sysrst K210_RST_WDT0>; + }; + + wdt1: watchdog@50410000 { + compatible = "snps,dw-wdt"; + reg = <0x50410000 0x100>; + interrupts = <22>; + clocks = <&sysclk K210_CLK_WDT1>, + <&sysclk K210_CLK_APB1>; + clock-names = "tclk", "pclk"; + resets = <&sysrst K210_RST_WDT1>; + }; + + sysctl: syscon@50440000 { + compatible = "canaan,k210-sysctl", + "syscon", "simple-mfd"; + reg = <0x50440000 0x100>; + clocks = <&sysclk K210_CLK_APB1>; + clock-names = "pclk"; + + sysclk: clock-controller { + #clock-cells = <1>; + compatible = "canaan,k210-clk"; + clocks = <&in0>; + }; + + sysrst: reset-controller { + compatible = "canaan,k210-rst"; + #reset-cells = <1>; + }; + + reboot: syscon-reboot { + compatible = "syscon-reboot"; + regmap = <&sysctl>; + offset = <48>; + mask = <1>; + value = <1>; + }; + }; + }; + + apb2: bus@52000000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-pm-bus"; + ranges; + clocks = <&sysclk K210_CLK_APB2>; + + spi0: spi@52000000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "canaan,k210-spi"; + reg = <0x52000000 0x100>; + interrupts = <1>; + clocks = <&sysclk K210_CLK_SPI0>, + <&sysclk K210_CLK_APB2>; + clock-names = "ssi_clk", "pclk"; + resets = <&sysrst K210_RST_SPI0>; + reset-names = "spi"; + spi-max-frequency = <25000000>; + num-cs = <4>; + reg-io-width = <4>; + }; + + spi1: spi@53000000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "canaan,k210-spi"; + reg = <0x53000000 0x100>; + interrupts = <2>; + clocks = <&sysclk K210_CLK_SPI1>, + <&sysclk K210_CLK_APB2>; + clock-names = "ssi_clk", "pclk"; + resets = <&sysrst K210_RST_SPI1>; + reset-names = "spi"; + spi-max-frequency = <25000000>; + num-cs = <4>; + reg-io-width = <4>; + }; + + spi3: spi@54000000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dwc-ssi-1.01a"; + reg = <0x54000000 0x200>; + interrupts = <4>; + clocks = <&sysclk K210_CLK_SPI3>, + <&sysclk K210_CLK_APB2>; + clock-names = "ssi_clk", "pclk"; + resets = <&sysrst K210_RST_SPI3>; + reset-names = "spi"; + /* Could possibly go up to 200 MHz */ + spi-max-frequency = <100000000>; + num-cs = <4>; + reg-io-width = <4>; + }; + }; + }; +}; diff --git a/arch/riscv/boot/dts/canaan/k210_generic.dts b/arch/riscv/boot/dts/canaan/k210_generic.dts new file mode 100644 index 000000000000..396c8ca4d24d --- /dev/null +++ b/arch/riscv/boot/dts/canaan/k210_generic.dts @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include + +/ { + model = "Kendryte K210 generic"; + compatible = "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; +}; + +&fpioa { + pinctrl-0 = <&jtag_pins>; + pinctrl-names = "default"; + status = "okay"; + + jtag_pins: jtag-pinmux { + pinmux = , + , + , + ; + }; + + uarths_pins: uarths-pinmux { + pinmux = , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pins>; + pinctrl-names = "default"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts new file mode 100644 index 000000000000..0bcaf35045e7 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include +#include + +/ { + model = "SiPeed MAIX BiT"; + compatible = "sipeed,maix-bit", "sipeed,maix-bitm", + "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; + + gpio-leds { + compatible = "gpio-leds"; + + led0 { + color = ; + label = "green"; + gpios = <&gpio1_0 4 GPIO_ACTIVE_LOW>; + }; + + led1 { + color = ; + label = "red"; + gpios = <&gpio1_0 5 GPIO_ACTIVE_LOW>; + }; + + led2 { + color = ; + label = "blue"; + gpios = <&gpio1_0 6 GPIO_ACTIVE_LOW>; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + + boot { + label = "BOOT"; + linux,code = ; + gpios = <&gpio0 0 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&fpioa { + pinctrl-names = "default"; + pinctrl-0 = <&jtag_pinctrl>; + status = "okay"; + + jtag_pinctrl: jtag-pinmux { + pinmux = , + , + , + ; + }; + + uarths_pinctrl: uarths-pinmux { + pinmux = , + ; + }; + + gpio_pinctrl: gpio-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + gpiohs_pinctrl: gpiohs-pinmux { + pinmux = , + , + , + , + , + , + , + , + , + , + ; + }; + + i2s0_pinctrl: i2s0-pinmux { + pinmux = , + , + ; + }; + + dvp_pinctrl: dvp-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + spi0_pinctrl: spi0-pinmux { + pinmux = , /* cs */ + , /* rst */ + , /* dc */ + ; /* wr */ + }; + + spi1_pinctrl: spi1-pinmux { + pinmux = , + , + , + ; /* cs */ + }; + + i2c1_pinctrl: i2c1-pinmux { + pinmux = , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio0 { + pinctrl-0 = <&gpiohs_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio1 { + pinctrl-0 = <&gpio_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0 { + #sound-dai-cells = <1>; + pinctrl-0 = <&i2s0_pinctrl>; + pinctrl-names = "default"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_pinctrl>; + pinctrl-names = "default"; + clock-frequency = <400000>; + status = "okay"; +}; + +&spi0 { + pinctrl-0 = <&spi0_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + + panel@0 { + compatible = "sitronix,st7789v"; + reg = <0>; + reset-gpios = <&gpio0 21 GPIO_ACTIVE_LOW>; + dc-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; + spi-max-frequency = <15000000>; + spi-cs-high; + status = "disabled"; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 13 GPIO_ACTIVE_LOW>; + status = "okay"; + + slot@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <25000000>; + broken-cd; + }; +}; + +&spi3 { + spi-flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + m25p,fast-read; + broken-flash-reset; + }; +}; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts new file mode 100644 index 000000000000..ac8a03f5867a --- /dev/null +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include +#include + +/ { + model = "SiPeed MAIX Dock"; + compatible = "sipeed,maix-dock-m1", "sipeed,maix-dock-m1w", + "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; + + gpio-leds { + compatible = "gpio-leds"; + + /* + * Note: the board wiring drawing documents green on + * gpio #4, red on gpio #5 and blue on gpio #6. However, + * the board is actually wired differently as defined here. + */ + led0 { + color = ; + label = "blue"; + gpios = <&gpio1_0 4 GPIO_ACTIVE_LOW>; + }; + + led1 { + color = ; + label = "green"; + gpios = <&gpio1_0 5 GPIO_ACTIVE_LOW>; + }; + + led2 { + color = ; + label = "red"; + gpios = <&gpio1_0 6 GPIO_ACTIVE_LOW>; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + + boot { + label = "BOOT"; + linux,code = ; + gpios = <&gpio0 0 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&fpioa { + pinctrl-0 = <&jtag_pinctrl>; + pinctrl-names = "default"; + status = "okay"; + + jtag_pinctrl: jtag-pinmux { + pinmux = , + , + , + ; + }; + + uarths_pinctrl: uarths-pinmux { + pinmux = , + ; + }; + + gpio_pinctrl: gpio-pinmux { + pinmux = , + , + , + , + , + ; + }; + + gpiohs_pinctrl: gpiohs-pinmux { + pinmux = , + , + , + , + , + , + , + , + , + , + ; + }; + + i2s0_pinctrl: i2s0-pinmux { + pinmux = , + , + ; + }; + + dvp_pinctrl: dvp-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + spi0_pinctrl: spi0-pinmux { + pinmux = , /* cs */ + , /* rst */ + , /* dc */ + ; /* wr */ + }; + + spi1_pinctrl: spi1-pinmux { + pinmux = , + , + , + ; /* cs */ + }; + + i2c1_pinctrl: i2c1-pinmux { + pinmux = , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio0 { + pinctrl-0 = <&gpiohs_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio1 { + pinctrl-0 = <&gpio_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0 { + #sound-dai-cells = <1>; + pinctrl-0 = <&i2s0_pinctrl>; + pinctrl-names = "default"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_pinctrl>; + pinctrl-names = "default"; + clock-frequency = <400000>; + status = "okay"; +}; + +&spi0 { + pinctrl-0 = <&spi0_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + + panel@0 { + compatible = "sitronix,st7789v"; + reg = <0>; + reset-gpios = <&gpio0 21 GPIO_ACTIVE_LOW>; + dc-gpios = <&gpio0 22 0>; + spi-max-frequency = <15000000>; + status = "disabled"; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 13 GPIO_ACTIVE_LOW>; + status = "okay"; + + slot@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <25000000>; + broken-cd; + }; +}; + +&spi3 { + spi-flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + m25p,fast-read; + broken-flash-reset; + }; +}; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts new file mode 100644 index 000000000000..623998194bc1 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include +#include + +/ { + model = "SiPeed MAIX GO"; + compatible = "sipeed,maix-go", "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; + + gpio-leds { + compatible = "gpio-leds"; + + led0 { + color = ; + label = "green"; + gpios = <&gpio1_0 4 GPIO_ACTIVE_LOW>; + }; + + led1 { + color = ; + label = "red"; + gpios = <&gpio1_0 5 GPIO_ACTIVE_LOW>; + }; + + led2 { + color = ; + label = "blue"; + gpios = <&gpio1_0 6 GPIO_ACTIVE_LOW>; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + + up { + label = "UP"; + linux,code = ; + gpios = <&gpio1_0 7 GPIO_ACTIVE_LOW>; + }; + + press { + label = "PRESS"; + linux,code = ; + gpios = <&gpio0 0 GPIO_ACTIVE_LOW>; + }; + + down { + label = "DOWN"; + linux,code = ; + gpios = <&gpio0 1 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&fpioa { + pinctrl-0 = <&jtag_pinctrl>; + pinctrl-names = "default"; + status = "okay"; + + jtag_pinctrl: jtag-pinmux { + pinmux = , + , + , + ; + }; + + uarths_pinctrl: uarths-pinmux { + pinmux = , + ; + }; + + gpio_pinctrl: gpio-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + gpiohs_pinctrl: gpiohs-pinmux { + pinmux = , + , + , + , + , + , + , + , + , + , + ; + }; + + i2s0_pinctrl: i2s0-pinmux { + pinmux = , + , + ; + }; + + dvp_pinctrl: dvp-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + spi0_pinctrl: spi0-pinmux { + pinmux = , /* cs */ + , /* rst */ + , /* dc */ + ; /* wr */ + }; + + spi1_pinctrl: spi1-pinmux { + pinmux = , + , + , + ; /* cs */ + }; + + i2c1_pinctrl: i2c1-pinmux { + pinmux = , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio0 { + pinctrl-0 = <&gpiohs_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio1 { + pinctrl-0 = <&gpio_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0 { + #sound-dai-cells = <1>; + pinctrl-0 = <&i2s0_pinctrl>; + pinctrl-names = "default"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_pinctrl>; + pinctrl-names = "default"; + clock-frequency = <400000>; + status = "okay"; +}; + +&spi0 { + pinctrl-0 = <&spi0_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + + panel@0 { + compatible = "sitronix,st7789v"; + reg = <0>; + reset-gpios = <&gpio0 21 GPIO_ACTIVE_LOW>; + dc-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; + spi-max-frequency = <15000000>; + status = "disabled"; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 13 GPIO_ACTIVE_LOW>; + status = "okay"; + + slot@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <25000000>; + broken-cd; + }; +}; + +&spi3 { + spi-flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + m25p,fast-read; + broken-flash-reset; + }; +}; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts new file mode 100644 index 000000000000..cf605ba0d67e --- /dev/null +++ b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include + +/ { + model = "SiPeed MAIXDUINO"; + compatible = "sipeed,maixduino", "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; + + gpio-keys { + compatible = "gpio-keys"; + + boot { + label = "BOOT"; + linux,code = ; + gpios = <&gpio0 0 GPIO_ACTIVE_LOW>; + }; + }; + + vcc_3v3: regulator-3v3 { + compatible = "regulator-fixed"; + regulator-name = "3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; +}; + +&fpioa { + status = "okay"; + + uarths_pinctrl: uarths-pinmux { + pinmux = , /* Header "0" */ + ; /* Header "1" */ + }; + + gpio_pinctrl: gpio-pinmux { + pinmux = , + ; + }; + + gpiohs_pinctrl: gpiohs-pinmux { + pinmux = , /* BOOT */ + , /* Header "2" */ + , /* Header "3" */ + , /* Header "4" */ + , /* Header "5" */ + , /* Header "6" */ + , /* Header "7" */ + , /* Header "8" */ + , /* Header "9" */ + , /* Header "10" */ + , /* Header "11" */ + , /* Header "12" */ + ; /* Header "13" */ + }; + + i2s0_pinctrl: i2s0-pinmux { + pinmux = , + , + ; + }; + + spi1_pinctrl: spi1-pinmux { + pinmux = , + , + , + ; /* cs */ + }; + + i2c1_pinctrl: i2c1-pinmux { + pinmux = , /* Header "scl" */ + ; /* Header "sda" */ + }; + + i2s1_pinctrl: i2s1-pinmux { + pinmux = , + , + ; + }; + + spi0_pinctrl: spi0-pinmux { + pinmux = , /* cs */ + , /* rst */ + , /* dc */ + ; /* wr */ + }; + + dvp_pinctrl: dvp-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio0 { + pinctrl-0 = <&gpiohs_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio1 { + pinctrl-0 = <&gpio_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0 { + #sound-dai-cells = <1>; + pinctrl-0 = <&i2s0_pinctrl>; + pinctrl-names = "default"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_pinctrl>; + pinctrl-names = "default"; + clock-frequency = <400000>; + status = "okay"; +}; + +&spi0 { + pinctrl-0 = <&spi0_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + + panel@0 { + compatible = "sitronix,st7789v"; + reg = <0>; + reset-gpios = <&gpio0 21 GPIO_ACTIVE_LOW>; + dc-gpios = <&gpio0 22 0>; + spi-max-frequency = <15000000>; + power-supply = <&vcc_3v3>; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio1_0 2 GPIO_ACTIVE_LOW>; + status = "okay"; + + slot@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <25000000>; + broken-cd; + }; +}; + +&spi3 { + spi-flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + m25p,fast-read; + broken-flash-reset; + }; +}; diff --git a/arch/riscv/boot/dts/kendryte/Makefile b/arch/riscv/boot/dts/kendryte/Makefile deleted file mode 100644 index 1a88e616f18e..000000000000 --- a/arch/riscv/boot/dts/kendryte/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_SOC_KENDRYTE_K210_DTB) += k210.dtb - -obj-$(CONFIG_SOC_KENDRYTE_K210_DTB_BUILTIN) += $(addsuffix .o, $(dtb-y)) diff --git a/arch/riscv/boot/dts/kendryte/k210.dts b/arch/riscv/boot/dts/kendryte/k210.dts deleted file mode 100644 index 0d1f28fce6b2..000000000000 --- a/arch/riscv/boot/dts/kendryte/k210.dts +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Copyright (C) 2020 Western Digital Corporation or its affiliates. - */ - -/dts-v1/; - -#include "k210.dtsi" - -/ { - model = "Kendryte K210 generic"; - compatible = "kendryte,k210"; - - chosen { - bootargs = "earlycon console=ttySIF0"; - stdout-path = "serial0"; - }; -}; - -&uarths0 { - status = "okay"; -}; - diff --git a/arch/riscv/boot/dts/kendryte/k210.dtsi b/arch/riscv/boot/dts/kendryte/k210.dtsi deleted file mode 100644 index d2d0ff645632..000000000000 --- a/arch/riscv/boot/dts/kendryte/k210.dtsi +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Copyright (C) 2019 Sean Anderson - * Copyright (C) 2020 Western Digital Corporation or its affiliates. - */ -#include - -/ { - /* - * Although the K210 is a 64-bit CPU, the address bus is only 32-bits - * wide, and the upper half of all addresses is ignored. - */ - #address-cells = <1>; - #size-cells = <1>; - compatible = "kendryte,k210"; - - aliases { - serial0 = &uarths0; - }; - - /* - * The K210 has an sv39 MMU following the priviledge specification v1.9. - * Since this is a non-ratified draft specification, the kernel does not - * support it and the K210 support enabled only for the !MMU case. - * Be consistent with this by setting the CPUs MMU type to "none". - */ - cpus { - #address-cells = <1>; - #size-cells = <0>; - timebase-frequency = <7800000>; - cpu0: cpu@0 { - device_type = "cpu"; - reg = <0>; - compatible = "kendryte,k210", "sifive,rocket0", "riscv"; - riscv,isa = "rv64imafdc"; - mmu-type = "none"; - i-cache-size = <0x8000>; - i-cache-block-size = <64>; - d-cache-size = <0x8000>; - d-cache-block-size = <64>; - clocks = <&sysctl K210_CLK_CPU>; - clock-frequency = <390000000>; - cpu0_intc: interrupt-controller { - #interrupt-cells = <1>; - interrupt-controller; - compatible = "riscv,cpu-intc"; - }; - }; - cpu1: cpu@1 { - device_type = "cpu"; - reg = <1>; - compatible = "kendryte,k210", "sifive,rocket0", "riscv"; - riscv,isa = "rv64imafdc"; - mmu-type = "none"; - i-cache-size = <0x8000>; - i-cache-block-size = <64>; - d-cache-size = <0x8000>; - d-cache-block-size = <64>; - clocks = <&sysctl K210_CLK_CPU>; - clock-frequency = <390000000>; - cpu1_intc: interrupt-controller { - #interrupt-cells = <1>; - interrupt-controller; - compatible = "riscv,cpu-intc"; - }; - }; - }; - - sram: memory@80000000 { - device_type = "memory"; - reg = <0x80000000 0x400000>, - <0x80400000 0x200000>, - <0x80600000 0x200000>; - reg-names = "sram0", "sram1", "aisram"; - }; - - clocks { - in0: oscillator { - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <26000000>; - }; - }; - - soc { - #address-cells = <1>; - #size-cells = <1>; - compatible = "kendryte,k210-soc", "simple-bus"; - ranges; - interrupt-parent = <&plic0>; - - sysctl: sysctl@50440000 { - compatible = "kendryte,k210-sysctl", "simple-mfd"; - reg = <0x50440000 0x1000>; - #clock-cells = <1>; - }; - - clint0: clint@2000000 { - #interrupt-cells = <1>; - compatible = "riscv,clint0"; - reg = <0x2000000 0xC000>; - interrupts-extended = <&cpu0_intc 3 &cpu0_intc 7 - &cpu1_intc 3 &cpu1_intc 7>; - clocks = <&sysctl K210_CLK_ACLK>; - }; - - plic0: interrupt-controller@c000000 { - #interrupt-cells = <1>; - interrupt-controller; - compatible = "kendryte,k210-plic0", "riscv,plic0"; - reg = <0xC000000 0x4000000>; - interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 0xffffffff>, - <&cpu1_intc 11>, <&cpu1_intc 0xffffffff>; - riscv,ndev = <65>; - riscv,max-priority = <7>; - }; - - uarths0: serial@38000000 { - compatible = "kendryte,k210-uarths", "sifive,uart0"; - reg = <0x38000000 0x1000>; - interrupts = <33>; - clocks = <&sysctl K210_CLK_CPU>; - }; - }; -}; diff --git a/arch/riscv/boot/dts/sifive/Makefile b/arch/riscv/boot/dts/sifive/Makefile index 6d6189e6e4af..74c47fe9fc22 100644 --- a/arch/riscv/boot/dts/sifive/Makefile +++ b/arch/riscv/boot/dts/sifive/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_SOC_SIFIVE) += hifive-unleashed-a00.dtb +dtb-$(CONFIG_SOC_SIFIVE) += hifive-unleashed-a00.dtb \ + hifive-unmatched-a00.dtb diff --git a/arch/riscv/boot/dts/sifive/fu740-c000.dtsi b/arch/riscv/boot/dts/sifive/fu740-c000.dtsi new file mode 100644 index 000000000000..eeb4f8c3e0e7 --- /dev/null +++ b/arch/riscv/boot/dts/sifive/fu740-c000.dtsi @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2020 SiFive, Inc */ + +/dts-v1/; + +#include + +/ { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sifive,fu740-c000", "sifive,fu740"; + + aliases { + serial0 = &uart0; + serial1 = &uart1; + ethernet0 = ð0; + }; + + chosen { + }; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + cpu0: cpu@0 { + compatible = "sifive,bullet0", "riscv"; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <16384>; + next-level-cache = <&ccache>; + reg = <0x0>; + riscv,isa = "rv64imac"; + status = "disabled"; + cpu0_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + cpu1: cpu@1 { + compatible = "sifive,bullet0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <40>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <40>; + mmu-type = "riscv,sv39"; + next-level-cache = <&ccache>; + reg = <0x1>; + riscv,isa = "rv64imafdc"; + tlb-split; + cpu1_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + cpu2: cpu@2 { + compatible = "sifive,bullet0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <40>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <40>; + mmu-type = "riscv,sv39"; + next-level-cache = <&ccache>; + reg = <0x2>; + riscv,isa = "rv64imafdc"; + tlb-split; + cpu2_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + cpu3: cpu@3 { + compatible = "sifive,bullet0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <40>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <40>; + mmu-type = "riscv,sv39"; + next-level-cache = <&ccache>; + reg = <0x3>; + riscv,isa = "rv64imafdc"; + tlb-split; + cpu3_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + cpu4: cpu@4 { + compatible = "sifive,bullet0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <40>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <40>; + mmu-type = "riscv,sv39"; + next-level-cache = <&ccache>; + reg = <0x4>; + riscv,isa = "rv64imafdc"; + tlb-split; + cpu4_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + }; + soc { + #address-cells = <2>; + #size-cells = <2>; + compatible = "simple-bus"; + ranges; + plic0: interrupt-controller@c000000 { + #interrupt-cells = <1>; + #address-cells = <0>; + compatible = "sifive,fu540-c000-plic", "sifive,plic-1.0.0"; + reg = <0x0 0xc000000 0x0 0x4000000>; + riscv,ndev = <69>; + interrupt-controller; + interrupts-extended = < + &cpu0_intc 0xffffffff + &cpu1_intc 0xffffffff &cpu1_intc 9 + &cpu2_intc 0xffffffff &cpu2_intc 9 + &cpu3_intc 0xffffffff &cpu3_intc 9 + &cpu4_intc 0xffffffff &cpu4_intc 9>; + }; + prci: clock-controller@10000000 { + compatible = "sifive,fu740-c000-prci"; + reg = <0x0 0x10000000 0x0 0x1000>; + clocks = <&hfclk>, <&rtcclk>; + #clock-cells = <1>; + }; + uart0: serial@10010000 { + compatible = "sifive,fu740-c000-uart", "sifive,uart0"; + reg = <0x0 0x10010000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <39>; + clocks = <&prci PRCI_CLK_PCLK>; + status = "disabled"; + }; + uart1: serial@10011000 { + compatible = "sifive,fu740-c000-uart", "sifive,uart0"; + reg = <0x0 0x10011000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <40>; + clocks = <&prci PRCI_CLK_PCLK>; + status = "disabled"; + }; + i2c0: i2c@10030000 { + compatible = "sifive,fu740-c000-i2c", "sifive,i2c0"; + reg = <0x0 0x10030000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <52>; + clocks = <&prci PRCI_CLK_PCLK>; + reg-shift = <2>; + reg-io-width = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + i2c1: i2c@10031000 { + compatible = "sifive,fu740-c000-i2c", "sifive,i2c0"; + reg = <0x0 0x10031000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <53>; + clocks = <&prci PRCI_CLK_PCLK>; + reg-shift = <2>; + reg-io-width = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + qspi0: spi@10040000 { + compatible = "sifive,fu740-c000-spi", "sifive,spi0"; + reg = <0x0 0x10040000 0x0 0x1000>, + <0x0 0x20000000 0x0 0x10000000>; + interrupt-parent = <&plic0>; + interrupts = <41>; + clocks = <&prci PRCI_CLK_PCLK>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + qspi1: spi@10041000 { + compatible = "sifive,fu740-c000-spi", "sifive,spi0"; + reg = <0x0 0x10041000 0x0 0x1000>, + <0x0 0x30000000 0x0 0x10000000>; + interrupt-parent = <&plic0>; + interrupts = <42>; + clocks = <&prci PRCI_CLK_PCLK>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + spi0: spi@10050000 { + compatible = "sifive,fu740-c000-spi", "sifive,spi0"; + reg = <0x0 0x10050000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <43>; + clocks = <&prci PRCI_CLK_PCLK>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + eth0: ethernet@10090000 { + compatible = "sifive,fu540-c000-gem"; + interrupt-parent = <&plic0>; + interrupts = <55>; + reg = <0x0 0x10090000 0x0 0x2000>, + <0x0 0x100a0000 0x0 0x1000>; + local-mac-address = [00 00 00 00 00 00]; + clock-names = "pclk", "hclk"; + clocks = <&prci PRCI_CLK_GEMGXLPLL>, + <&prci PRCI_CLK_GEMGXLPLL>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + pwm0: pwm@10020000 { + compatible = "sifive,fu740-c000-pwm", "sifive,pwm0"; + reg = <0x0 0x10020000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <44>, <45>, <46>, <47>; + clocks = <&prci PRCI_CLK_PCLK>; + #pwm-cells = <3>; + status = "disabled"; + }; + pwm1: pwm@10021000 { + compatible = "sifive,fu740-c000-pwm", "sifive,pwm0"; + reg = <0x0 0x10021000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <48>, <49>, <50>, <51>; + clocks = <&prci PRCI_CLK_PCLK>; + #pwm-cells = <3>; + status = "disabled"; + }; + ccache: cache-controller@2010000 { + compatible = "sifive,fu740-c000-ccache", "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-sets = <2048>; + cache-size = <2097152>; + cache-unified; + interrupt-parent = <&plic0>; + interrupts = <19 20 21 22>; + reg = <0x0 0x2010000 0x0 0x1000>; + }; + gpio: gpio@10060000 { + compatible = "sifive,fu740-c000-gpio", "sifive,gpio0"; + interrupt-parent = <&plic0>; + interrupts = <23>, <24>, <25>, <26>, <27>, <28>, <29>, + <30>, <31>, <32>, <33>, <34>, <35>, <36>, + <37>, <38>; + reg = <0x0 0x10060000 0x0 0x1000>; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&prci PRCI_CLK_PCLK>; + status = "disabled"; + }; + }; +}; diff --git a/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts new file mode 100644 index 000000000000..b1c3c596578f --- /dev/null +++ b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2020 SiFive, Inc */ + +#include "fu740-c000.dtsi" +#include + +/* Clock frequency (in Hz) of the PCB crystal for rtcclk */ +#define RTCCLK_FREQ 1000000 + +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "SiFive HiFive Unmatched A00"; + compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000", + "sifive,fu740"; + + chosen { + stdout-path = "serial0"; + }; + + cpus { + timebase-frequency = ; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x0 0x80000000 0x2 0x00000000>; + }; + + soc { + }; + + hfclk: hfclk { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <26000000>; + clock-output-names = "hfclk"; + }; + + rtcclk: rtcclk { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = ; + clock-output-names = "rtcclk"; + }; +}; + +&uart0 { + status = "okay"; +}; + +&uart1 { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + temperature-sensor@4c { + compatible = "ti,tmp451"; + reg = <0x4c>; + interrupt-parent = <&gpio>; + interrupts = <6 IRQ_TYPE_LEVEL_LOW>; + }; + + pmic@58 { + compatible = "dlg,da9063"; + reg = <0x58>; + interrupt-parent = <&gpio>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + interrupt-controller; + + regulators { + vdd_bcore1: bcore1 { + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-min-microamp = <5000000>; + regulator-max-microamp = <5000000>; + regulator-always-on; + }; + + vdd_bcore2: bcore2 { + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-min-microamp = <5000000>; + regulator-max-microamp = <5000000>; + regulator-always-on; + }; + + vdd_bpro: bpro { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <2500000>; + regulator-max-microamp = <2500000>; + regulator-always-on; + }; + + vdd_bperi: bperi { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-min-microamp = <1500000>; + regulator-max-microamp = <1500000>; + regulator-always-on; + }; + + vdd_bmem: bmem { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-min-microamp = <3000000>; + regulator-max-microamp = <3000000>; + regulator-always-on; + }; + + vdd_bio: bio { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-min-microamp = <3000000>; + regulator-max-microamp = <3000000>; + regulator-always-on; + }; + + vdd_ldo1: ldo1 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <100000>; + regulator-max-microamp = <100000>; + regulator-always-on; + }; + + vdd_ldo2: ldo2 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ldo3: ldo3 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ldo4: ldo4 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ldo5: ldo5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <100000>; + regulator-max-microamp = <100000>; + regulator-always-on; + }; + + vdd_ldo6: ldo6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ldo7: ldo7 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ldo8: ldo8 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ld09: ldo9 { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + }; + + vdd_ldo10: ldo10 { + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-min-microamp = <300000>; + regulator-max-microamp = <300000>; + }; + + vdd_ldo11: ldo11 { + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + regulator-min-microamp = <300000>; + regulator-max-microamp = <300000>; + regulator-always-on; + }; + }; + }; +}; + +&qspi0 { + status = "okay"; + flash@0 { + compatible = "issi,is25wp256", "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + m25p,fast-read; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; + }; +}; + +&spi0 { + status = "okay"; + mmc@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + spi-max-frequency = <20000000>; + voltage-ranges = <3300 3300>; + disable-wp; + }; +}; + +ð0 { + status = "okay"; + phy-mode = "gmii"; + phy-handle = <&phy0>; + phy0: ethernet-phy@0 { + reg = <0>; + }; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm1 { + status = "okay"; +}; + +&gpio { + status = "okay"; +}; diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 8c3d1e451703..6c0625aa96c7 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -17,6 +17,7 @@ CONFIG_BPF_SYSCALL=y CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_SMP=y +CONFIG_HOTPLUG_CPU=y CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index cd1df62b13c7..b16a2a12c82a 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -1,17 +1,19 @@ # CONFIG_CPU_ISOLATION is not set -CONFIG_LOG_BUF_SHIFT=15 +CONFIG_LOG_BUF_SHIFT=13 CONFIG_PRINTK_SAFE_LOG_BUF_SHIFT=12 CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_FORCE=y +# CONFIG_RD_GZIP is not set # CONFIG_RD_BZIP2 is not set # CONFIG_RD_LZMA is not set # CONFIG_RD_XZ is not set # CONFIG_RD_LZO is not set # CONFIG_RD_LZ4 is not set +# CONFIG_RD_ZSTD is not set CONFIG_CC_OPTIMIZE_FOR_SIZE=y # CONFIG_SYSFS_SYSCALL is not set # CONFIG_FHANDLE is not set # CONFIG_BASE_FULL is not set +# CONFIG_FUTEX is not set # CONFIG_EPOLL is not set # CONFIG_SIGNALFD is not set # CONFIG_TIMERFD is not set @@ -25,15 +27,17 @@ CONFIG_EMBEDDED=y # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set CONFIG_SLOB=y -# CONFIG_SLAB_MERGE_DEFAULT is not set # CONFIG_MMU is not set -CONFIG_SOC_KENDRYTE=y +CONFIG_SOC_CANAAN=y +CONFIG_SOC_CANAAN_K210_DTB_SOURCE="k210_generic" CONFIG_MAXPHYSMEM_2GB=y CONFIG_SMP=y CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0" CONFIG_CMDLINE_FORCE=y -CONFIG_JUMP_LABEL=y +# CONFIG_SECCOMP is not set +# CONFIG_STACKPROTECTOR is not set +# CONFIG_GCC_PLUGINS is not set # CONFIG_BLOCK is not set CONFIG_BINFMT_FLAT=y # CONFIG_COREDUMP is not set @@ -41,23 +45,47 @@ CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y # CONFIG_FW_LOADER is not set # CONFIG_ALLOW_DEV_COREDUMP is not set -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set +# CONFIG_INPUT is not set # CONFIG_SERIO is not set +# CONFIG_VT is not set +# CONFIG_UNIX98_PTYS is not set # CONFIG_LEGACY_PTYS is not set # CONFIG_LDISC_AUTOLOAD is not set # CONFIG_HW_RANDOM is not set # CONFIG_DEVMEM is not set +CONFIG_I2C=y +# CONFIG_I2C_COMPAT is not set +CONFIG_I2C_CHARDEV=y +# CONFIG_I2C_HELPER_AUTO is not set +CONFIG_I2C_DESIGNWARE_PLATFORM=y +CONFIG_SPI=y +# CONFIG_SPI_MEM is not set +CONFIG_SPI_DESIGNWARE=y +CONFIG_SPI_DW_MMIO=y +# CONFIG_GPIO_CDEV_V1 is not set +CONFIG_GPIO_DWAPB=y +CONFIG_GPIO_SIFIVE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set -# CONFIG_VGA_CONSOLE is not set -# CONFIG_HID is not set # CONFIG_USB_SUPPORT is not set +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_USER=y # CONFIG_VIRTIO_MENU is not set +# CONFIG_VHOST_MENU is not set +# CONFIG_SURFACE_PLATFORMS is not set +# CONFIG_FILE_LOCKING is not set # CONFIG_DNOTIFY is not set # CONFIG_INOTIFY_USER is not set # CONFIG_MISC_FILESYSTEMS is not set CONFIG_LSM="[]" CONFIG_PRINTK_TIME=y +# CONFIG_SYMBOLIC_ERRNAME is not set +# CONFIG_DEBUG_BUGVERBOSE is not set +# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set +# CONFIG_FRAME_POINTER is not set # CONFIG_DEBUG_MISC is not set CONFIG_PANIC_ON_OOPS=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/riscv/configs/nommu_k210_sdcard_defconfig b/arch/riscv/configs/nommu_k210_sdcard_defconfig new file mode 100644 index 000000000000..61f887f65419 --- /dev/null +++ b/arch/riscv/configs/nommu_k210_sdcard_defconfig @@ -0,0 +1,92 @@ +# CONFIG_CPU_ISOLATION is not set +CONFIG_LOG_BUF_SHIFT=13 +CONFIG_PRINTK_SAFE_LOG_BUF_SHIFT=12 +CONFIG_CC_OPTIMIZE_FOR_SIZE=y +# CONFIG_SYSFS_SYSCALL is not set +# CONFIG_FHANDLE is not set +# CONFIG_BASE_FULL is not set +# CONFIG_FUTEX is not set +# CONFIG_EPOLL is not set +# CONFIG_SIGNALFD is not set +# CONFIG_TIMERFD is not set +# CONFIG_EVENTFD is not set +# CONFIG_AIO is not set +# CONFIG_IO_URING is not set +# CONFIG_ADVISE_SYSCALLS is not set +# CONFIG_MEMBARRIER is not set +# CONFIG_KALLSYMS is not set +CONFIG_EMBEDDED=y +# CONFIG_VM_EVENT_COUNTERS is not set +# CONFIG_COMPAT_BRK is not set +CONFIG_SLOB=y +# CONFIG_MMU is not set +CONFIG_SOC_CANAAN=y +CONFIG_SOC_CANAAN_K210_DTB_SOURCE="k210_generic" +CONFIG_MAXPHYSMEM_2GB=y +CONFIG_SMP=y +CONFIG_NR_CPUS=2 +CONFIG_CMDLINE="earlycon console=ttySIF0 rootdelay=2 root=/dev/mmcblk0p1 ro" +CONFIG_CMDLINE_FORCE=y +# CONFIG_SECCOMP is not set +# CONFIG_STACKPROTECTOR is not set +# CONFIG_GCC_PLUGINS is not set +# CONFIG_BLK_DEV_BSG is not set +# CONFIG_MQ_IOSCHED_DEADLINE is not set +# CONFIG_MQ_IOSCHED_KYBER is not set +CONFIG_BINFMT_FLAT=y +# CONFIG_COREDUMP is not set +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +# CONFIG_FW_LOADER is not set +# CONFIG_ALLOW_DEV_COREDUMP is not set +# CONFIG_BLK_DEV is not set +# CONFIG_INPUT is not set +# CONFIG_SERIO is not set +# CONFIG_VT is not set +# CONFIG_LEGACY_PTYS is not set +# CONFIG_LDISC_AUTOLOAD is not set +# CONFIG_HW_RANDOM is not set +# CONFIG_DEVMEM is not set +CONFIG_I2C=y +CONFIG_I2C_CHARDEV=y +# CONFIG_I2C_HELPER_AUTO is not set +CONFIG_I2C_DESIGNWARE_PLATFORM=y +CONFIG_SPI=y +# CONFIG_SPI_MEM is not set +CONFIG_SPI_DESIGNWARE=y +CONFIG_SPI_DW_MMIO=y +# CONFIG_GPIO_CDEV_V1 is not set +CONFIG_GPIO_DWAPB=y +CONFIG_GPIO_SIFIVE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +# CONFIG_HWMON is not set +# CONFIG_USB_SUPPORT is not set +CONFIG_MMC=y +# CONFIG_PWRSEQ_EMMC is not set +# CONFIG_PWRSEQ_SIMPLE is not set +CONFIG_MMC_SPI=y +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_USER=y +# CONFIG_VIRTIO_MENU is not set +# CONFIG_VHOST_MENU is not set +# CONFIG_SURFACE_PLATFORMS is not set +CONFIG_EXT2_FS=y +# CONFIG_FILE_LOCKING is not set +# CONFIG_DNOTIFY is not set +# CONFIG_INOTIFY_USER is not set +# CONFIG_MISC_FILESYSTEMS is not set +CONFIG_LSM="[]" +CONFIG_PRINTK_TIME=y +# CONFIG_SYMBOLIC_ERRNAME is not set +# CONFIG_DEBUG_BUGVERBOSE is not set +# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set +# CONFIG_FRAME_POINTER is not set +# CONFIG_DEBUG_MISC is not set +CONFIG_PANIC_ON_OOPS=y +# CONFIG_SCHED_DEBUG is not set +# CONFIG_RCU_TRACE is not set +# CONFIG_FTRACE is not set +# CONFIG_RUNTIME_TESTING_MENU is not set diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index 2c2cda6cc1c5..8dd02b842fef 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -18,6 +18,7 @@ CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_ARCH_RV32I=y CONFIG_SMP=y +CONFIG_HOTPLUG_CPU=y CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/riscv/include/asm/bug.h b/arch/riscv/include/asm/bug.h index d6f1ec08d97b..d3804a2f9aad 100644 --- a/arch/riscv/include/asm/bug.h +++ b/arch/riscv/include/asm/bug.h @@ -85,6 +85,7 @@ do { \ struct pt_regs; struct task_struct; +void __show_regs(struct pt_regs *regs); void die(struct pt_regs *regs, const char *str); void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr); diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index cec462e198ce..caadfc1d7487 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -41,10 +41,16 @@ #define SATP_PPN _AC(0x003FFFFF, UL) #define SATP_MODE_32 _AC(0x80000000, UL) #define SATP_MODE SATP_MODE_32 +#define SATP_ASID_BITS 9 +#define SATP_ASID_SHIFT 22 +#define SATP_ASID_MASK _AC(0x1FF, UL) #else #define SATP_PPN _AC(0x00000FFFFFFFFFFF, UL) #define SATP_MODE_39 _AC(0x8000000000000000, UL) #define SATP_MODE SATP_MODE_39 +#define SATP_ASID_BITS 16 +#define SATP_ASID_SHIFT 44 +#define SATP_ASID_MASK _AC(0xFFFF, UL) #endif /* Exception cause high bit - is an interrupt if set */ diff --git a/arch/riscv/include/asm/kasan.h b/arch/riscv/include/asm/kasan.h index b04028c6218c..a2b3d9cdbc86 100644 --- a/arch/riscv/include/asm/kasan.h +++ b/arch/riscv/include/asm/kasan.h @@ -8,12 +8,28 @@ #ifdef CONFIG_KASAN +/* + * The following comment was copied from arm64: + * KASAN_SHADOW_START: beginning of the kernel virtual addresses. + * KASAN_SHADOW_END: KASAN_SHADOW_START + 1/N of kernel virtual addresses, + * where N = (1 << KASAN_SHADOW_SCALE_SHIFT). + * + * KASAN_SHADOW_OFFSET: + * This value is used to map an address to the corresponding shadow + * address by the following formula: + * shadow_addr = (address >> KASAN_SHADOW_SCALE_SHIFT) + KASAN_SHADOW_OFFSET + * + * (1 << (64 - KASAN_SHADOW_SCALE_SHIFT)) shadow addresses that lie in range + * [KASAN_SHADOW_OFFSET, KASAN_SHADOW_END) cover all 64-bits of virtual + * addresses. So KASAN_SHADOW_OFFSET should satisfy the following equation: + * KASAN_SHADOW_OFFSET = KASAN_SHADOW_END - + * (1ULL << (64 - KASAN_SHADOW_SCALE_SHIFT)) + */ #define KASAN_SHADOW_SCALE_SHIFT 3 -#define KASAN_SHADOW_SIZE (UL(1) << (38 - KASAN_SHADOW_SCALE_SHIFT)) -#define KASAN_SHADOW_START KERN_VIRT_START /* 2^64 - 2^38 */ +#define KASAN_SHADOW_SIZE (UL(1) << ((CONFIG_VA_BITS - 1) - KASAN_SHADOW_SCALE_SHIFT)) +#define KASAN_SHADOW_START KERN_VIRT_START #define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE) - #define KASAN_SHADOW_OFFSET (KASAN_SHADOW_END - (1ULL << \ (64 - KASAN_SHADOW_SCALE_SHIFT))) diff --git a/arch/riscv/include/asm/kprobes.h b/arch/riscv/include/asm/kprobes.h index 56a98ea30731..4647d38018f6 100644 --- a/arch/riscv/include/asm/kprobes.h +++ b/arch/riscv/include/asm/kprobes.h @@ -11,4 +11,44 @@ #include +#ifdef CONFIG_KPROBES +#include +#include +#include + +#define __ARCH_WANT_KPROBES_INSN_SLOT +#define MAX_INSN_SIZE 2 + +#define flush_insn_slot(p) do { } while (0) +#define kretprobe_blacklist_size 0 + +#include + +struct prev_kprobe { + struct kprobe *kp; + unsigned int status; +}; + +/* Single step context for kprobe */ +struct kprobe_step_ctx { + unsigned long ss_pending; + unsigned long match_addr; +}; + +/* per-cpu kprobe control block */ +struct kprobe_ctlblk { + unsigned int kprobe_status; + unsigned long saved_status; + struct prev_kprobe prev_kprobe; + struct kprobe_step_ctx ss_ctx; +}; + +void arch_remove_kprobe(struct kprobe *p); +int kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr); +bool kprobe_breakpoint_handler(struct pt_regs *regs); +bool kprobe_single_step_handler(struct pt_regs *regs); +void kretprobe_trampoline(void); +void __kprobes *trampoline_probe_handler(struct pt_regs *regs); + +#endif /* CONFIG_KPROBES */ #endif /* _ASM_RISCV_KPROBES_H */ diff --git a/arch/riscv/include/asm/mmu.h b/arch/riscv/include/asm/mmu.h index dabcf2cfb3dc..0099dc116168 100644 --- a/arch/riscv/include/asm/mmu.h +++ b/arch/riscv/include/asm/mmu.h @@ -12,6 +12,8 @@ typedef struct { #ifndef CONFIG_MMU unsigned long end_brk; +#else + atomic_long_t id; #endif void *vdso; #ifdef CONFIG_SMP diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h index 250defa06f3a..b0659413a080 100644 --- a/arch/riscv/include/asm/mmu_context.h +++ b/arch/riscv/include/asm/mmu_context.h @@ -23,6 +23,16 @@ static inline void activate_mm(struct mm_struct *prev, switch_mm(prev, next, NULL); } +#define init_new_context init_new_context +static inline int init_new_context(struct task_struct *tsk, + struct mm_struct *mm) +{ +#ifdef CONFIG_MMU + atomic_long_set(&mm->context.id, 0); +#endif + return 0; +} + #include #endif /* _ASM_RISCV_MMU_CONTEXT_H */ diff --git a/arch/riscv/include/asm/mmzone.h b/arch/riscv/include/asm/mmzone.h new file mode 100644 index 000000000000..fa17e01d9ab2 --- /dev/null +++ b/arch/riscv/include/asm/mmzone.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MMZONE_H +#define __ASM_MMZONE_H + +#ifdef CONFIG_NUMA + +#include + +extern struct pglist_data *node_data[]; +#define NODE_DATA(nid) (node_data[(nid)]) + +#endif /* CONFIG_NUMA */ +#endif /* __ASM_MMZONE_H */ diff --git a/arch/riscv/include/asm/numa.h b/arch/riscv/include/asm/numa.h new file mode 100644 index 000000000000..8c8cf4297cc3 --- /dev/null +++ b/arch/riscv/include/asm/numa.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_NUMA_H +#define __ASM_NUMA_H + +#include +#include + +#endif /* __ASM_NUMA_H */ diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 64a675c5c30a..adc9d26f3d75 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -97,9 +97,6 @@ extern unsigned long pfn_base; #define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) #endif /* CONFIG_MMU */ -extern unsigned long max_low_pfn; -extern unsigned long min_low_pfn; - #define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset)) #define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset) diff --git a/arch/riscv/include/asm/pci.h b/arch/riscv/include/asm/pci.h index 1c473a1bd986..658e112c3ce7 100644 --- a/arch/riscv/include/asm/pci.h +++ b/arch/riscv/include/asm/pci.h @@ -32,6 +32,20 @@ static inline int pci_proc_domain(struct pci_bus *bus) /* always show the domain in /proc */ return 1; } + +#ifdef CONFIG_NUMA + +static inline int pcibus_to_node(struct pci_bus *bus) +{ + return dev_to_node(&bus->dev); +} +#ifndef cpumask_of_pcibus +#define cpumask_of_pcibus(bus) (pcibus_to_node(bus) == -1 ? \ + cpu_all_mask : \ + cpumask_of_node(pcibus_to_node(bus))) +#endif +#endif /* CONFIG_NUMA */ + #endif /* CONFIG_PCI */ #endif /* _ASM_RISCV_PCI_H */ diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 251e1db088fa..ebf817c1bdf4 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -186,6 +186,11 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) return (unsigned long)pfn_to_virt(pmd_val(pmd) >> _PAGE_PFN_SHIFT); } +static inline pte_t pmd_pte(pmd_t pmd) +{ + return __pte(pmd_val(pmd)); +} + /* Yields the page frame number (PFN) of a page table entry */ static inline unsigned long pte_pfn(pte_t pte) { @@ -289,6 +294,21 @@ static inline pte_t pte_mkhuge(pte_t pte) return pte; } +#ifdef CONFIG_NUMA_BALANCING +/* + * See the comment in include/asm-generic/pgtable.h + */ +static inline int pte_protnone(pte_t pte) +{ + return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) == _PAGE_PROT_NONE; +} + +static inline int pmd_protnone(pmd_t pmd) +{ + return pte_protnone(pmd_pte(pmd)); +} +#endif + /* Modify page protection bits */ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { @@ -468,6 +488,7 @@ extern void *dtb_early_va; extern uintptr_t dtb_early_pa; void setup_bootmem(void); void paging_init(void); +void misc_mem_init(void); #define FIRST_USER_ADDRESS 0 diff --git a/arch/riscv/include/asm/probes.h b/arch/riscv/include/asm/probes.h new file mode 100644 index 000000000000..a787e6d537b9 --- /dev/null +++ b/arch/riscv/include/asm/probes.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_PROBES_H +#define _ASM_RISCV_PROBES_H + +typedef u32 probe_opcode_t; +typedef bool (probes_handler_t) (u32 opcode, unsigned long addr, struct pt_regs *); + +/* architecture specific copy of original instruction */ +struct arch_probe_insn { + probe_opcode_t *insn; + probes_handler_t *handler; + /* restore address after simulation */ + unsigned long restore; +}; + +#ifdef CONFIG_KPROBES +typedef u32 kprobe_opcode_t; +struct arch_specific_insn { + struct arch_probe_insn api; +}; +#endif + +#endif /* _ASM_RISCV_PROBES_H */ diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index bdddcd5c1b71..3a240037bde2 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -34,6 +34,7 @@ struct thread_struct { unsigned long sp; /* Kernel mode stack */ unsigned long s[12]; /* s[0]: frame pointer */ struct __riscv_d_ext_state fstate; + unsigned long bad_cause; }; #define INIT_THREAD { \ diff --git a/arch/riscv/include/asm/ptrace.h b/arch/riscv/include/asm/ptrace.h index ee49f80c9533..cb4abb639e8d 100644 --- a/arch/riscv/include/asm/ptrace.h +++ b/arch/riscv/include/asm/ptrace.h @@ -8,6 +8,7 @@ #include #include +#include #ifndef __ASSEMBLY__ @@ -60,6 +61,7 @@ struct pt_regs { #define user_mode(regs) (((regs)->status & SR_PP) == 0) +#define MAX_REG_OFFSET offsetof(struct pt_regs, orig_a0) /* Helpers for working with the instruction pointer */ static inline unsigned long instruction_pointer(struct pt_regs *regs) @@ -85,6 +87,12 @@ static inline void user_stack_pointer_set(struct pt_regs *regs, regs->sp = val; } +/* Valid only for Kernel mode traps. */ +static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +{ + return regs->sp; +} + /* Helpers for working with the frame pointer */ static inline unsigned long frame_pointer(struct pt_regs *regs) { @@ -101,6 +109,33 @@ static inline unsigned long regs_return_value(struct pt_regs *regs) return regs->a0; } +static inline void regs_set_return_value(struct pt_regs *regs, + unsigned long val) +{ + regs->a0 = val; +} + +extern int regs_query_register_offset(const char *name); +extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, + unsigned int n); + +/** + * regs_get_register() - get register value from its offset + * @regs: pt_regs from which register value is gotten + * @offset: offset of the register. + * + * regs_get_register returns the value of a register whose offset from @regs. + * The @offset is the offset of the register in struct pt_regs. + * If @offset is bigger than MAX_REG_OFFSET, this returns 0. + */ +static inline unsigned long regs_get_register(struct pt_regs *regs, + unsigned int offset) +{ + if (unlikely(offset > MAX_REG_OFFSET)) + return 0; + + return *(unsigned long *)((unsigned long)regs + offset); +} #endif /* __ASSEMBLY__ */ #endif /* _ASM_RISCV_PTRACE_H */ diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 653edb25d495..99895d9c3bdd 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -89,7 +89,7 @@ struct sbiret { long value; }; -int sbi_init(void); +void sbi_init(void); struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4, @@ -100,13 +100,13 @@ int sbi_console_getchar(void); void sbi_set_timer(uint64_t stime_value); void sbi_shutdown(void); void sbi_clear_ipi(void); -void sbi_send_ipi(const unsigned long *hart_mask); -void sbi_remote_fence_i(const unsigned long *hart_mask); -void sbi_remote_sfence_vma(const unsigned long *hart_mask, +int sbi_send_ipi(const unsigned long *hart_mask); +int sbi_remote_fence_i(const unsigned long *hart_mask); +int sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size); -void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, +int sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, unsigned long size, unsigned long asid); @@ -147,11 +147,7 @@ static inline unsigned long sbi_minor_version(void) int sbi_err_map_linux_errno(int err); #else /* CONFIG_RISCV_SBI */ -/* stubs for code that is only reachable under IS_ENABLED(CONFIG_RISCV_SBI): */ -void sbi_set_timer(uint64_t stime_value); -void sbi_clear_ipi(void); -void sbi_send_ipi(const unsigned long *hart_mask); -void sbi_remote_fence_i(const unsigned long *hart_mask); -void sbi_init(void); +static inline int sbi_remote_fence_i(const unsigned long *hart_mask) { return -1; } +static inline void sbi_init(void) {} #endif /* CONFIG_RISCV_SBI */ #endif /* _ASM_RISCV_SBI_H */ diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index 8b80c80c7f1a..6887b3d9f371 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -22,7 +22,7 @@ static inline int set_memory_ro(unsigned long addr, int numpages) { return 0; } static inline int set_memory_rw(unsigned long addr, int numpages) { return 0; } static inline int set_memory_x(unsigned long addr, int numpages) { return 0; } static inline int set_memory_nx(unsigned long addr, int numpages) { return 0; } -static inline void protect_kernel_text_data(void) {}; +static inline void protect_kernel_text_data(void) {} static inline int set_memory_rw_nx(unsigned long addr, int numpages) { return 0; } #endif diff --git a/arch/riscv/include/asm/soc.h b/arch/riscv/include/asm/soc.h index 6c8363b1f327..f494066051a2 100644 --- a/arch/riscv/include/asm/soc.h +++ b/arch/riscv/include/asm/soc.h @@ -21,42 +21,4 @@ void soc_early_init(void); extern unsigned long __soc_early_init_table_start; extern unsigned long __soc_early_init_table_end; -/* - * Allows Linux to provide a device tree, which is necessary for SOCs that - * don't provide a useful one on their own. - */ -struct soc_builtin_dtb { - unsigned long vendor_id; - unsigned long arch_id; - unsigned long imp_id; - void *(*dtb_func)(void); -}; - -/* - * The argument name must specify a valid DTS file name without the dts - * extension. - */ -#define SOC_BUILTIN_DTB_DECLARE(name, vendor, arch, impl) \ - extern void *__dtb_##name##_begin; \ - \ - static __init __used \ - void *__soc_builtin_dtb_f__##name(void) \ - { \ - return (void *)&__dtb_##name##_begin; \ - } \ - \ - static const struct soc_builtin_dtb __soc_builtin_dtb__##name \ - __used __section("__soc_builtin_dtb_table") = \ - { \ - .vendor_id = vendor, \ - .arch_id = arch, \ - .imp_id = impl, \ - .dtb_func = __soc_builtin_dtb_f__##name, \ - } - -extern unsigned long __soc_builtin_dtb_table_start; -extern unsigned long __soc_builtin_dtb_table_end; - -void *soc_lookup_builtin_dtb(void); - #endif diff --git a/arch/riscv/include/asm/stackprotector.h b/arch/riscv/include/asm/stackprotector.h index 5962f8891f06..09093af46565 100644 --- a/arch/riscv/include/asm/stackprotector.h +++ b/arch/riscv/include/asm/stackprotector.h @@ -24,6 +24,7 @@ static __always_inline void boot_init_stack_canary(void) canary &= CANARY_MASK; current->stack_canary = canary; - __stack_chk_guard = current->stack_canary; + if (!IS_ENABLED(CONFIG_STACKPROTECTOR_PER_TASK)) + __stack_chk_guard = current->stack_canary; } #endif /* _ASM_RISCV_STACKPROTECTOR_H */ diff --git a/arch/riscv/include/asm/stacktrace.h b/arch/riscv/include/asm/stacktrace.h index 470a65c4ccdc..3450c1912afd 100644 --- a/arch/riscv/include/asm/stacktrace.h +++ b/arch/riscv/include/asm/stacktrace.h @@ -13,5 +13,7 @@ struct stackframe { extern void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, bool (*fn)(void *, unsigned long), void *arg); +extern void dump_backtrace(struct pt_regs *regs, struct task_struct *task, + const char *loglvl); #endif /* _ASM_RISCV_STACKTRACE_H */ diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 97bf5a1575d2..0e549a3089b3 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -75,6 +75,7 @@ struct thread_info { #define TIF_SYSCALL_AUDIT 7 /* syscall auditing */ #define TIF_SECCOMP 8 /* syscall secure computing */ #define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */ +#define TIF_UPROBE 10 /* uprobe breakpoint or singlestep */ #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) @@ -84,10 +85,11 @@ struct thread_info { #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_SECCOMP (1 << TIF_SECCOMP) #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) +#define _TIF_UPROBE (1 << TIF_UPROBE) #define _TIF_WORK_MASK \ (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \ - _TIF_NOTIFY_SIGNAL) + _TIF_NOTIFY_SIGNAL | _TIF_UPROBE) #define _TIF_SYSCALL_WORK \ (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \ diff --git a/arch/riscv/include/asm/uprobes.h b/arch/riscv/include/asm/uprobes.h new file mode 100644 index 000000000000..f2183e00fdd2 --- /dev/null +++ b/arch/riscv/include/asm/uprobes.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ASM_RISCV_UPROBES_H +#define _ASM_RISCV_UPROBES_H + +#include +#include +#include + +#define MAX_UINSN_BYTES 8 + +#ifdef CONFIG_RISCV_ISA_C +#define UPROBE_SWBP_INSN __BUG_INSN_16 +#define UPROBE_SWBP_INSN_SIZE 2 +#else +#define UPROBE_SWBP_INSN __BUG_INSN_32 +#define UPROBE_SWBP_INSN_SIZE 4 +#endif +#define UPROBE_XOL_SLOT_BYTES MAX_UINSN_BYTES + +typedef u32 uprobe_opcode_t; + +struct arch_uprobe_task { + unsigned long saved_cause; +}; + +struct arch_uprobe { + union { + u8 insn[MAX_UINSN_BYTES]; + u8 ixol[MAX_UINSN_BYTES]; + }; + struct arch_probe_insn api; + unsigned long insn_size; + bool simulate; +}; + +bool uprobe_breakpoint_handler(struct pt_regs *regs); +bool uprobe_single_step_handler(struct pt_regs *regs); + +#endif /* _ASM_RISCV_UPROBES_H */ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index f6caf4d9ca15..3dc0abde988a 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -4,8 +4,9 @@ # ifdef CONFIG_FTRACE -CFLAGS_REMOVE_ftrace.o = -pg -CFLAGS_REMOVE_patch.o = -pg +CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_patch.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_sbi.o = $(CC_FLAGS_FTRACE) endif extra-y += head.o @@ -29,6 +30,7 @@ obj-y += riscv_ksyms.o obj-y += stacktrace.o obj-y += cacheinfo.o obj-y += patch.o +obj-y += probes/ obj-$(CONFIG_MMU) += vdso.o vdso/ obj-$(CONFIG_RISCV_M_MODE) += traps_misaligned.o diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index b79ffa3561fd..9ef33346853c 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -68,6 +68,9 @@ void asm_offsets(void) OFFSET(TASK_THREAD_F30, task_struct, thread.fstate.f[30]); OFFSET(TASK_THREAD_F31, task_struct, thread.fstate.f[31]); OFFSET(TASK_THREAD_FCSR, task_struct, thread.fstate.fcsr); +#ifdef CONFIG_STACKPROTECTOR + OFFSET(TSK_STACK_CANARY, task_struct, stack_canary); +#endif DEFINE(PT_SIZE, sizeof(struct pt_regs)); OFFSET(PT_EPC, pt_regs, epc); diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 765b62434f30..7f1e5203de88 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -72,29 +72,56 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target, return 0; } +/* + * Put 5 instructions with 16 bytes at the front of function within + * patchable function entry nops' area. + * + * 0: REG_S ra, -SZREG(sp) + * 1: auipc ra, 0x? + * 2: jalr -?(ra) + * 3: REG_L ra, -SZREG(sp) + * + * So the opcodes is: + * 0: 0xfe113c23 (sd)/0xfe112e23 (sw) + * 1: 0x???????? -> auipc + * 2: 0x???????? -> jalr + * 3: 0xff813083 (ld)/0xffc12083 (lw) + */ +#if __riscv_xlen == 64 +#define INSN0 0xfe113c23 +#define INSN3 0xff813083 +#elif __riscv_xlen == 32 +#define INSN0 0xfe112e23 +#define INSN3 0xffc12083 +#endif + +#define FUNC_ENTRY_SIZE 16 +#define FUNC_ENTRY_JMP 4 + int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { - int ret = ftrace_check_current_call(rec->ip, NULL); + unsigned int call[4] = {INSN0, 0, 0, INSN3}; + unsigned long target = addr; + unsigned long caller = rec->ip + FUNC_ENTRY_JMP; - if (ret) - return ret; + call[1] = to_auipc_insn((unsigned int)(target - caller)); + call[2] = to_jalr_insn((unsigned int)(target - caller)); - return __ftrace_modify_call(rec->ip, addr, true); + if (patch_text_nosync((void *)rec->ip, call, FUNC_ENTRY_SIZE)) + return -EPERM; + + return 0; } int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) { - unsigned int call[2]; - int ret; + unsigned int nops[4] = {NOP4, NOP4, NOP4, NOP4}; - make_call(rec->ip, addr, call); - ret = ftrace_check_current_call(rec->ip, call); + if (patch_text_nosync((void *)rec->ip, nops, FUNC_ENTRY_SIZE)) + return -EPERM; - if (ret) - return ret; - - return __ftrace_modify_call(rec->ip, addr, false); + return 0; } @@ -139,15 +166,16 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { unsigned int call[2]; + unsigned long caller = rec->ip + FUNC_ENTRY_JMP; int ret; - make_call(rec->ip, old_addr, call); - ret = ftrace_check_current_call(rec->ip, call); + make_call(caller, old_addr, call); + ret = ftrace_check_current_call(caller, call); if (ret) return ret; - return __ftrace_modify_call(rec->ip, addr, true); + return __ftrace_modify_call(caller, addr, true); } #endif @@ -176,53 +204,30 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, #ifdef CONFIG_DYNAMIC_FTRACE extern void ftrace_graph_call(void); +extern void ftrace_graph_regs_call(void); int ftrace_enable_ftrace_graph_caller(void) { - unsigned int call[2]; - static int init_graph = 1; int ret; - make_call(&ftrace_graph_call, &ftrace_stub, call); - - /* - * When enabling graph tracer for the first time, ftrace_graph_call - * should contains a call to ftrace_stub. Once it has been disabled, - * the 8-bytes at the position becomes NOPs. - */ - if (init_graph) { - ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, - call); - init_graph = 0; - } else { - ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, - NULL); - } - + ret = __ftrace_modify_call((unsigned long)&ftrace_graph_call, + (unsigned long)&prepare_ftrace_return, true); if (ret) return ret; - return __ftrace_modify_call((unsigned long)&ftrace_graph_call, + return __ftrace_modify_call((unsigned long)&ftrace_graph_regs_call, (unsigned long)&prepare_ftrace_return, true); } int ftrace_disable_ftrace_graph_caller(void) { - unsigned int call[2]; int ret; - make_call(&ftrace_graph_call, &prepare_ftrace_return, call); - - /* - * This is to make sure that ftrace_enable_ftrace_graph_caller - * did the right thing. - */ - ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, - call); - + ret = __ftrace_modify_call((unsigned long)&ftrace_graph_call, + (unsigned long)&prepare_ftrace_return, false); if (ret) return ret; - return __ftrace_modify_call((unsigned long)&ftrace_graph_call, + return __ftrace_modify_call((unsigned long)&ftrace_graph_regs_call, (unsigned long)&prepare_ftrace_return, false); } #endif /* CONFIG_DYNAMIC_FTRACE */ diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 16e9941900c4..f5a9bad86e58 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -260,7 +260,11 @@ clear_bss_done: /* Initialize page tables and relocate to virtual addresses */ la sp, init_thread_union + THREAD_SIZE +#ifdef CONFIG_BUILTIN_DTB + la a0, __dtb_start +#else mv a0, s1 +#endif /* CONFIG_BUILTIN_DTB */ call setup_vm #ifdef CONFIG_MMU la a0, early_pg_dir diff --git a/arch/riscv/kernel/image-vars.h b/arch/riscv/kernel/image-vars.h index 8c212efb37a6..71a76a623257 100644 --- a/arch/riscv/kernel/image-vars.h +++ b/arch/riscv/kernel/image-vars.h @@ -3,7 +3,7 @@ * Copyright (C) 2020 Western Digital Corporation or its affiliates. * Linker script variables to be set after section resolution, as * ld.lld does not like variables assigned before SECTIONS is processed. - * Based on arch/arm64/kerne/image-vars.h + * Based on arch/arm64/kernel/image-vars.h */ #ifndef __RISCV_KERNEL_IMAGE_VARS_H #define __RISCV_KERNEL_IMAGE_VARS_H diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S index 35a6ed76cb8b..d171eca623b6 100644 --- a/arch/riscv/kernel/mcount-dyn.S +++ b/arch/riscv/kernel/mcount-dyn.S @@ -13,224 +13,186 @@ .text - .macro SAVE_ABI_STATE -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - addi sp, sp, -48 - sd s0, 32(sp) - sd ra, 40(sp) - addi s0, sp, 48 - sd t0, 24(sp) - sd t1, 16(sp) -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - sd t2, 8(sp) -#endif -#else - addi sp, sp, -16 - sd s0, 0(sp) - sd ra, 8(sp) - addi s0, sp, 16 -#endif +#define FENTRY_RA_OFFSET 12 +#define ABI_SIZE_ON_STACK 72 +#define ABI_A0 0 +#define ABI_A1 8 +#define ABI_A2 16 +#define ABI_A3 24 +#define ABI_A4 32 +#define ABI_A5 40 +#define ABI_A6 48 +#define ABI_A7 56 +#define ABI_RA 64 + + .macro SAVE_ABI + addi sp, sp, -SZREG + addi sp, sp, -ABI_SIZE_ON_STACK + + REG_S a0, ABI_A0(sp) + REG_S a1, ABI_A1(sp) + REG_S a2, ABI_A2(sp) + REG_S a3, ABI_A3(sp) + REG_S a4, ABI_A4(sp) + REG_S a5, ABI_A5(sp) + REG_S a6, ABI_A6(sp) + REG_S a7, ABI_A7(sp) + REG_S ra, ABI_RA(sp) .endm - .macro RESTORE_ABI_STATE -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - ld s0, 32(sp) - ld ra, 40(sp) - addi sp, sp, 48 -#else - ld ra, 8(sp) - ld s0, 0(sp) - addi sp, sp, 16 -#endif + .macro RESTORE_ABI + REG_L a0, ABI_A0(sp) + REG_L a1, ABI_A1(sp) + REG_L a2, ABI_A2(sp) + REG_L a3, ABI_A3(sp) + REG_L a4, ABI_A4(sp) + REG_L a5, ABI_A5(sp) + REG_L a6, ABI_A6(sp) + REG_L a7, ABI_A7(sp) + REG_L ra, ABI_RA(sp) + + addi sp, sp, ABI_SIZE_ON_STACK + addi sp, sp, SZREG .endm - .macro RESTORE_GRAPH_ARGS - ld a0, 24(sp) - ld a1, 16(sp) -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - ld a2, 8(sp) -#endif +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + .macro SAVE_ALL + addi sp, sp, -SZREG + addi sp, sp, -PT_SIZE_ON_STACK + + REG_S x1, PT_EPC(sp) + addi sp, sp, PT_SIZE_ON_STACK + REG_L x1, (sp) + addi sp, sp, -PT_SIZE_ON_STACK + REG_S x1, PT_RA(sp) + REG_L x1, PT_EPC(sp) + + REG_S x2, PT_SP(sp) + REG_S x3, PT_GP(sp) + REG_S x4, PT_TP(sp) + REG_S x5, PT_T0(sp) + REG_S x6, PT_T1(sp) + REG_S x7, PT_T2(sp) + REG_S x8, PT_S0(sp) + REG_S x9, PT_S1(sp) + REG_S x10, PT_A0(sp) + REG_S x11, PT_A1(sp) + REG_S x12, PT_A2(sp) + REG_S x13, PT_A3(sp) + REG_S x14, PT_A4(sp) + REG_S x15, PT_A5(sp) + REG_S x16, PT_A6(sp) + REG_S x17, PT_A7(sp) + REG_S x18, PT_S2(sp) + REG_S x19, PT_S3(sp) + REG_S x20, PT_S4(sp) + REG_S x21, PT_S5(sp) + REG_S x22, PT_S6(sp) + REG_S x23, PT_S7(sp) + REG_S x24, PT_S8(sp) + REG_S x25, PT_S9(sp) + REG_S x26, PT_S10(sp) + REG_S x27, PT_S11(sp) + REG_S x28, PT_T3(sp) + REG_S x29, PT_T4(sp) + REG_S x30, PT_T5(sp) + REG_S x31, PT_T6(sp) .endm -ENTRY(ftrace_graph_caller) - addi sp, sp, -16 - sd s0, 0(sp) - sd ra, 8(sp) - addi s0, sp, 16 -ftrace_graph_call: - .global ftrace_graph_call - /* - * Calling ftrace_enable/disable_ftrace_graph_caller would overwrite the - * call below. Check ftrace_modify_all_code for details. - */ - call ftrace_stub - ld ra, 8(sp) - ld s0, 0(sp) - addi sp, sp, 16 - ret -ENDPROC(ftrace_graph_caller) + .macro RESTORE_ALL + REG_L x1, PT_RA(sp) + addi sp, sp, PT_SIZE_ON_STACK + REG_S x1, (sp) + addi sp, sp, -PT_SIZE_ON_STACK + REG_L x1, PT_EPC(sp) + REG_L x2, PT_SP(sp) + REG_L x3, PT_GP(sp) + REG_L x4, PT_TP(sp) + REG_L x5, PT_T0(sp) + REG_L x6, PT_T1(sp) + REG_L x7, PT_T2(sp) + REG_L x8, PT_S0(sp) + REG_L x9, PT_S1(sp) + REG_L x10, PT_A0(sp) + REG_L x11, PT_A1(sp) + REG_L x12, PT_A2(sp) + REG_L x13, PT_A3(sp) + REG_L x14, PT_A4(sp) + REG_L x15, PT_A5(sp) + REG_L x16, PT_A6(sp) + REG_L x17, PT_A7(sp) + REG_L x18, PT_S2(sp) + REG_L x19, PT_S3(sp) + REG_L x20, PT_S4(sp) + REG_L x21, PT_S5(sp) + REG_L x22, PT_S6(sp) + REG_L x23, PT_S7(sp) + REG_L x24, PT_S8(sp) + REG_L x25, PT_S9(sp) + REG_L x26, PT_S10(sp) + REG_L x27, PT_S11(sp) + REG_L x28, PT_T3(sp) + REG_L x29, PT_T4(sp) + REG_L x30, PT_T5(sp) + REG_L x31, PT_T6(sp) + + addi sp, sp, PT_SIZE_ON_STACK + addi sp, sp, SZREG + .endm +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ ENTRY(ftrace_caller) - /* - * a0: the address in the caller when calling ftrace_caller - * a1: the caller's return address - * a2: the address of global variable function_trace_op - */ - ld a1, -8(s0) - addi a0, ra, -MCOUNT_INSN_SIZE - la t5, function_trace_op - ld a2, 0(t5) + SAVE_ABI -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - /* - * the graph tracer (specifically, prepare_ftrace_return) needs these - * arguments but for now the function tracer occupies the regs, so we - * save them in temporary regs to recover later. - */ - addi t0, s0, -8 - mv t1, a0 -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - ld t2, -16(s0) -#endif -#endif + addi a0, ra, -FENTRY_RA_OFFSET + la a1, function_trace_op + REG_L a2, 0(a1) + REG_L a1, ABI_SIZE_ON_STACK(sp) + mv a3, sp - SAVE_ABI_STATE ftrace_call: .global ftrace_call - /* - * For the dynamic ftrace to work, here we should reserve at least - * 8 bytes for a functional auipc-jalr pair. The following call - * serves this purpose. - * - * Calling ftrace_update_ftrace_func would overwrite the nops below. - * Check ftrace_modify_all_code for details. - */ call ftrace_stub #ifdef CONFIG_FUNCTION_GRAPH_TRACER - RESTORE_GRAPH_ARGS - call ftrace_graph_caller + addi a0, sp, ABI_SIZE_ON_STACK + REG_L a1, ABI_RA(sp) + addi a1, a1, -FENTRY_RA_OFFSET +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + mv a2, s0 #endif - - RESTORE_ABI_STATE +ftrace_graph_call: + .global ftrace_graph_call + call ftrace_stub +#endif + RESTORE_ABI ret ENDPROC(ftrace_caller) #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS - .macro SAVE_ALL - addi sp, sp, -(PT_SIZE_ON_STACK+16) - sd s0, (PT_SIZE_ON_STACK)(sp) - sd ra, (PT_SIZE_ON_STACK+8)(sp) - addi s0, sp, (PT_SIZE_ON_STACK+16) - - sd x1, PT_RA(sp) - sd x2, PT_SP(sp) - sd x3, PT_GP(sp) - sd x4, PT_TP(sp) - sd x5, PT_T0(sp) - sd x6, PT_T1(sp) - sd x7, PT_T2(sp) - sd x8, PT_S0(sp) - sd x9, PT_S1(sp) - sd x10, PT_A0(sp) - sd x11, PT_A1(sp) - sd x12, PT_A2(sp) - sd x13, PT_A3(sp) - sd x14, PT_A4(sp) - sd x15, PT_A5(sp) - sd x16, PT_A6(sp) - sd x17, PT_A7(sp) - sd x18, PT_S2(sp) - sd x19, PT_S3(sp) - sd x20, PT_S4(sp) - sd x21, PT_S5(sp) - sd x22, PT_S6(sp) - sd x23, PT_S7(sp) - sd x24, PT_S8(sp) - sd x25, PT_S9(sp) - sd x26, PT_S10(sp) - sd x27, PT_S11(sp) - sd x28, PT_T3(sp) - sd x29, PT_T4(sp) - sd x30, PT_T5(sp) - sd x31, PT_T6(sp) - .endm - - .macro RESTORE_ALL - ld x1, PT_RA(sp) - ld x2, PT_SP(sp) - ld x3, PT_GP(sp) - ld x4, PT_TP(sp) - ld x5, PT_T0(sp) - ld x6, PT_T1(sp) - ld x7, PT_T2(sp) - ld x8, PT_S0(sp) - ld x9, PT_S1(sp) - ld x10, PT_A0(sp) - ld x11, PT_A1(sp) - ld x12, PT_A2(sp) - ld x13, PT_A3(sp) - ld x14, PT_A4(sp) - ld x15, PT_A5(sp) - ld x16, PT_A6(sp) - ld x17, PT_A7(sp) - ld x18, PT_S2(sp) - ld x19, PT_S3(sp) - ld x20, PT_S4(sp) - ld x21, PT_S5(sp) - ld x22, PT_S6(sp) - ld x23, PT_S7(sp) - ld x24, PT_S8(sp) - ld x25, PT_S9(sp) - ld x26, PT_S10(sp) - ld x27, PT_S11(sp) - ld x28, PT_T3(sp) - ld x29, PT_T4(sp) - ld x30, PT_T5(sp) - ld x31, PT_T6(sp) - - ld s0, (PT_SIZE_ON_STACK)(sp) - ld ra, (PT_SIZE_ON_STACK+8)(sp) - addi sp, sp, (PT_SIZE_ON_STACK+16) - .endm - - .macro RESTORE_GRAPH_REG_ARGS - ld a0, PT_T0(sp) - ld a1, PT_T1(sp) -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - ld a2, PT_T2(sp) -#endif - .endm - -/* - * Most of the contents are the same as ftrace_caller. - */ ENTRY(ftrace_regs_caller) - /* - * a3: the address of all registers in the stack - */ - ld a1, -8(s0) - addi a0, ra, -MCOUNT_INSN_SIZE - la t5, function_trace_op - ld a2, 0(t5) - addi a3, sp, -(PT_SIZE_ON_STACK+16) - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - addi t0, s0, -8 - mv t1, a0 -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - ld t2, -16(s0) -#endif -#endif SAVE_ALL + addi a0, ra, -FENTRY_RA_OFFSET + la a1, function_trace_op + REG_L a2, 0(a1) + REG_L a1, PT_SIZE_ON_STACK(sp) + mv a3, sp + ftrace_regs_call: .global ftrace_regs_call call ftrace_stub #ifdef CONFIG_FUNCTION_GRAPH_TRACER - RESTORE_GRAPH_REG_ARGS - call ftrace_graph_caller + addi a0, sp, PT_RA + REG_L a1, PT_EPC(sp) + addi a1, a1, -FENTRY_RA_OFFSET +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + mv a2, s0 +#endif +ftrace_graph_regs_call: + .global ftrace_graph_regs_call + call ftrace_stub #endif RESTORE_ALL diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index 3fe7a5296aa5..0b552873a577 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -20,7 +20,12 @@ struct patch_insn { }; #ifdef CONFIG_MMU -static void *patch_map(void *addr, int fixmap) +/* + * The fix_to_virt(, idx) needs a const value (not a dynamic variable of + * reg-a0) or BUILD_BUG_ON failed with "idx >= __end_of_fixed_addresses". + * So use '__always_inline' and 'const unsigned int fixmap' here. + */ +static __always_inline void *patch_map(void *addr, const unsigned int fixmap) { uintptr_t uintaddr = (uintptr_t) addr; struct page *page; @@ -37,7 +42,6 @@ static void *patch_map(void *addr, int fixmap) return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + (uintaddr & ~PAGE_MASK)); } -NOKPROBE_SYMBOL(patch_map); static void patch_unmap(int fixmap) { diff --git a/arch/riscv/kernel/probes/Makefile b/arch/riscv/kernel/probes/Makefile new file mode 100644 index 000000000000..7f0840dcc31b --- /dev/null +++ b/arch/riscv/kernel/probes/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_KPROBES) += kprobes.o decode-insn.o simulate-insn.o +obj-$(CONFIG_KPROBES) += kprobes_trampoline.o +obj-$(CONFIG_KPROBES_ON_FTRACE) += ftrace.o +obj-$(CONFIG_UPROBES) += uprobes.o decode-insn.o simulate-insn.o +CFLAGS_REMOVE_simulate-insn.o = $(CC_FLAGS_FTRACE) diff --git a/arch/riscv/kernel/probes/decode-insn.c b/arch/riscv/kernel/probes/decode-insn.c new file mode 100644 index 000000000000..0ed043acc882 --- /dev/null +++ b/arch/riscv/kernel/probes/decode-insn.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include +#include +#include +#include +#include + +#include "decode-insn.h" +#include "simulate-insn.h" + +/* Return: + * INSN_REJECTED If instruction is one not allowed to kprobe, + * INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot. + */ +enum probe_insn __kprobes +riscv_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *api) +{ + probe_opcode_t insn = *addr; + + /* + * Reject instructions list: + */ + RISCV_INSN_REJECTED(system, insn); + RISCV_INSN_REJECTED(fence, insn); + + /* + * Simulate instructions list: + * TODO: the REJECTED ones below need to be implemented + */ +#ifdef CONFIG_RISCV_ISA_C + RISCV_INSN_REJECTED(c_j, insn); + RISCV_INSN_REJECTED(c_jr, insn); + RISCV_INSN_REJECTED(c_jal, insn); + RISCV_INSN_REJECTED(c_jalr, insn); + RISCV_INSN_REJECTED(c_beqz, insn); + RISCV_INSN_REJECTED(c_bnez, insn); + RISCV_INSN_REJECTED(c_ebreak, insn); +#endif + + RISCV_INSN_REJECTED(auipc, insn); + RISCV_INSN_REJECTED(branch, insn); + + RISCV_INSN_SET_SIMULATE(jal, insn); + RISCV_INSN_SET_SIMULATE(jalr, insn); + + return INSN_GOOD; +} diff --git a/arch/riscv/kernel/probes/decode-insn.h b/arch/riscv/kernel/probes/decode-insn.h new file mode 100644 index 000000000000..42269a7d676d --- /dev/null +++ b/arch/riscv/kernel/probes/decode-insn.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef _RISCV_KERNEL_KPROBES_DECODE_INSN_H +#define _RISCV_KERNEL_KPROBES_DECODE_INSN_H + +#include +#include + +enum probe_insn { + INSN_REJECTED, + INSN_GOOD_NO_SLOT, + INSN_GOOD, +}; + +enum probe_insn __kprobes +riscv_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *asi); + +#endif /* _RISCV_KERNEL_KPROBES_DECODE_INSN_H */ diff --git a/arch/riscv/kernel/probes/ftrace.c b/arch/riscv/kernel/probes/ftrace.c new file mode 100644 index 000000000000..e6372490aa0b --- /dev/null +++ b/arch/riscv/kernel/probes/ftrace.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/* Ftrace callback handler for kprobes -- called under preepmt disabed */ +void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *ops, struct ftrace_regs *regs) +{ + struct kprobe *p; + struct kprobe_ctlblk *kcb; + + p = get_kprobe((kprobe_opcode_t *)ip); + if (unlikely(!p) || kprobe_disabled(p)) + return; + + kcb = get_kprobe_ctlblk(); + if (kprobe_running()) { + kprobes_inc_nmissed_count(p); + } else { + unsigned long orig_ip = instruction_pointer(&(regs->regs)); + + instruction_pointer_set(&(regs->regs), ip); + + __this_cpu_write(current_kprobe, p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + if (!p->pre_handler || !p->pre_handler(p, &(regs->regs))) { + /* + * Emulate singlestep (and also recover regs->pc) + * as if there is a nop + */ + instruction_pointer_set(&(regs->regs), + (unsigned long)p->addr + MCOUNT_INSN_SIZE); + if (unlikely(p->post_handler)) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + p->post_handler(p, &(regs->regs), 0); + } + instruction_pointer_set(&(regs->regs), orig_ip); + } + + /* + * If pre_handler returns !0, it changes regs->pc. We have to + * skip emulating post_handler. + */ + __this_cpu_write(current_kprobe, NULL); + } +} +NOKPROBE_SYMBOL(kprobe_ftrace_handler); + +int arch_prepare_kprobe_ftrace(struct kprobe *p) +{ + p->ainsn.api.insn = NULL; + return 0; +} diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c new file mode 100644 index 000000000000..a2ec18662fee --- /dev/null +++ b/arch/riscv/kernel/probes/kprobes.c @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "decode-insn.h" + +DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; +DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); + +static void __kprobes +post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *); + +static void __kprobes arch_prepare_ss_slot(struct kprobe *p) +{ + unsigned long offset = GET_INSN_LENGTH(p->opcode); + + p->ainsn.api.restore = (unsigned long)p->addr + offset; + + patch_text(p->ainsn.api.insn, p->opcode); + patch_text((void *)((unsigned long)(p->ainsn.api.insn) + offset), + __BUG_INSN_32); +} + +static void __kprobes arch_prepare_simulate(struct kprobe *p) +{ + p->ainsn.api.restore = 0; +} + +static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if (p->ainsn.api.handler) + p->ainsn.api.handler((u32)p->opcode, + (unsigned long)p->addr, regs); + + post_kprobe_handler(kcb, regs); +} + +int __kprobes arch_prepare_kprobe(struct kprobe *p) +{ + unsigned long probe_addr = (unsigned long)p->addr; + + if (probe_addr & 0x1) { + pr_warn("Address not aligned.\n"); + + return -EINVAL; + } + + /* copy instruction */ + p->opcode = *p->addr; + + /* decode instruction */ + switch (riscv_probe_decode_insn(p->addr, &p->ainsn.api)) { + case INSN_REJECTED: /* insn not supported */ + return -EINVAL; + + case INSN_GOOD_NO_SLOT: /* insn need simulation */ + p->ainsn.api.insn = NULL; + break; + + case INSN_GOOD: /* instruction uses slot */ + p->ainsn.api.insn = get_insn_slot(); + if (!p->ainsn.api.insn) + return -ENOMEM; + break; + } + + /* prepare the instruction */ + if (p->ainsn.api.insn) + arch_prepare_ss_slot(p); + else + arch_prepare_simulate(p); + + return 0; +} + +/* install breakpoint in text */ +void __kprobes arch_arm_kprobe(struct kprobe *p) +{ + if ((p->opcode & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) + patch_text(p->addr, __BUG_INSN_32); + else + patch_text(p->addr, __BUG_INSN_16); +} + +/* remove breakpoint from text */ +void __kprobes arch_disarm_kprobe(struct kprobe *p) +{ + patch_text(p->addr, p->opcode); +} + +void __kprobes arch_remove_kprobe(struct kprobe *p) +{ +} + +static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + kcb->prev_kprobe.kp = kprobe_running(); + kcb->prev_kprobe.status = kcb->kprobe_status; +} + +static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); + kcb->kprobe_status = kcb->prev_kprobe.status; +} + +static void __kprobes set_current_kprobe(struct kprobe *p) +{ + __this_cpu_write(current_kprobe, p); +} + +/* + * Interrupts need to be disabled before single-step mode is set, and not + * reenabled until after single-step mode ends. + * Without disabling interrupt on local CPU, there is a chance of + * interrupt occurrence in the period of exception return and start of + * out-of-line single-step, that result in wrongly single stepping + * into the interrupt handler. + */ +static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + kcb->saved_status = regs->status; + regs->status &= ~SR_SPIE; +} + +static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + regs->status = kcb->saved_status; +} + +static void __kprobes +set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr, struct kprobe *p) +{ + unsigned long offset = GET_INSN_LENGTH(p->opcode); + + kcb->ss_ctx.ss_pending = true; + kcb->ss_ctx.match_addr = addr + offset; +} + +static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb) +{ + kcb->ss_ctx.ss_pending = false; + kcb->ss_ctx.match_addr = 0; +} + +static void __kprobes setup_singlestep(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb, int reenter) +{ + unsigned long slot; + + if (reenter) { + save_previous_kprobe(kcb); + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_REENTER; + } else { + kcb->kprobe_status = KPROBE_HIT_SS; + } + + if (p->ainsn.api.insn) { + /* prepare for single stepping */ + slot = (unsigned long)p->ainsn.api.insn; + + set_ss_context(kcb, slot, p); /* mark pending ss */ + + /* IRQs and single stepping do not mix well. */ + kprobes_save_local_irqflag(kcb, regs); + + instruction_pointer_set(regs, slot); + } else { + /* insn simulation */ + arch_simulate_insn(p, regs); + } +} + +static int __kprobes reenter_kprobe(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + switch (kcb->kprobe_status) { + case KPROBE_HIT_SSDONE: + case KPROBE_HIT_ACTIVE: + kprobes_inc_nmissed_count(p); + setup_singlestep(p, regs, kcb, 1); + break; + case KPROBE_HIT_SS: + case KPROBE_REENTER: + pr_warn("Unrecoverable kprobe detected.\n"); + dump_kprobe(p); + BUG(); + break; + default: + WARN_ON(1); + return 0; + } + + return 1; +} + +static void __kprobes +post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs) +{ + struct kprobe *cur = kprobe_running(); + + if (!cur) + return; + + /* return addr restore if non-branching insn */ + if (cur->ainsn.api.restore != 0) + regs->epc = cur->ainsn.api.restore; + + /* restore back original saved kprobe variables and continue */ + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + return; + } + + /* call post handler */ + kcb->kprobe_status = KPROBE_HIT_SSDONE; + if (cur->post_handler) { + /* post_handler can hit breakpoint and single step + * again, so we enable D-flag for recursive exception. + */ + cur->post_handler(cur, regs, 0); + } + + reset_current_kprobe(); +} + +int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + switch (kcb->kprobe_status) { + case KPROBE_HIT_SS: + case KPROBE_REENTER: + /* + * We are here because the instruction being single + * stepped caused a page fault. We reset the current + * kprobe and the ip points back to the probe address + * and allow the page fault handler to continue as a + * normal page fault. + */ + regs->epc = (unsigned long) cur->addr; + if (!instruction_pointer(regs)) + BUG(); + + if (kcb->kprobe_status == KPROBE_REENTER) + restore_previous_kprobe(kcb); + else + reset_current_kprobe(); + + break; + case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SSDONE: + /* + * We increment the nmissed count for accounting, + * we can also use npre/npostfault count for accounting + * these specific fault cases. + */ + kprobes_inc_nmissed_count(cur); + + /* + * We come here because instructions in the pre/post + * handler caused the page_fault, this could happen + * if handler tries to access user space by + * copy_from_user(), get_user() etc. Let the + * user-specified handler try to fix it first. + */ + if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) + return 1; + + /* + * In case the user-specified fault handler returned + * zero, try to fix up. + */ + if (fixup_exception(regs)) + return 1; + } + return 0; +} + +bool __kprobes +kprobe_breakpoint_handler(struct pt_regs *regs) +{ + struct kprobe *p, *cur_kprobe; + struct kprobe_ctlblk *kcb; + unsigned long addr = instruction_pointer(regs); + + kcb = get_kprobe_ctlblk(); + cur_kprobe = kprobe_running(); + + p = get_kprobe((kprobe_opcode_t *) addr); + + if (p) { + if (cur_kprobe) { + if (reenter_kprobe(p, regs, kcb)) + return true; + } else { + /* Probe hit */ + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + + /* + * If we have no pre-handler or it returned 0, we + * continue with normal processing. If we have a + * pre-handler and it returned non-zero, it will + * modify the execution path and no need to single + * stepping. Let's just reset current kprobe and exit. + * + * pre_handler can hit a breakpoint and can step thru + * before return. + */ + if (!p->pre_handler || !p->pre_handler(p, regs)) + setup_singlestep(p, regs, kcb, 0); + else + reset_current_kprobe(); + } + return true; + } + + /* + * The breakpoint instruction was removed right + * after we hit it. Another cpu has removed + * either a probepoint or a debugger breakpoint + * at this address. In either case, no further + * handling of this interrupt is appropriate. + * Return back to original instruction, and continue. + */ + return false; +} + +bool __kprobes +kprobe_single_step_handler(struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if ((kcb->ss_ctx.ss_pending) + && (kcb->ss_ctx.match_addr == instruction_pointer(regs))) { + clear_ss_context(kcb); /* clear pending ss */ + + kprobes_restore_local_irqflag(kcb, regs); + + post_kprobe_handler(kcb, regs); + return true; + } + return false; +} + +/* + * Provide a blacklist of symbols identifying ranges which cannot be kprobed. + * This blacklist is exposed to userspace via debugfs (kprobes/blacklist). + */ +int __init arch_populate_kprobe_blacklist(void) +{ + int ret; + + ret = kprobe_add_area_blacklist((unsigned long)__irqentry_text_start, + (unsigned long)__irqentry_text_end); + return ret; +} + +void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs) +{ + return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL); +} + +void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, + struct pt_regs *regs) +{ + ri->ret_addr = (kprobe_opcode_t *)regs->ra; + ri->fp = NULL; + regs->ra = (unsigned long) &kretprobe_trampoline; +} + +int __kprobes arch_trampoline_kprobe(struct kprobe *p) +{ + return 0; +} + +int __init arch_init_kprobes(void) +{ + return 0; +} diff --git a/arch/riscv/kernel/probes/kprobes_trampoline.S b/arch/riscv/kernel/probes/kprobes_trampoline.S new file mode 100644 index 000000000000..6e85d021e2a2 --- /dev/null +++ b/arch/riscv/kernel/probes/kprobes_trampoline.S @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Author: Patrick Stählin + */ +#include + +#include +#include + + .text + .altmacro + + .macro save_all_base_regs + REG_S x1, PT_RA(sp) + REG_S x3, PT_GP(sp) + REG_S x4, PT_TP(sp) + REG_S x5, PT_T0(sp) + REG_S x6, PT_T1(sp) + REG_S x7, PT_T2(sp) + REG_S x8, PT_S0(sp) + REG_S x9, PT_S1(sp) + REG_S x10, PT_A0(sp) + REG_S x11, PT_A1(sp) + REG_S x12, PT_A2(sp) + REG_S x13, PT_A3(sp) + REG_S x14, PT_A4(sp) + REG_S x15, PT_A5(sp) + REG_S x16, PT_A6(sp) + REG_S x17, PT_A7(sp) + REG_S x18, PT_S2(sp) + REG_S x19, PT_S3(sp) + REG_S x20, PT_S4(sp) + REG_S x21, PT_S5(sp) + REG_S x22, PT_S6(sp) + REG_S x23, PT_S7(sp) + REG_S x24, PT_S8(sp) + REG_S x25, PT_S9(sp) + REG_S x26, PT_S10(sp) + REG_S x27, PT_S11(sp) + REG_S x28, PT_T3(sp) + REG_S x29, PT_T4(sp) + REG_S x30, PT_T5(sp) + REG_S x31, PT_T6(sp) + .endm + + .macro restore_all_base_regs + REG_L x3, PT_GP(sp) + REG_L x4, PT_TP(sp) + REG_L x5, PT_T0(sp) + REG_L x6, PT_T1(sp) + REG_L x7, PT_T2(sp) + REG_L x8, PT_S0(sp) + REG_L x9, PT_S1(sp) + REG_L x10, PT_A0(sp) + REG_L x11, PT_A1(sp) + REG_L x12, PT_A2(sp) + REG_L x13, PT_A3(sp) + REG_L x14, PT_A4(sp) + REG_L x15, PT_A5(sp) + REG_L x16, PT_A6(sp) + REG_L x17, PT_A7(sp) + REG_L x18, PT_S2(sp) + REG_L x19, PT_S3(sp) + REG_L x20, PT_S4(sp) + REG_L x21, PT_S5(sp) + REG_L x22, PT_S6(sp) + REG_L x23, PT_S7(sp) + REG_L x24, PT_S8(sp) + REG_L x25, PT_S9(sp) + REG_L x26, PT_S10(sp) + REG_L x27, PT_S11(sp) + REG_L x28, PT_T3(sp) + REG_L x29, PT_T4(sp) + REG_L x30, PT_T5(sp) + REG_L x31, PT_T6(sp) + .endm + +ENTRY(kretprobe_trampoline) + addi sp, sp, -(PT_SIZE_ON_STACK) + save_all_base_regs + + move a0, sp /* pt_regs */ + + call trampoline_probe_handler + + /* use the result as the return-address */ + move ra, a0 + + restore_all_base_regs + addi sp, sp, PT_SIZE_ON_STACK + + ret +ENDPROC(kretprobe_trampoline) diff --git a/arch/riscv/kernel/probes/simulate-insn.c b/arch/riscv/kernel/probes/simulate-insn.c new file mode 100644 index 000000000000..2519ce26377d --- /dev/null +++ b/arch/riscv/kernel/probes/simulate-insn.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include +#include +#include + +#include "decode-insn.h" +#include "simulate-insn.h" + +static inline bool rv_insn_reg_get_val(struct pt_regs *regs, u32 index, + unsigned long *ptr) +{ + if (index == 0) + *ptr = 0; + else if (index <= 31) + *ptr = *((unsigned long *)regs + index); + else + return false; + + return true; +} + +static inline bool rv_insn_reg_set_val(struct pt_regs *regs, u32 index, + unsigned long val) +{ + if (index == 0) + return false; + else if (index <= 31) + *((unsigned long *)regs + index) = val; + else + return false; + + return true; +} + +bool __kprobes simulate_jal(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + /* + * 31 30 21 20 19 12 11 7 6 0 + * imm [20] | imm[10:1] | imm[11] | imm[19:12] | rd | opcode + * 1 10 1 8 5 JAL/J + */ + bool ret; + u32 imm; + u32 index = (opcode >> 7) & 0x1f; + + ret = rv_insn_reg_set_val(regs, index, addr + 4); + if (!ret) + return ret; + + imm = ((opcode >> 21) & 0x3ff) << 1; + imm |= ((opcode >> 20) & 0x1) << 11; + imm |= ((opcode >> 12) & 0xff) << 12; + imm |= ((opcode >> 31) & 0x1) << 20; + + instruction_pointer_set(regs, addr + sign_extend32((imm), 20)); + + return ret; +} + +bool __kprobes simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + /* + * 31 20 19 15 14 12 11 7 6 0 + * offset[11:0] | rs1 | 010 | rd | opcode + * 12 5 3 5 JALR/JR + */ + bool ret; + unsigned long base_addr; + u32 imm = (opcode >> 20) & 0xfff; + u32 rd_index = (opcode >> 7) & 0x1f; + u32 rs1_index = (opcode >> 15) & 0x1f; + + ret = rv_insn_reg_set_val(regs, rd_index, addr + 4); + if (!ret) + return ret; + + ret = rv_insn_reg_get_val(regs, rs1_index, &base_addr); + if (!ret) + return ret; + + instruction_pointer_set(regs, (base_addr + sign_extend32((imm), 11))&~1); + + return ret; +} diff --git a/arch/riscv/kernel/probes/simulate-insn.h b/arch/riscv/kernel/probes/simulate-insn.h new file mode 100644 index 000000000000..cb6ff7dccb92 --- /dev/null +++ b/arch/riscv/kernel/probes/simulate-insn.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef _RISCV_KERNEL_PROBES_SIMULATE_INSN_H +#define _RISCV_KERNEL_PROBES_SIMULATE_INSN_H + +#define __RISCV_INSN_FUNCS(name, mask, val) \ +static __always_inline bool riscv_insn_is_##name(probe_opcode_t code) \ +{ \ + BUILD_BUG_ON(~(mask) & (val)); \ + return (code & (mask)) == (val); \ +} \ +bool simulate_##name(u32 opcode, unsigned long addr, \ + struct pt_regs *regs) + +#define RISCV_INSN_REJECTED(name, code) \ + do { \ + if (riscv_insn_is_##name(code)) { \ + return INSN_REJECTED; \ + } \ + } while (0) + +__RISCV_INSN_FUNCS(system, 0x7f, 0x73); +__RISCV_INSN_FUNCS(fence, 0x7f, 0x0f); + +#define RISCV_INSN_SET_SIMULATE(name, code) \ + do { \ + if (riscv_insn_is_##name(code)) { \ + api->handler = simulate_##name; \ + return INSN_GOOD_NO_SLOT; \ + } \ + } while (0) + +__RISCV_INSN_FUNCS(c_j, 0xe003, 0xa001); +__RISCV_INSN_FUNCS(c_jr, 0xf007, 0x8002); +__RISCV_INSN_FUNCS(c_jal, 0xe003, 0x2001); +__RISCV_INSN_FUNCS(c_jalr, 0xf007, 0x9002); +__RISCV_INSN_FUNCS(c_beqz, 0xe003, 0xc001); +__RISCV_INSN_FUNCS(c_bnez, 0xe003, 0xe001); +__RISCV_INSN_FUNCS(c_ebreak, 0xffff, 0x9002); + +__RISCV_INSN_FUNCS(auipc, 0x7f, 0x17); +__RISCV_INSN_FUNCS(branch, 0x7f, 0x63); + +__RISCV_INSN_FUNCS(jal, 0x7f, 0x6f); +__RISCV_INSN_FUNCS(jalr, 0x707f, 0x67); + +#endif /* _RISCV_KERNEL_PROBES_SIMULATE_INSN_H */ diff --git a/arch/riscv/kernel/probes/uprobes.c b/arch/riscv/kernel/probes/uprobes.c new file mode 100644 index 000000000000..7a057b5f0adc --- /dev/null +++ b/arch/riscv/kernel/probes/uprobes.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include + +#include "decode-insn.h" + +#define UPROBE_TRAP_NR UINT_MAX + +bool is_swbp_insn(uprobe_opcode_t *insn) +{ +#ifdef CONFIG_RISCV_ISA_C + return (*insn & 0xffff) == UPROBE_SWBP_INSN; +#else + return *insn == UPROBE_SWBP_INSN; +#endif +} + +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long addr) +{ + probe_opcode_t opcode; + + opcode = *(probe_opcode_t *)(&auprobe->insn[0]); + + auprobe->insn_size = GET_INSN_LENGTH(opcode); + + switch (riscv_probe_decode_insn(&opcode, &auprobe->api)) { + case INSN_REJECTED: + return -EINVAL; + + case INSN_GOOD_NO_SLOT: + auprobe->simulate = true; + break; + + case INSN_GOOD: + auprobe->simulate = false; + break; + + default: + return -EINVAL; + } + + return 0; +} + +int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + utask->autask.saved_cause = current->thread.bad_cause; + current->thread.bad_cause = UPROBE_TRAP_NR; + + instruction_pointer_set(regs, utask->xol_vaddr); + + regs->status &= ~SR_SPIE; + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + WARN_ON_ONCE(current->thread.bad_cause != UPROBE_TRAP_NR); + + instruction_pointer_set(regs, utask->vaddr + auprobe->insn_size); + + regs->status |= SR_SPIE; + + return 0; +} + +bool arch_uprobe_xol_was_trapped(struct task_struct *t) +{ + if (t->thread.bad_cause != UPROBE_TRAP_NR) + return true; + + return false; +} + +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + probe_opcode_t insn; + unsigned long addr; + + if (!auprobe->simulate) + return false; + + insn = *(probe_opcode_t *)(&auprobe->insn[0]); + addr = instruction_pointer(regs); + + if (auprobe->api.handler) + auprobe->api.handler(insn, addr, regs); + + return true; +} + +void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + /* + * Task has received a fatal signal, so reset back to probbed + * address. + */ + instruction_pointer_set(regs, utask->vaddr); + + regs->status &= ~SR_SPIE; +} + +bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx, + struct pt_regs *regs) +{ + if (ctx == RP_CHECK_CHAIN_CALL) + return regs->sp <= ret->stack; + else + return regs->sp < ret->stack; +} + +unsigned long +arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, + struct pt_regs *regs) +{ + unsigned long ra; + + ra = regs->ra; + + regs->ra = trampoline_vaddr; + + return ra; +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} + +bool uprobe_breakpoint_handler(struct pt_regs *regs) +{ + if (uprobe_pre_sstep_notifier(regs)) + return true; + + return false; +} + +bool uprobe_single_step_handler(struct pt_regs *regs) +{ + if (uprobe_post_sstep_notifier(regs)) + return true; + + return false; +} + +void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, + void *src, unsigned long len) +{ + /* Initialize the slot */ + void *kaddr = kmap_atomic(page); + void *dst = kaddr + (vaddr & ~PAGE_MASK); + + memcpy(dst, src, len); + + /* Add ebreak behind opcode to simulate singlestep */ + if (vaddr) { + dst += GET_INSN_LENGTH(*(probe_opcode_t *)src); + *(uprobe_opcode_t *)dst = __BUG_INSN_32; + } + + kunmap_atomic(kaddr); + + /* + * We probably need flush_icache_user_page() but it needs vma. + * This should work on most of architectures by default. If + * architecture needs to do something different it can define + * its own version of the function. + */ + flush_dcache_page(page); +} diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index dd5f985b1f40..6f728e731bed 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -18,13 +18,14 @@ #include #include #include +#include #include #include #include register unsigned long gp_in_global __asm__("gp"); -#ifdef CONFIG_STACKPROTECTOR +#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); @@ -39,11 +40,16 @@ void arch_cpu_idle(void) raw_local_irq_enable(); } -void show_regs(struct pt_regs *regs) +void __show_regs(struct pt_regs *regs) { show_regs_print_info(KERN_DEFAULT); - pr_cont("epc: " REG_FMT " ra : " REG_FMT " sp : " REG_FMT "\n", + if (!user_mode(regs)) { + pr_cont("epc : %pS\n", (void *)regs->epc); + pr_cont(" ra : %pS\n", (void *)regs->ra); + } + + pr_cont("epc : " REG_FMT " ra : " REG_FMT " sp : " REG_FMT "\n", regs->epc, regs->ra, regs->sp); pr_cont(" gp : " REG_FMT " tp : " REG_FMT " t0 : " REG_FMT "\n", regs->gp, regs->tp, regs->t0); @@ -69,6 +75,12 @@ void show_regs(struct pt_regs *regs) pr_cont("status: " REG_FMT " badaddr: " REG_FMT " cause: " REG_FMT "\n", regs->status, regs->badaddr, regs->cause); } +void show_regs(struct pt_regs *regs) +{ + __show_regs(regs); + if (!user_mode(regs)) + dump_backtrace(regs, NULL, KERN_DEFAULT); +} void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) @@ -112,7 +124,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, struct pt_regs *childregs = task_pt_regs(p); /* p->thread holds context to be restored by __switch_to() */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* Kernel thread */ memset(childregs, 0, sizeof(struct pt_regs)); childregs->gp = gp_in_global; diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index 2d6395f5ad54..1a85305720e8 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -114,6 +114,105 @@ const struct user_regset_view *task_user_regset_view(struct task_struct *task) return &riscv_user_native_view; } +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)} +#define REG_OFFSET_END {.name = NULL, .offset = 0} + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME(epc), + REG_OFFSET_NAME(ra), + REG_OFFSET_NAME(sp), + REG_OFFSET_NAME(gp), + REG_OFFSET_NAME(tp), + REG_OFFSET_NAME(t0), + REG_OFFSET_NAME(t1), + REG_OFFSET_NAME(t2), + REG_OFFSET_NAME(s0), + REG_OFFSET_NAME(s1), + REG_OFFSET_NAME(a0), + REG_OFFSET_NAME(a1), + REG_OFFSET_NAME(a2), + REG_OFFSET_NAME(a3), + REG_OFFSET_NAME(a4), + REG_OFFSET_NAME(a5), + REG_OFFSET_NAME(a6), + REG_OFFSET_NAME(a7), + REG_OFFSET_NAME(s2), + REG_OFFSET_NAME(s3), + REG_OFFSET_NAME(s4), + REG_OFFSET_NAME(s5), + REG_OFFSET_NAME(s6), + REG_OFFSET_NAME(s7), + REG_OFFSET_NAME(s8), + REG_OFFSET_NAME(s9), + REG_OFFSET_NAME(s10), + REG_OFFSET_NAME(s11), + REG_OFFSET_NAME(t3), + REG_OFFSET_NAME(t4), + REG_OFFSET_NAME(t5), + REG_OFFSET_NAME(t6), + REG_OFFSET_NAME(status), + REG_OFFSET_NAME(badaddr), + REG_OFFSET_NAME(cause), + REG_OFFSET_NAME(orig_a0), + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) +{ + return (addr & ~(THREAD_SIZE - 1)) == + (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1)); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); + + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; +} + void ptrace_disable(struct task_struct *child) { clear_tsk_thread_flag(child, TIF_SYSCALL_TRACE); diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c index 226ccce0f9e0..f4a7db3d309e 100644 --- a/arch/riscv/kernel/sbi.c +++ b/arch/riscv/kernel/sbi.c @@ -351,7 +351,7 @@ static int __sbi_rfence_v02(int fid, const unsigned long *hart_mask, * sbi_set_timer() - Program the timer for next timer event. * @stime_value: The value after which next timer event should fire. * - * Return: None + * Return: None. */ void sbi_set_timer(uint64_t stime_value) { @@ -362,11 +362,11 @@ void sbi_set_timer(uint64_t stime_value) * sbi_send_ipi() - Send an IPI to any hart. * @hart_mask: A cpu mask containing all the target harts. * - * Return: None + * Return: 0 on success, appropriate linux error code otherwise. */ -void sbi_send_ipi(const unsigned long *hart_mask) +int sbi_send_ipi(const unsigned long *hart_mask) { - __sbi_send_ipi(hart_mask); + return __sbi_send_ipi(hart_mask); } EXPORT_SYMBOL(sbi_send_ipi); @@ -374,12 +374,12 @@ EXPORT_SYMBOL(sbi_send_ipi); * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts. * @hart_mask: A cpu mask containing all the target harts. * - * Return: None + * Return: 0 on success, appropriate linux error code otherwise. */ -void sbi_remote_fence_i(const unsigned long *hart_mask) +int sbi_remote_fence_i(const unsigned long *hart_mask) { - __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, - hart_mask, 0, 0, 0, 0); + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, + hart_mask, 0, 0, 0, 0); } EXPORT_SYMBOL(sbi_remote_fence_i); @@ -390,14 +390,14 @@ EXPORT_SYMBOL(sbi_remote_fence_i); * @start: Start of the virtual address * @size: Total size of the virtual address range. * - * Return: None + * Return: 0 on success, appropriate linux error code otherwise. */ -void sbi_remote_sfence_vma(const unsigned long *hart_mask, +int sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size) { - __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, - hart_mask, start, size, 0, 0); + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, + hart_mask, start, size, 0, 0); } EXPORT_SYMBOL(sbi_remote_sfence_vma); @@ -410,15 +410,15 @@ EXPORT_SYMBOL(sbi_remote_sfence_vma); * @size: Total size of the virtual address range. * @asid: The value of address space identifier (ASID). * - * Return: None + * Return: 0 on success, appropriate linux error code otherwise. */ -void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, +int sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, unsigned long size, unsigned long asid) { - __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, - hart_mask, start, size, asid, 0); + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, + hart_mask, start, size, asid, 0); } EXPORT_SYMBOL(sbi_remote_sfence_vma_asid); @@ -560,7 +560,7 @@ static struct riscv_ipi_ops sbi_ipi_ops = { .ipi_inject = sbi_send_cpumask_ipi }; -int __init sbi_init(void) +void __init sbi_init(void) { int ret; @@ -600,6 +600,4 @@ int __init sbi_init(void) } riscv_set_ipi_ops(&sbi_ipi_ops); - - return 0; } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index c7c0655dd45b..e85bacff1b50 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -216,8 +216,15 @@ static void __init init_resources(void) static void __init parse_dtb(void) { /* Early scan of device tree from init memory */ - if (early_init_dt_scan(dtb_early_va)) + if (early_init_dt_scan(dtb_early_va)) { + const char *name = of_flat_dt_get_machine_name(); + + if (name) { + pr_info("Machine model: %s\n", name); + dump_stack_set_arch_desc("%s (DT)", name); + } return; + } pr_err("No DTB passed to the kernel\n"); #ifdef CONFIG_CMDLINE_FORCE @@ -252,9 +259,9 @@ void __init setup_arch(char **cmdline_p) else pr_err("No DTB found in kernel mappings\n"); #endif + misc_mem_init(); - if (IS_ENABLED(CONFIG_RISCV_SBI)) - sbi_init(); + sbi_init(); if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) protect_kernel_text_data(); @@ -275,13 +282,19 @@ void __init setup_arch(char **cmdline_p) static int __init topology_init(void) { - int i; + int i, ret; + + for_each_online_node(i) + register_one_node(i); for_each_possible_cpu(i) { struct cpu *cpu = &per_cpu(cpu_devices, i); cpu->hotpluggable = cpu_has_hotplug(i); - register_cpu(cpu, i); + ret = register_cpu(cpu, i); + if (unlikely(ret)) + pr_warn("Warning: %s: register_cpu %d failed (%d)\n", + __func__, i, ret); } return 0; diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index 469aef8ed922..65942b3748b4 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -309,6 +309,9 @@ static void do_signal(struct pt_regs *regs) asmlinkage __visible void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) { + if (thread_info_flags & _TIF_UPROBE) + uprobe_notify_resume(regs); + /* Handle pending signal delivery */ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) do_signal(regs); diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index 96167d55ed98..5e276c25646f 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -45,13 +46,18 @@ void __init smp_prepare_cpus(unsigned int max_cpus) { int cpuid; int ret; + unsigned int curr_cpuid; + + curr_cpuid = smp_processor_id(); + numa_store_cpu_info(curr_cpuid); + numa_add_cpu(curr_cpuid); /* This covers non-smp usecase mandated by "nosmp" option */ if (max_cpus == 0) return; for_each_possible_cpu(cpuid) { - if (cpuid == smp_processor_id()) + if (cpuid == curr_cpuid) continue; if (cpu_ops[cpuid]->cpu_prepare) { ret = cpu_ops[cpuid]->cpu_prepare(cpuid); @@ -59,6 +65,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) continue; } set_cpu_present(cpuid, true); + numa_store_cpu_info(cpuid); } } @@ -79,6 +86,7 @@ void __init setup_smp(void) if (hart == cpuid_to_hartid_map(0)) { BUG_ON(found_boot_cpu); found_boot_cpu = 1; + early_map_cpu_to_node(0, of_node_to_nid(dn)); continue; } if (cpuid >= NR_CPUS) { @@ -88,6 +96,7 @@ void __init setup_smp(void) } cpuid_to_hartid_map(cpuid) = hart; + early_map_cpu_to_node(cpuid, of_node_to_nid(dn)); cpuid++; } @@ -153,6 +162,7 @@ asmlinkage __visible void smp_callin(void) current->active_mm = mm; notify_cpu_starting(curr_cpuid); + numa_add_cpu(curr_cpuid); update_siblings_masks(curr_cpuid); set_cpu_online(curr_cpuid, 1); diff --git a/arch/riscv/kernel/soc.c b/arch/riscv/kernel/soc.c index c7b0a73e382e..a0516172a33c 100644 --- a/arch/riscv/kernel/soc.c +++ b/arch/riscv/kernel/soc.c @@ -26,30 +26,3 @@ void __init soc_early_init(void) } } } - -static bool soc_builtin_dtb_match(unsigned long vendor_id, - unsigned long arch_id, unsigned long imp_id, - const struct soc_builtin_dtb *entry) -{ - return entry->vendor_id == vendor_id && - entry->arch_id == arch_id && - entry->imp_id == imp_id; -} - -void * __init soc_lookup_builtin_dtb(void) -{ - unsigned long vendor_id, arch_id, imp_id; - const struct soc_builtin_dtb *s; - - __asm__ ("csrr %0, mvendorid" : "=r"(vendor_id)); - __asm__ ("csrr %0, marchid" : "=r"(arch_id)); - __asm__ ("csrr %0, mimpid" : "=r"(imp_id)); - - for (s = (void *)&__soc_builtin_dtb_table_start; - (void *)s < (void *)&__soc_builtin_dtb_table_end; s++) { - if (soc_builtin_dtb_match(vendor_id, arch_id, imp_id, s)) - return s->dtb_func(); - } - - return NULL; -} diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index df5d2da7c40b..3f893c9d9d85 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -53,9 +53,15 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, /* Unwind stack frame */ frame = (struct stackframe *)fp - 1; sp = fp; - fp = frame->fp; - pc = ftrace_graph_ret_addr(current, NULL, frame->ra, - (unsigned long *)(fp - 8)); + if (regs && (regs->epc == pc) && (frame->fp & 0x7)) { + fp = frame->ra; + pc = regs->ra; + } else { + fp = frame->fp; + pc = ftrace_graph_ret_addr(current, NULL, frame->ra, + (unsigned long *)(fp - 8)); + } + } } @@ -100,10 +106,16 @@ static bool print_trace_address(void *arg, unsigned long pc) return true; } +void dump_backtrace(struct pt_regs *regs, struct task_struct *task, + const char *loglvl) +{ + pr_cont("%sCall Trace:\n", loglvl); + walk_stackframe(task, regs, print_trace_address, (void *)loglvl); +} + void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) { - pr_cont("Call Trace:\n"); - walk_stackframe(task, NULL, print_trace_address, (void *)loglvl); + dump_backtrace(NULL, task, loglvl); } static bool save_wchan(void *arg, unsigned long pc) diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index ad14f4466d92..3ed2c23601a0 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -12,10 +12,12 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -66,7 +68,7 @@ void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) tsk->comm, task_pid_nr(tsk), signo, code, addr); print_vma_addr(KERN_CONT " in ", instruction_pointer(regs)); pr_cont("\n"); - show_regs(regs); + __show_regs(regs); } force_sig_fault(signo, code, (void __user *)addr); @@ -75,6 +77,8 @@ void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) static void do_trap_error(struct pt_regs *regs, int signo, int code, unsigned long addr, const char *str) { + current->thread.bad_cause = regs->cause; + if (user_mode(regs)) { do_trap(regs, signo, code, addr); } else { @@ -145,6 +149,22 @@ static inline unsigned long get_break_insn_length(unsigned long pc) asmlinkage __visible void do_trap_break(struct pt_regs *regs) { +#ifdef CONFIG_KPROBES + if (kprobe_single_step_handler(regs)) + return; + + if (kprobe_breakpoint_handler(regs)) + return; +#endif +#ifdef CONFIG_UPROBES + if (uprobe_single_step_handler(regs)) + return; + + if (uprobe_breakpoint_handler(regs)) + return; +#endif + current->thread.bad_cause = regs->cause; + if (user_mode(regs)) force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc); #ifdef CONFIG_KGDB diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 0cfd6da784f8..71a315e73cbe 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -32,9 +32,10 @@ CPPFLAGS_vdso.lds += -P -C -U$(ARCH) # Disable -pg to prevent insert call site CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os -# Disable gcov profiling for VDSO code +# Disable profiling and instrumentation for VDSO code GCOV_PROFILE := n KCOV_INSTRUMENT := n +KASAN_SANITIZE := n # Force dependency $(obj)/vdso.o: $(obj)/vdso.so diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index ac6171e9c19e..25d5c9664e57 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -5,3 +5,5 @@ lib-y += memset.o lib-y += memmove.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o + +obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o diff --git a/arch/riscv/lib/error-inject.c b/arch/riscv/lib/error-inject.c new file mode 100644 index 000000000000..d667ade2bc41 --- /dev/null +++ b/arch/riscv/lib/error-inject.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +void override_function_with_return(struct pt_regs *regs) +{ + instruction_pointer_set(regs, regs->ra); +} +NOKPROBE_SYMBOL(override_function_with_return); diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index c0185e556ca5..7ebaef10ea1b 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -2,7 +2,8 @@ CFLAGS_init.o := -mcmodel=medany ifdef CONFIG_FTRACE -CFLAGS_REMOVE_init.o = -pg +CFLAGS_REMOVE_init.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_cacheflush.o = $(CC_FLAGS_FTRACE) endif KCOV_INSTRUMENT_init.o := n diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 613ec81a8979..68aa312fc352 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -2,13 +2,273 @@ /* * Copyright (C) 2012 Regents of the University of California * Copyright (C) 2017 SiFive + * Copyright (C) 2021 Western Digital Corporation or its affiliates. */ +#include +#include #include +#include +#include +#include +#include #include #include #include +#ifdef CONFIG_MMU + +static DEFINE_STATIC_KEY_FALSE(use_asid_allocator); + +static unsigned long asid_bits; +static unsigned long num_asids; +static unsigned long asid_mask; + +static atomic_long_t current_version; + +static DEFINE_RAW_SPINLOCK(context_lock); +static cpumask_t context_tlb_flush_pending; +static unsigned long *context_asid_map; + +static DEFINE_PER_CPU(atomic_long_t, active_context); +static DEFINE_PER_CPU(unsigned long, reserved_context); + +static bool check_update_reserved_context(unsigned long cntx, + unsigned long newcntx) +{ + int cpu; + bool hit = false; + + /* + * Iterate over the set of reserved CONTEXT looking for a match. + * If we find one, then we can update our mm to use new CONTEXT + * (i.e. the same CONTEXT in the current_version) but we can't + * exit the loop early, since we need to ensure that all copies + * of the old CONTEXT are updated to reflect the mm. Failure to do + * so could result in us missing the reserved CONTEXT in a future + * version. + */ + for_each_possible_cpu(cpu) { + if (per_cpu(reserved_context, cpu) == cntx) { + hit = true; + per_cpu(reserved_context, cpu) = newcntx; + } + } + + return hit; +} + +static void __flush_context(void) +{ + int i; + unsigned long cntx; + + /* Must be called with context_lock held */ + lockdep_assert_held(&context_lock); + + /* Update the list of reserved ASIDs and the ASID bitmap. */ + bitmap_clear(context_asid_map, 0, num_asids); + + /* Mark already active ASIDs as used */ + for_each_possible_cpu(i) { + cntx = atomic_long_xchg_relaxed(&per_cpu(active_context, i), 0); + /* + * If this CPU has already been through a rollover, but + * hasn't run another task in the meantime, we must preserve + * its reserved CONTEXT, as this is the only trace we have of + * the process it is still running. + */ + if (cntx == 0) + cntx = per_cpu(reserved_context, i); + + __set_bit(cntx & asid_mask, context_asid_map); + per_cpu(reserved_context, i) = cntx; + } + + /* Mark ASID #0 as used because it is used at boot-time */ + __set_bit(0, context_asid_map); + + /* Queue a TLB invalidation for each CPU on next context-switch */ + cpumask_setall(&context_tlb_flush_pending); +} + +static unsigned long __new_context(struct mm_struct *mm) +{ + static u32 cur_idx = 1; + unsigned long cntx = atomic_long_read(&mm->context.id); + unsigned long asid, ver = atomic_long_read(¤t_version); + + /* Must be called with context_lock held */ + lockdep_assert_held(&context_lock); + + if (cntx != 0) { + unsigned long newcntx = ver | (cntx & asid_mask); + + /* + * If our current CONTEXT was active during a rollover, we + * can continue to use it and this was just a false alarm. + */ + if (check_update_reserved_context(cntx, newcntx)) + return newcntx; + + /* + * We had a valid CONTEXT in a previous life, so try to + * re-use it if possible. + */ + if (!__test_and_set_bit(cntx & asid_mask, context_asid_map)) + return newcntx; + } + + /* + * Allocate a free ASID. If we can't find one then increment + * current_version and flush all ASIDs. + */ + asid = find_next_zero_bit(context_asid_map, num_asids, cur_idx); + if (asid != num_asids) + goto set_asid; + + /* We're out of ASIDs, so increment current_version */ + ver = atomic_long_add_return_relaxed(num_asids, ¤t_version); + + /* Flush everything */ + __flush_context(); + + /* We have more ASIDs than CPUs, so this will always succeed */ + asid = find_next_zero_bit(context_asid_map, num_asids, 1); + +set_asid: + __set_bit(asid, context_asid_map); + cur_idx = asid; + return asid | ver; +} + +static void set_mm_asid(struct mm_struct *mm, unsigned int cpu) +{ + unsigned long flags; + bool need_flush_tlb = false; + unsigned long cntx, old_active_cntx; + + cntx = atomic_long_read(&mm->context.id); + + /* + * If our active_context is non-zero and the context matches the + * current_version, then we update the active_context entry with a + * relaxed cmpxchg. + * + * Following is how we handle racing with a concurrent rollover: + * + * - We get a zero back from the cmpxchg and end up waiting on the + * lock. Taking the lock synchronises with the rollover and so + * we are forced to see the updated verion. + * + * - We get a valid context back from the cmpxchg then we continue + * using old ASID because __flush_context() would have marked ASID + * of active_context as used and next context switch we will + * allocate new context. + */ + old_active_cntx = atomic_long_read(&per_cpu(active_context, cpu)); + if (old_active_cntx && + ((cntx & ~asid_mask) == atomic_long_read(¤t_version)) && + atomic_long_cmpxchg_relaxed(&per_cpu(active_context, cpu), + old_active_cntx, cntx)) + goto switch_mm_fast; + + raw_spin_lock_irqsave(&context_lock, flags); + + /* Check that our ASID belongs to the current_version. */ + cntx = atomic_long_read(&mm->context.id); + if ((cntx & ~asid_mask) != atomic_long_read(¤t_version)) { + cntx = __new_context(mm); + atomic_long_set(&mm->context.id, cntx); + } + + if (cpumask_test_and_clear_cpu(cpu, &context_tlb_flush_pending)) + need_flush_tlb = true; + + atomic_long_set(&per_cpu(active_context, cpu), cntx); + + raw_spin_unlock_irqrestore(&context_lock, flags); + +switch_mm_fast: + csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | + ((cntx & asid_mask) << SATP_ASID_SHIFT) | + SATP_MODE); + + if (need_flush_tlb) + local_flush_tlb_all(); +} + +static void set_mm_noasid(struct mm_struct *mm) +{ + /* Switch the page table and blindly nuke entire local TLB */ + csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | SATP_MODE); + local_flush_tlb_all(); +} + +static inline void set_mm(struct mm_struct *mm, unsigned int cpu) +{ + if (static_branch_unlikely(&use_asid_allocator)) + set_mm_asid(mm, cpu); + else + set_mm_noasid(mm); +} + +static int asids_init(void) +{ + unsigned long old; + + /* Figure-out number of ASID bits in HW */ + old = csr_read(CSR_SATP); + asid_bits = old | (SATP_ASID_MASK << SATP_ASID_SHIFT); + csr_write(CSR_SATP, asid_bits); + asid_bits = (csr_read(CSR_SATP) >> SATP_ASID_SHIFT) & SATP_ASID_MASK; + asid_bits = fls_long(asid_bits); + csr_write(CSR_SATP, old); + + /* + * In the process of determining number of ASID bits (above) + * we polluted the TLB of current HART so let's do TLB flushed + * to remove unwanted TLB enteries. + */ + local_flush_tlb_all(); + + /* Pre-compute ASID details */ + num_asids = 1 << asid_bits; + asid_mask = num_asids - 1; + + /* + * Use ASID allocator only if number of HW ASIDs are + * at-least twice more than CPUs + */ + if (num_asids > (2 * num_possible_cpus())) { + atomic_long_set(¤t_version, num_asids); + + context_asid_map = kcalloc(BITS_TO_LONGS(num_asids), + sizeof(*context_asid_map), GFP_KERNEL); + if (!context_asid_map) + panic("Failed to allocate bitmap for %lu ASIDs\n", + num_asids); + + __set_bit(0, context_asid_map); + + static_branch_enable(&use_asid_allocator); + + pr_info("ASID allocator using %lu bits (%lu entries)\n", + asid_bits, num_asids); + } else { + pr_info("ASID allocator disabled\n"); + } + + return 0; +} +early_initcall(asids_init); +#else +static inline void set_mm(struct mm_struct *mm, unsigned int cpu) +{ + /* Nothing to do here when there is no MMU */ +} +#endif + /* * When necessary, performs a deferred icache flush for the given MM context, * on the local CPU. RISC-V has no direct mechanism for instruction cache @@ -58,10 +318,7 @@ void switch_mm(struct mm_struct *prev, struct mm_struct *next, cpumask_clear_cpu(cpu, mm_cpumask(prev)); cpumask_set_cpu(cpu, mm_cpumask(next)); -#ifdef CONFIG_MMU - csr_write(CSR_SATP, virt_to_pfn(next->pgd) | SATP_MODE); - local_flush_tlb_all(); -#endif + set_mm(next, cpu); flush_icache_deferred(next); } diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 3c8b9e433c67..8f17519208c7 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -13,14 +13,30 @@ #include #include #include +#include #include #include #include "../kernel/head.h" +static void die_kernel_fault(const char *msg, unsigned long addr, + struct pt_regs *regs) +{ + bust_spinlocks(1); + + pr_alert("Unable to handle kernel %s at virtual address " REG_FMT "\n", msg, + addr); + + bust_spinlocks(0); + die(regs, "Oops"); + do_exit(SIGKILL); +} + static inline void no_context(struct pt_regs *regs, unsigned long addr) { + const char *msg; + /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) return; @@ -29,12 +45,8 @@ static inline void no_context(struct pt_regs *regs, unsigned long addr) * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ - bust_spinlocks(1); - pr_alert("Unable to handle kernel %s at virtual address " REG_FMT "\n", - (addr < PAGE_SIZE) ? "NULL pointer dereference" : - "paging request", addr); - die(regs, "Oops"); - do_exit(SIGKILL); + msg = (addr < PAGE_SIZE) ? "NULL pointer dereference" : "paging request"; + die_kernel_fault(msg, addr, regs); } static inline void mm_fault_error(struct pt_regs *regs, unsigned long addr, vm_fault_t fault) @@ -202,6 +214,9 @@ asmlinkage void do_page_fault(struct pt_regs *regs) tsk = current; mm = tsk->mm; + if (kprobe_page_fault(regs, cause)) + return; + /* * Fault-in kernel-space virtual memory on-demand. * The 'reference' page table is init_mm.pgd. @@ -225,6 +240,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) * in an atomic region, then we must not take the fault. */ if (unlikely(faulthandler_disabled() || !mm)) { + tsk->thread.bad_cause = cause; no_context(regs, addr); return; } @@ -232,6 +248,11 @@ asmlinkage void do_page_fault(struct pt_regs *regs) if (user_mode(regs)) flags |= FAULT_FLAG_USER; + if (!user_mode(regs) && addr < TASK_SIZE && + unlikely(!(regs->status & SR_SUM))) + die_kernel_fault("access to user memory without uaccess routines", + addr, regs); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); if (cause == EXC_STORE_PAGE_FAULT) @@ -242,16 +263,19 @@ retry: mmap_read_lock(mm); vma = find_vma(mm, addr); if (unlikely(!vma)) { + tsk->thread.bad_cause = cause; bad_area(regs, mm, code, addr); return; } if (likely(vma->vm_start <= addr)) goto good_area; if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) { + tsk->thread.bad_cause = cause; bad_area(regs, mm, code, addr); return; } if (unlikely(expand_stack(vma, addr))) { + tsk->thread.bad_cause = cause; bad_area(regs, mm, code, addr); return; } @@ -264,6 +288,7 @@ good_area: code = SEGV_ACCERR; if (unlikely(access_error(cause, vma))) { + tsk->thread.bad_cause = cause; bad_area(regs, mm, code, addr); return; } @@ -297,6 +322,7 @@ good_area: mmap_read_unlock(mm); if (unlikely(fault & VM_FAULT_ERROR)) { + tsk->thread.bad_cause = cause; mm_fault_error(regs, addr, fault); return; } diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index f9f9568d689e..067583ab1bd7 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "../kernel/head.h" @@ -105,85 +106,19 @@ void __init mem_init(void) print_vm_layout(); } -#ifdef CONFIG_BLK_DEV_INITRD -static void __init setup_initrd(void) -{ - phys_addr_t start; - unsigned long size; - - /* Ignore the virtul address computed during device tree parsing */ - initrd_start = initrd_end = 0; - - if (!phys_initrd_size) - return; - /* - * Round the memory region to page boundaries as per free_initrd_mem() - * This allows us to detect whether the pages overlapping the initrd - * are in use, but more importantly, reserves the entire set of pages - * as we don't want these pages allocated for other purposes. - */ - start = round_down(phys_initrd_start, PAGE_SIZE); - size = phys_initrd_size + (phys_initrd_start - start); - size = round_up(size, PAGE_SIZE); - - if (!memblock_is_region_memory(start, size)) { - pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region", - (u64)start, size); - goto disable; - } - - if (memblock_is_region_reserved(start, size)) { - pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region\n", - (u64)start, size); - goto disable; - } - - memblock_reserve(start, size); - /* Now convert initrd to virtual addresses */ - initrd_start = (unsigned long)__va(phys_initrd_start); - initrd_end = initrd_start + phys_initrd_size; - initrd_below_start_ok = 1; - - pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", - (void *)(initrd_start), size); - return; -disable: - pr_cont(" - disabling initrd\n"); - initrd_start = 0; - initrd_end = 0; -} -#endif /* CONFIG_BLK_DEV_INITRD */ - void __init setup_bootmem(void) { - phys_addr_t mem_start = 0; - phys_addr_t start, dram_end, end = 0; phys_addr_t vmlinux_end = __pa_symbol(&_end); phys_addr_t vmlinux_start = __pa_symbol(&_start); + phys_addr_t dram_end = memblock_end_of_DRAM(); phys_addr_t max_mapped_addr = __pa(~(ulong)0); - u64 i; - /* Find the memory region containing the kernel */ - for_each_mem_range(i, &start, &end) { - phys_addr_t size = end - start; - if (!mem_start) - mem_start = start; - if (start <= vmlinux_start && vmlinux_end <= end) - BUG_ON(size == 0); - } - - /* - * The maximal physical memory size is -PAGE_OFFSET. - * Make sure that any memory beyond mem_start + (-PAGE_OFFSET) is removed - * as it is unusable by kernel. - */ + /* The maximal physical memory size is -PAGE_OFFSET. */ memblock_enforce_memory_limit(-PAGE_OFFSET); /* Reserve from the start of the kernel to the end of the kernel */ memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); - dram_end = memblock_end_of_DRAM(); - /* * memblock allocator is not aware of the fact that last 4K bytes of * the addressable memory can not be mapped because of IS_ERR_VALUE @@ -198,20 +133,19 @@ void __init setup_bootmem(void) dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); -#ifdef CONFIG_BLK_DEV_INITRD - setup_initrd(); -#endif /* CONFIG_BLK_DEV_INITRD */ - + reserve_initrd_mem(); /* - * Avoid using early_init_fdt_reserve_self() since __pa() does + * If DTB is built in, no need to reserve its memblock. + * Otherwise, do reserve it but avoid using + * early_init_fdt_reserve_self() since __pa() does * not work for DTB pointers that are fixmap addresses */ - memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); + if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) + memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); early_init_fdt_scan_reserved_mem(); dma_contiguous_reserve(dma32_phys_limit); memblock_allow_resize(); - memblock_dump_all(); } #ifdef CONFIG_MMU @@ -226,8 +160,6 @@ pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; -#define MAX_EARLY_MAPPING_SIZE SZ_128M - pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) @@ -302,13 +234,7 @@ static void __init create_pte_mapping(pte_t *ptep, pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; - -#if MAX_EARLY_MAPPING_SIZE < PGDIR_SIZE -#define NUM_EARLY_PMDS 1UL -#else -#define NUM_EARLY_PMDS (1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE) -#endif -pmd_t early_pmd[PTRS_PER_PMD * NUM_EARLY_PMDS] __initdata __aligned(PAGE_SIZE); +pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); pmd_t early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) @@ -330,11 +256,9 @@ static pmd_t *get_pmd_virt_late(phys_addr_t pa) static phys_addr_t __init alloc_pmd_early(uintptr_t va) { - uintptr_t pmd_num; + BUG_ON((va - PAGE_OFFSET) >> PGDIR_SHIFT); - pmd_num = (va - PAGE_OFFSET) >> PGDIR_SHIFT; - BUG_ON(pmd_num >= NUM_EARLY_PMDS); - return (uintptr_t)&early_pmd[pmd_num * PTRS_PER_PMD]; + return (uintptr_t)early_pmd; } static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) @@ -452,7 +376,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) uintptr_t va, pa, end_va; uintptr_t load_pa = (uintptr_t)(&_start); uintptr_t load_sz = (uintptr_t)(&_end) - load_pa; - uintptr_t map_size = best_map_size(load_pa, MAX_EARLY_MAPPING_SIZE); + uintptr_t map_size; #ifndef __PAGETABLE_PMD_FOLDED pmd_t fix_bmap_spmd, fix_bmap_epmd; #endif @@ -464,12 +388,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. */ - BUG_ON(map_size == PAGE_SIZE); + map_size = PMD_SIZE; /* Sanity check alignment and size */ BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); BUG_ON((load_pa % map_size) != 0); - BUG_ON(load_sz > MAX_EARLY_MAPPING_SIZE); pt_ops.alloc_pte = alloc_pte_early; pt_ops.get_pte_virt = get_pte_virt_early; @@ -511,6 +434,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) /* Setup early PMD for DTB */ create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, (uintptr_t)early_dtb_pmd, PGDIR_SIZE, PAGE_TABLE); +#ifndef CONFIG_BUILTIN_DTB /* Create two consecutive PMD mappings for FDT early scan */ pa = dtb_pa & ~(PMD_SIZE - 1); create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA, @@ -518,7 +442,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE, pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL); dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1)); +#else /* CONFIG_BUILTIN_DTB */ + dtb_early_va = __va(dtb_pa); +#endif /* CONFIG_BUILTIN_DTB */ #else +#ifndef CONFIG_BUILTIN_DTB /* Create two consecutive PGD mappings for FDT early scan */ pa = dtb_pa & ~(PGDIR_SIZE - 1); create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, @@ -526,6 +454,9 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA + PGDIR_SIZE, pa + PGDIR_SIZE, PGDIR_SIZE, PAGE_KERNEL); dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PGDIR_SIZE - 1)); +#else /* CONFIG_BUILTIN_DTB */ + dtb_early_va = __va(dtb_pa); +#endif /* CONFIG_BUILTIN_DTB */ #endif dtb_early_pa = dtb_pa; @@ -616,15 +547,7 @@ static void __init setup_vm_final(void) #else asmlinkage void __init setup_vm(uintptr_t dtb_pa) { -#ifdef CONFIG_BUILTIN_DTB - dtb_early_va = soc_lookup_builtin_dtb(); - if (!dtb_early_va) { - /* Fallback to first available DTS */ - dtb_early_va = (void *) __dtb_start; - } -#else dtb_early_va = (void *)dtb_pa; -#endif dtb_early_pa = dtb_pa; } @@ -665,9 +588,15 @@ void mark_rodata_ro(void) void __init paging_init(void) { setup_vm_final(); - sparse_init(); setup_zero_page(); +} + +void __init misc_mem_init(void) +{ + arch_numa_init(); + sparse_init(); zone_sizes_init(); + memblock_dump_all(); } #ifdef CONFIG_SPARSEMEM_VMEMMAP diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index a8a2ffd9114a..3fc18f469efb 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -9,6 +9,19 @@ #include #include #include +#include + +static __init void *early_alloc(size_t size, int node) +{ + void *ptr = memblock_alloc_try_nid(size, size, + __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, node); + + if (!ptr) + panic("%pS: Failed to allocate %zu bytes align=%zx nid=%d from=%llx\n", + __func__, size, size, node, (u64)__pa(MAX_DMA_ADDRESS)); + + return ptr; +} extern pgd_t early_pg_dir[PTRS_PER_PGD]; asmlinkage void __init kasan_early_init(void) @@ -47,40 +60,133 @@ asmlinkage void __init kasan_early_init(void) local_flush_tlb_all(); } -static void __init populate(void *start, void *end) +static void kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned long end) +{ + phys_addr_t phys_addr; + pte_t *ptep, *base_pte; + + if (pmd_none(*pmd)) + base_pte = memblock_alloc(PTRS_PER_PTE * sizeof(pte_t), PAGE_SIZE); + else + base_pte = (pte_t *)pmd_page_vaddr(*pmd); + + ptep = base_pte + pte_index(vaddr); + + do { + if (pte_none(*ptep)) { + phys_addr = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); + set_pte(ptep, pfn_pte(PFN_DOWN(phys_addr), PAGE_KERNEL)); + } + } while (ptep++, vaddr += PAGE_SIZE, vaddr != end); + + set_pmd(pmd, pfn_pmd(PFN_DOWN(__pa(base_pte)), PAGE_TABLE)); +} + +static void kasan_populate_pmd(pgd_t *pgd, unsigned long vaddr, unsigned long end) +{ + phys_addr_t phys_addr; + pmd_t *pmdp, *base_pmd; + unsigned long next; + + base_pmd = (pmd_t *)pgd_page_vaddr(*pgd); + if (base_pmd == lm_alias(kasan_early_shadow_pmd)) + base_pmd = memblock_alloc(PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); + + pmdp = base_pmd + pmd_index(vaddr); + + do { + next = pmd_addr_end(vaddr, end); + + if (pmd_none(*pmdp) && IS_ALIGNED(vaddr, PMD_SIZE) && (next - vaddr) >= PMD_SIZE) { + phys_addr = memblock_phys_alloc(PMD_SIZE, PMD_SIZE); + if (phys_addr) { + set_pmd(pmdp, pfn_pmd(PFN_DOWN(phys_addr), PAGE_KERNEL)); + continue; + } + } + + kasan_populate_pte(pmdp, vaddr, next); + } while (pmdp++, vaddr = next, vaddr != end); + + /* + * Wait for the whole PGD to be populated before setting the PGD in + * the page table, otherwise, if we did set the PGD before populating + * it entirely, memblock could allocate a page at a physical address + * where KASAN is not populated yet and then we'd get a page fault. + */ + set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE)); +} + +static void kasan_populate_pgd(unsigned long vaddr, unsigned long end) +{ + phys_addr_t phys_addr; + pgd_t *pgdp = pgd_offset_k(vaddr); + unsigned long next; + + do { + next = pgd_addr_end(vaddr, end); + + /* + * pgdp can't be none since kasan_early_init initialized all KASAN + * shadow region with kasan_early_shadow_pmd: if this is stillthe case, + * that means we can try to allocate a hugepage as a replacement. + */ + if (pgd_page_vaddr(*pgdp) == (unsigned long)lm_alias(kasan_early_shadow_pmd) && + IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) { + phys_addr = memblock_phys_alloc(PGDIR_SIZE, PGDIR_SIZE); + if (phys_addr) { + set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_KERNEL)); + continue; + } + } + + kasan_populate_pmd(pgdp, vaddr, next); + } while (pgdp++, vaddr = next, vaddr != end); +} + +static void __init kasan_populate(void *start, void *end) { - unsigned long i, offset; unsigned long vaddr = (unsigned long)start & PAGE_MASK; unsigned long vend = PAGE_ALIGN((unsigned long)end); - unsigned long n_pages = (vend - vaddr) / PAGE_SIZE; - unsigned long n_ptes = - ((n_pages + PTRS_PER_PTE) & -PTRS_PER_PTE) / PTRS_PER_PTE; - unsigned long n_pmds = - ((n_ptes + PTRS_PER_PMD) & -PTRS_PER_PMD) / PTRS_PER_PMD; - pte_t *pte = - memblock_alloc(n_ptes * PTRS_PER_PTE * sizeof(pte_t), PAGE_SIZE); - pmd_t *pmd = - memblock_alloc(n_pmds * PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); - pgd_t *pgd = pgd_offset_k(vaddr); - - for (i = 0; i < n_pages; i++) { - phys_addr_t phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); - set_pte(&pte[i], pfn_pte(PHYS_PFN(phys), PAGE_KERNEL)); - } - - for (i = 0, offset = 0; i < n_ptes; i++, offset += PTRS_PER_PTE) - set_pmd(&pmd[i], - pfn_pmd(PFN_DOWN(__pa(&pte[offset])), - __pgprot(_PAGE_TABLE))); - - for (i = 0, offset = 0; i < n_pmds; i++, offset += PTRS_PER_PMD) - set_pgd(&pgd[i], - pfn_pgd(PFN_DOWN(__pa(&pmd[offset])), - __pgprot(_PAGE_TABLE))); + kasan_populate_pgd(vaddr, vend); local_flush_tlb_all(); - memset(start, 0, end - start); + memset(start, KASAN_SHADOW_INIT, end - start); +} + +void __init kasan_shallow_populate(void *start, void *end) +{ + unsigned long vaddr = (unsigned long)start & PAGE_MASK; + unsigned long vend = PAGE_ALIGN((unsigned long)end); + unsigned long pfn; + int index; + void *p; + pud_t *pud_dir, *pud_k; + pgd_t *pgd_dir, *pgd_k; + p4d_t *p4d_dir, *p4d_k; + + while (vaddr < vend) { + index = pgd_index(vaddr); + pfn = csr_read(CSR_SATP) & SATP_PPN; + pgd_dir = (pgd_t *)pfn_to_virt(pfn) + index; + pgd_k = init_mm.pgd + index; + pgd_dir = pgd_offset_k(vaddr); + set_pgd(pgd_dir, *pgd_k); + + p4d_dir = p4d_offset(pgd_dir, vaddr); + p4d_k = p4d_offset(pgd_k, vaddr); + + vaddr = (vaddr + PUD_SIZE) & PUD_MASK; + pud_dir = pud_offset(p4d_dir, vaddr); + pud_k = pud_offset(p4d_k, vaddr); + + if (pud_present(*pud_dir)) { + p = early_alloc(PAGE_SIZE, NUMA_NO_NODE); + pud_populate(&init_mm, pud_dir, p); + } + vaddr += PAGE_SIZE; + } } void __init kasan_init(void) @@ -90,7 +196,15 @@ void __init kasan_init(void) kasan_populate_early_shadow((void *)KASAN_SHADOW_START, (void *)kasan_mem_to_shadow((void *) - VMALLOC_END)); + VMEMMAP_END)); + if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) + kasan_shallow_populate( + (void *)kasan_mem_to_shadow((void *)VMALLOC_START), + (void *)kasan_mem_to_shadow((void *)VMALLOC_END)); + else + kasan_populate_early_shadow( + (void *)kasan_mem_to_shadow((void *)VMALLOC_START), + (void *)kasan_mem_to_shadow((void *)VMALLOC_END)); for_each_mem_range(i, &_start, &_end) { void *start = (void *)__va(_start); @@ -99,7 +213,7 @@ void __init kasan_init(void) if (start >= end) break; - populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end)); + kasan_populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end)); }; for (i = 0; i < PTRS_PER_PTE; i++) @@ -108,6 +222,6 @@ void __init kasan_init(void) __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_ACCESSED))); - memset(kasan_early_shadow_page, 0, PAGE_SIZE); + memset(kasan_early_shadow_page, KASAN_SHADOW_INIT, PAGE_SIZE); init_task.kasan_depth = 0; } diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index e8f7216f6c63..c1ff874e6c2e 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -184,6 +184,7 @@ config S390 select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_RSEQ + select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_SYSCALL_TRACEPOINTS select HAVE_VIRT_CPU_ACCOUNTING select HAVE_VIRT_CPU_ACCOUNTING_IDLE diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 6896b4213c49..02056b024091 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -71,7 +71,6 @@ CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_SIG_SHA256=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_BLK_DEV_INTEGRITY=y CONFIG_BLK_DEV_THROTTLING=y CONFIG_BLK_WBT=y diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index ef4df9d60d6a..bac721a501da 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -66,7 +66,6 @@ CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_SIG_SHA256=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_BLK_DEV_THROTTLING=y CONFIG_BLK_WBT=y CONFIG_BLK_CGROUP_IOLATENCY=y diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 73044634d342..54c7536f2482 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -1055,3 +1056,4 @@ MODULE_ALIAS_CRYPTO("aes-all"); MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/arch/s390/include/asm/facility.h b/arch/s390/include/asm/facility.h index 68c476b20b57..91b5d714d28f 100644 --- a/arch/s390/include/asm/facility.h +++ b/arch/s390/include/asm/facility.h @@ -44,7 +44,7 @@ static inline int __test_facility(unsigned long nr, void *facilities) } /* - * The test_facility function uses the bit odering where the MSB is bit 0. + * The test_facility function uses the bit ordering where the MSB is bit 0. * That makes it easier to query facility bits with the bit number as * documented in the Principles of Operation. */ diff --git a/arch/s390/include/asm/hardirq.h b/arch/s390/include/asm/hardirq.h index dfbc3c6c0674..58668ffb5488 100644 --- a/arch/s390/include/asm/hardirq.h +++ b/arch/s390/include/asm/hardirq.h @@ -18,7 +18,6 @@ #define or_softirq_pending(x) (S390_lowcore.softirq_pending |= (x)) #define __ARCH_IRQ_STAT -#define __ARCH_HAS_DO_SOFTIRQ #define __ARCH_IRQ_EXIT_IRQS_DISABLED static inline void ack_bad_irq(unsigned int irq) diff --git a/arch/s390/include/asm/irq_work.h b/arch/s390/include/asm/irq_work.h new file mode 100644 index 000000000000..603783766d0a --- /dev/null +++ b/arch/s390/include/asm/irq_work.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_IRQ_WORK_H +#define _ASM_S390_IRQ_WORK_H + +static inline bool arch_irq_work_has_interrupt(void) +{ + return true; +} + +void arch_irq_work_raise(void); + +#endif /* _ASM_S390_IRQ_WORK_H */ diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h index d1297d6bbdcf..6b187cd72251 100644 --- a/arch/s390/include/asm/pgalloc.h +++ b/arch/s390/include/asm/pgalloc.h @@ -135,7 +135,7 @@ static inline void pmd_populate(struct mm_struct *mm, #define pmd_populate_kernel(mm, pmd, pte) pmd_populate(mm, pmd, pte) #define pmd_pgtable(pmd) \ - (pgtable_t)(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE) + ((pgtable_t)__va(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE)) /* * page table entry allocation/free routines. diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 794746a32806..29c7ecd5ad1d 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1219,8 +1219,8 @@ static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) -#define p4d_deref(pud) (p4d_val(pud) & _REGION_ENTRY_ORIGIN) -#define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) +#define p4d_deref(pud) ((unsigned long)__va(p4d_val(pud) & _REGION_ENTRY_ORIGIN)) +#define pgd_deref(pgd) ((unsigned long)__va(pgd_val(pgd) & _REGION_ENTRY_ORIGIN)) static inline unsigned long pmd_deref(pmd_t pmd) { @@ -1229,12 +1229,12 @@ static inline unsigned long pmd_deref(pmd_t pmd) origin_mask = _SEGMENT_ENTRY_ORIGIN; if (pmd_large(pmd)) origin_mask = _SEGMENT_ENTRY_ORIGIN_LARGE; - return pmd_val(pmd) & origin_mask; + return (unsigned long)__va(pmd_val(pmd) & origin_mask); } static inline unsigned long pmd_pfn(pmd_t pmd) { - return pmd_deref(pmd) >> PAGE_SHIFT; + return __pa(pmd_deref(pmd)) >> PAGE_SHIFT; } static inline unsigned long pud_deref(pud_t pud) @@ -1244,12 +1244,12 @@ static inline unsigned long pud_deref(pud_t pud) origin_mask = _REGION_ENTRY_ORIGIN; if (pud_large(pud)) origin_mask = _REGION3_ENTRY_ORIGIN_LARGE; - return pud_val(pud) & origin_mask; + return (unsigned long)__va(pud_val(pud) & origin_mask); } static inline unsigned long pud_pfn(pud_t pud) { - return pud_deref(pud) >> PAGE_SHIFT; + return __pa(pud_deref(pud)) >> PAGE_SHIFT; } /* @@ -1329,7 +1329,7 @@ static inline bool gup_fast_permitted(unsigned long start, unsigned long end) } #define gup_fast_permitted gup_fast_permitted -#define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot)) +#define pfn_pte(pfn, pgprot) mk_pte_phys(((pfn) << PAGE_SHIFT), (pgprot)) #define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT) #define pte_page(x) pfn_to_page(pte_pfn(x)) @@ -1636,7 +1636,7 @@ static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, } #define pmdp_collapse_flush pmdp_collapse_flush -#define pfn_pmd(pfn, pgprot) mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot)) +#define pfn_pmd(pfn, pgprot) mk_pmd_phys(((pfn) << PAGE_SHIFT), (pgprot)) #define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot)) static inline int pmd_trans_huge(pmd_t pmd) diff --git a/arch/s390/include/uapi/asm/perf_cpum_cf_diag.h b/arch/s390/include/uapi/asm/perf_cpum_cf_diag.h new file mode 100644 index 000000000000..3d8284b95f87 --- /dev/null +++ b/arch/s390/include/uapi/asm/perf_cpum_cf_diag.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright IBM Corp. 2021 + * Interface implementation for communication with the CPU Measurement + * counter facility device driver. + * + * Author(s): Thomas Richter + * + * Define for ioctl() commands to communicate with the CPU Measurement + * counter facility device driver. + */ + +#ifndef _PERF_CPUM_CF_DIAG_H +#define _PERF_CPUM_CF_DIAG_H + +#include +#include + +#define S390_HWCTR_DEVICE "hwctr" +#define S390_HWCTR_START_VERSION 1 + +struct s390_ctrset_start { /* Set CPUs to operate on */ + __u64 version; /* Version of interface */ + __u64 data_bytes; /* # of bytes required */ + __u64 cpumask_len; /* Length of CPU mask in bytes */ + __u64 *cpumask; /* Pointer to CPU mask */ + __u64 counter_sets; /* Bit mask of counter sets to get */ +}; + +struct s390_ctrset_setdata { /* Counter set data */ + __u32 set; /* Counter set number */ + __u32 no_cnts; /* # of counters stored in cv[] */ + __u64 cv[0]; /* Counter values (variable length) */ +}; + +struct s390_ctrset_cpudata { /* Counter set data per CPU */ + __u32 cpu_nr; /* CPU number */ + __u32 no_sets; /* # of counters sets in data[] */ + struct s390_ctrset_setdata data[0]; +}; + +struct s390_ctrset_read { /* Structure to get all ctr sets */ + __u64 no_cpus; /* Total # of CPUs data taken from */ + struct s390_ctrset_cpudata data[0]; +}; + +#define S390_HWCTR_MAGIC 'C' /* Random magic # for ioctls */ +#define S390_HWCTR_START _IOWR(S390_HWCTR_MAGIC, 1, struct s390_ctrset_start) +#define S390_HWCTR_STOP _IO(S390_HWCTR_MAGIC, 2) +#define S390_HWCTR_READ _IOWR(S390_HWCTR_MAGIC, 3, struct s390_ctrset_read) +#endif diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index c6d40bcf4a68..601c21791338 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "entry.h" DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat); diff --git a/arch/s390/kernel/perf_cpum_cf_diag.c b/arch/s390/kernel/perf_cpum_cf_diag.c index b5c86fb70d63..db4877bbb9aa 100644 --- a/arch/s390/kernel/perf_cpum_cf_diag.c +++ b/arch/s390/kernel/perf_cpum_cf_diag.c @@ -2,7 +2,7 @@ /* * Performance event support for s390x - CPU-measurement Counter Sets * - * Copyright IBM Corp. 2019 + * Copyright IBM Corp. 2019, 2021 * Author(s): Hendrik Brueckner * Thomas Richer */ @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include @@ -24,15 +26,20 @@ #include #include -#define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ +#include +#define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ +#define CF_DIAG_MIN_INTERVAL 60 /* Minimum counter set read */ + /* interval in seconds */ +static unsigned long cf_diag_interval = CF_DIAG_MIN_INTERVAL; static unsigned int cf_diag_cpu_speed; static debug_info_t *cf_diag_dbg; -struct cf_diag_csd { /* Counter set data per CPU */ +struct cf_diag_csd { /* Counter set data per CPU */ size_t used; /* Bytes used in data/start */ unsigned char start[PAGE_SIZE]; /* Counter set at event start */ unsigned char data[PAGE_SIZE]; /* Counter set at event delete */ + unsigned int sets; /* # Counter set saved in data */ }; static DEFINE_PER_CPU(struct cf_diag_csd, cf_diag_csd); @@ -178,18 +185,35 @@ static void cf_diag_disable(struct pmu *pmu) /* Number of perf events counting hardware events */ static atomic_t cf_diag_events = ATOMIC_INIT(0); +/* Used to avoid races in calling reserve/release_cpumf_hardware */ +static DEFINE_MUTEX(cf_diag_reserve_mutex); /* Release the PMU if event is the last perf event */ static void cf_diag_perf_event_destroy(struct perf_event *event) { debug_sprintf_event(cf_diag_dbg, 5, "%s event %p cpu %d cf_diag_events %d\n", - __func__, event, event->cpu, + __func__, event, smp_processor_id(), atomic_read(&cf_diag_events)); if (atomic_dec_return(&cf_diag_events) == 0) __kernel_cpumcf_end(); } +static int get_authctrsets(void) +{ + struct cpu_cf_events *cpuhw; + unsigned long auth = 0; + enum cpumf_ctr_set i; + + cpuhw = &get_cpu_var(cpu_cf_events); + for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { + if (cpuhw->info.auth_ctl & cpumf_ctr_ctl[i]) + auth |= cpumf_ctr_ctl[i]; + } + put_cpu_var(cpu_cf_events); + return auth; +} + /* Setup the event. Test for authorized counter sets and only include counter * sets which are authorized at the time of the setup. Including unauthorized * counter sets result in specification exception (and panic). @@ -197,15 +221,12 @@ static void cf_diag_perf_event_destroy(struct perf_event *event) static int __hw_perf_event_init(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; - struct cpu_cf_events *cpuhw; - enum cpumf_ctr_set i; int err = 0; debug_sprintf_event(cf_diag_dbg, 5, "%s event %p cpu %d\n", __func__, event, event->cpu); event->hw.config = attr->config; - event->hw.config_base = 0; /* Add all authorized counter sets to config_base. The * the hardware init function is either called per-cpu or just once @@ -215,11 +236,7 @@ static int __hw_perf_event_init(struct perf_event *event) * Checking the authorization on any CPU is fine as the hardware * applies the same authorization settings to all CPUs. */ - cpuhw = &get_cpu_var(cpu_cf_events); - for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) - if (cpuhw->info.auth_ctl & cpumf_ctr_ctl[i]) - event->hw.config_base |= cpumf_ctr_ctl[i]; - put_cpu_var(cpu_cf_events); + event->hw.config_base = get_authctrsets(); /* No authorized counter sets, nothing to count/sample */ if (!event->hw.config_base) { @@ -237,6 +254,25 @@ out: return err; } +/* Return 0 if the CPU-measurement counter facility is currently free + * and an error otherwise. + */ +static int cf_diag_perf_event_inuse(void) +{ + int err = 0; + + if (!atomic_inc_not_zero(&cf_diag_events)) { + mutex_lock(&cf_diag_reserve_mutex); + if (atomic_read(&cf_diag_events) == 0 && + __kernel_cpumcf_begin()) + err = -EBUSY; + else + err = atomic_inc_return(&cf_diag_events); + mutex_unlock(&cf_diag_reserve_mutex); + } + return err; +} + static int cf_diag_event_init(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; @@ -264,13 +300,9 @@ static int cf_diag_event_init(struct perf_event *event) } /* Initialize for using the CPU-measurement counter facility */ - if (atomic_inc_return(&cf_diag_events) == 1) { - if (__kernel_cpumcf_begin()) { - atomic_dec(&cf_diag_events); - err = -EBUSY; - goto out; - } - } + err = cf_diag_perf_event_inuse(); + if (err < 0) + goto out; event->destroy = cf_diag_perf_event_destroy; err = __hw_perf_event_init(event); @@ -599,6 +631,8 @@ static void cf_diag_del(struct perf_event *event, int flags) cpuhw->flags &= ~PMU_F_IN_USE; } +/* Default counter set events and format attribute groups */ + CPUMF_EVENT_ATTR(CF_DIAG, CF_DIAG, PERF_EVENT_CPUM_CF_DIAG); static struct attribute *cf_diag_events_attr[] = { @@ -663,6 +697,452 @@ static void cf_diag_get_cpu_speed(void) } } +/* Code to create device and file I/O operations */ +static atomic_t ctrset_opencnt = ATOMIC_INIT(0); /* Excl. access */ + +static int cf_diag_open(struct inode *inode, struct file *file) +{ + int err = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (atomic_xchg(&ctrset_opencnt, 1)) + return -EBUSY; + + /* Avoid concurrent access with perf_event_open() system call */ + mutex_lock(&cf_diag_reserve_mutex); + if (atomic_read(&cf_diag_events) || __kernel_cpumcf_begin()) + err = -EBUSY; + mutex_unlock(&cf_diag_reserve_mutex); + if (err) { + atomic_set(&ctrset_opencnt, 0); + return err; + } + file->private_data = NULL; + debug_sprintf_event(cf_diag_dbg, 2, "%s\n", __func__); + /* nonseekable_open() never fails */ + return nonseekable_open(inode, file); +} + +/* Variables for ioctl() interface support */ +static DEFINE_MUTEX(cf_diag_ctrset_mutex); +static struct cf_diag_ctrset { + unsigned long ctrset; /* Bit mask of counter set to read */ + cpumask_t mask; /* CPU mask to read from */ + time64_t lastread; /* Epoch counter set last read */ +} cf_diag_ctrset; + +static void cf_diag_ctrset_clear(void) +{ + cpumask_clear(&cf_diag_ctrset.mask); + cf_diag_ctrset.ctrset = 0; +} + +static void cf_diag_release_cpu(void *p) +{ + struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + + debug_sprintf_event(cf_diag_dbg, 3, "%s cpu %d\n", __func__, + smp_processor_id()); + lcctl(0); /* Reset counter sets */ + cpuhw->state = 0; /* Save state in CPU hardware state */ +} + +/* Release function is also called when application gets terminated without + * doing a proper ioctl(..., S390_HWCTR_STOP, ...) command. + * Since only one application is allowed to open the device, simple stop all + * CPU counter sets. + */ +static int cf_diag_release(struct inode *inode, struct file *file) +{ + on_each_cpu(cf_diag_release_cpu, NULL, 1); + cf_diag_ctrset_clear(); + atomic_set(&ctrset_opencnt, 0); + __kernel_cpumcf_end(); + debug_sprintf_event(cf_diag_dbg, 2, "%s\n", __func__); + return 0; +} + +struct cf_diag_call_on_cpu_parm { /* Parm struct for smp_call_on_cpu */ + unsigned int sets; /* Counter set bit mask */ + atomic_t cpus_ack; /* # CPUs successfully executed func */ +}; + +static int cf_diag_all_copy(unsigned long arg, cpumask_t *mask) +{ + struct s390_ctrset_read __user *ctrset_read; + unsigned int cpu, cpus, rc; + void __user *uptr; + + ctrset_read = (struct s390_ctrset_read __user *)arg; + uptr = ctrset_read->data; + for_each_cpu(cpu, mask) { + struct cf_diag_csd *csd = per_cpu_ptr(&cf_diag_csd, cpu); + struct s390_ctrset_cpudata __user *ctrset_cpudata; + + ctrset_cpudata = uptr; + debug_sprintf_event(cf_diag_dbg, 5, "%s cpu %d used %zd\n", + __func__, cpu, csd->used); + rc = put_user(cpu, &ctrset_cpudata->cpu_nr); + rc |= put_user(csd->sets, &ctrset_cpudata->no_sets); + rc |= copy_to_user(ctrset_cpudata->data, csd->data, csd->used); + if (rc) + return -EFAULT; + uptr += sizeof(struct s390_ctrset_cpudata) + csd->used; + cond_resched(); + } + cpus = cpumask_weight(mask); + if (put_user(cpus, &ctrset_read->no_cpus)) + return -EFAULT; + debug_sprintf_event(cf_diag_dbg, 5, "%s copied %ld\n", + __func__, uptr - (void __user *)ctrset_read->data); + return 0; +} + +static size_t cf_diag_cpuset_read(struct s390_ctrset_setdata *p, int ctrset, + int ctrset_size, size_t room) +{ + size_t need = 0; + int rc = -1; + + need = sizeof(*p) + sizeof(u64) * ctrset_size; + debug_sprintf_event(cf_diag_dbg, 5, + "%s room %zd need %zd set %#x set_size %d\n", + __func__, room, need, ctrset, ctrset_size); + if (need <= room) { + p->set = cpumf_ctr_ctl[ctrset]; + p->no_cnts = ctrset_size; + rc = ctr_stcctm(ctrset, ctrset_size, (u64 *)p->cv); + if (rc == 3) /* Nothing stored */ + need = 0; + } + debug_sprintf_event(cf_diag_dbg, 5, "%s need %zd rc %d\n", __func__, + need, rc); + return need; +} + +/* Read all counter sets. Since the perf_event_open() system call with + * event cpum_cf_diag/.../ is blocked when this interface is active, reuse + * the perf_event_open() data buffer to store the counter sets. + */ +static void cf_diag_cpu_read(void *parm) +{ + struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cf_diag_csd *csd = this_cpu_ptr(&cf_diag_csd); + struct cf_diag_call_on_cpu_parm *p = parm; + int set, set_size; + size_t space; + + debug_sprintf_event(cf_diag_dbg, 5, + "%s new %#x flags %#x state %#llx\n", + __func__, p->sets, cpuhw->flags, + cpuhw->state); + /* No data saved yet */ + csd->used = 0; + csd->sets = 0; + memset(csd->data, 0, sizeof(csd->data)); + + /* Scan the counter sets */ + for (set = CPUMF_CTR_SET_BASIC; set < CPUMF_CTR_SET_MAX; ++set) { + struct s390_ctrset_setdata *sp = (void *)csd->data + csd->used; + + if (!(p->sets & cpumf_ctr_ctl[set])) + continue; /* Counter set not in list */ + set_size = cf_diag_ctrset_size(set, &cpuhw->info); + space = sizeof(csd->data) - csd->used; + space = cf_diag_cpuset_read(sp, set, set_size, space); + if (space) { + csd->used += space; + csd->sets += 1; + } + debug_sprintf_event(cf_diag_dbg, 5, "%s sp %px space %zd\n", + __func__, sp, space); + } + debug_sprintf_event(cf_diag_dbg, 5, "%s sets %d used %zd\n", __func__, + csd->sets, csd->used); +} + +static int cf_diag_all_read(unsigned long arg) +{ + struct cf_diag_call_on_cpu_parm p; + cpumask_var_t mask; + time64_t now; + int rc = 0; + + debug_sprintf_event(cf_diag_dbg, 5, "%s\n", __func__); + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) + return -ENOMEM; + now = ktime_get_seconds(); + if (cf_diag_ctrset.lastread + cf_diag_interval > now) { + debug_sprintf_event(cf_diag_dbg, 5, "%s now %lld " + " lastread %lld\n", __func__, now, + cf_diag_ctrset.lastread); + rc = -EAGAIN; + goto out; + } else { + cf_diag_ctrset.lastread = now; + } + p.sets = cf_diag_ctrset.ctrset; + cpumask_and(mask, &cf_diag_ctrset.mask, cpu_online_mask); + on_each_cpu_mask(mask, cf_diag_cpu_read, &p, 1); + rc = cf_diag_all_copy(arg, mask); +out: + free_cpumask_var(mask); + debug_sprintf_event(cf_diag_dbg, 5, "%s rc %d\n", __func__, rc); + return rc; +} + +/* Stop all counter sets via ioctl interface */ +static void cf_diag_ioctl_off(void *parm) +{ + struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cf_diag_call_on_cpu_parm *p = parm; + int rc; + + debug_sprintf_event(cf_diag_dbg, 5, + "%s new %#x flags %#x state %#llx\n", + __func__, p->sets, cpuhw->flags, + cpuhw->state); + + ctr_set_multiple_disable(&cpuhw->state, p->sets); + ctr_set_multiple_stop(&cpuhw->state, p->sets); + rc = lcctl(cpuhw->state); /* Stop counter sets */ + if (!cpuhw->state) + cpuhw->flags &= ~PMU_F_IN_USE; + debug_sprintf_event(cf_diag_dbg, 5, + "%s rc %d flags %#x state %#llx\n", __func__, + rc, cpuhw->flags, cpuhw->state); +} + +/* Start counter sets on particular CPU */ +static void cf_diag_ioctl_on(void *parm) +{ + struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cf_diag_call_on_cpu_parm *p = parm; + int rc; + + debug_sprintf_event(cf_diag_dbg, 5, + "%s new %#x flags %#x state %#llx\n", + __func__, p->sets, cpuhw->flags, + cpuhw->state); + + if (!(cpuhw->flags & PMU_F_IN_USE)) + cpuhw->state = 0; + cpuhw->flags |= PMU_F_IN_USE; + rc = lcctl(cpuhw->state); /* Reset unused counter sets */ + ctr_set_multiple_enable(&cpuhw->state, p->sets); + ctr_set_multiple_start(&cpuhw->state, p->sets); + rc |= lcctl(cpuhw->state); /* Start counter sets */ + if (!rc) + atomic_inc(&p->cpus_ack); + debug_sprintf_event(cf_diag_dbg, 5, "%s rc %d state %#llx\n", + __func__, rc, cpuhw->state); +} + +static int cf_diag_all_stop(void) +{ + struct cf_diag_call_on_cpu_parm p = { + .sets = cf_diag_ctrset.ctrset, + }; + cpumask_var_t mask; + + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) + return -ENOMEM; + cpumask_and(mask, &cf_diag_ctrset.mask, cpu_online_mask); + on_each_cpu_mask(mask, cf_diag_ioctl_off, &p, 1); + free_cpumask_var(mask); + return 0; +} + +static int cf_diag_all_start(void) +{ + struct cf_diag_call_on_cpu_parm p = { + .sets = cf_diag_ctrset.ctrset, + .cpus_ack = ATOMIC_INIT(0), + }; + cpumask_var_t mask; + int rc = 0; + + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) + return -ENOMEM; + cpumask_and(mask, &cf_diag_ctrset.mask, cpu_online_mask); + on_each_cpu_mask(mask, cf_diag_ioctl_on, &p, 1); + if (atomic_read(&p.cpus_ack) != cpumask_weight(mask)) { + on_each_cpu_mask(mask, cf_diag_ioctl_off, &p, 1); + rc = -EIO; + } + free_cpumask_var(mask); + return rc; +} + +/* Return the maximum required space for all possible CPUs in case one + * CPU will be onlined during the START, READ, STOP cycles. + * To find out the size of the counter sets, any one CPU will do. They + * all have the same counter sets. + */ +static size_t cf_diag_needspace(unsigned int sets) +{ + struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + size_t bytes = 0; + int i; + + for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { + if (!(sets & cpumf_ctr_ctl[i])) + continue; + bytes += cf_diag_ctrset_size(i, &cpuhw->info) * sizeof(u64) + + sizeof(((struct s390_ctrset_setdata *)0)->set) + + sizeof(((struct s390_ctrset_setdata *)0)->no_cnts); + } + bytes = sizeof(((struct s390_ctrset_read *)0)->no_cpus) + nr_cpu_ids * + (bytes + sizeof(((struct s390_ctrset_cpudata *)0)->cpu_nr) + + sizeof(((struct s390_ctrset_cpudata *)0)->no_sets)); + debug_sprintf_event(cf_diag_dbg, 5, "%s bytes %ld\n", __func__, + bytes); + return bytes; +} + +static long cf_diag_ioctl_read(unsigned long arg) +{ + struct s390_ctrset_read read; + int ret = 0; + + debug_sprintf_event(cf_diag_dbg, 5, "%s\n", __func__); + if (copy_from_user(&read, (char __user *)arg, sizeof(read))) + return -EFAULT; + ret = cf_diag_all_read(arg); + debug_sprintf_event(cf_diag_dbg, 5, "%s ret %d\n", __func__, ret); + return ret; +} + +static long cf_diag_ioctl_stop(void) +{ + int ret; + + debug_sprintf_event(cf_diag_dbg, 5, "%s\n", __func__); + ret = cf_diag_all_stop(); + cf_diag_ctrset_clear(); + debug_sprintf_event(cf_diag_dbg, 5, "%s ret %d\n", __func__, ret); + return ret; +} + +static long cf_diag_ioctl_start(unsigned long arg) +{ + struct s390_ctrset_start __user *ustart; + struct s390_ctrset_start start; + void __user *umask; + unsigned int len; + int ret = 0; + size_t need; + + if (cf_diag_ctrset.ctrset) + return -EBUSY; + ustart = (struct s390_ctrset_start __user *)arg; + if (copy_from_user(&start, ustart, sizeof(start))) + return -EFAULT; + if (start.version != S390_HWCTR_START_VERSION) + return -EINVAL; + if (start.counter_sets & ~(cpumf_ctr_ctl[CPUMF_CTR_SET_BASIC] | + cpumf_ctr_ctl[CPUMF_CTR_SET_USER] | + cpumf_ctr_ctl[CPUMF_CTR_SET_CRYPTO] | + cpumf_ctr_ctl[CPUMF_CTR_SET_EXT] | + cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG])) + return -EINVAL; /* Invalid counter set */ + if (!start.counter_sets) + return -EINVAL; /* No counter set at all? */ + cpumask_clear(&cf_diag_ctrset.mask); + len = min_t(u64, start.cpumask_len, cpumask_size()); + umask = (void __user *)start.cpumask; + if (copy_from_user(&cf_diag_ctrset.mask, umask, len)) + return -EFAULT; + if (cpumask_empty(&cf_diag_ctrset.mask)) + return -EINVAL; + need = cf_diag_needspace(start.counter_sets); + if (put_user(need, &ustart->data_bytes)) + ret = -EFAULT; + if (ret) + goto out; + cf_diag_ctrset.ctrset = start.counter_sets; + ret = cf_diag_all_start(); +out: + if (ret) + cf_diag_ctrset_clear(); + debug_sprintf_event(cf_diag_dbg, 2, "%s sets %#lx need %ld ret %d\n", + __func__, cf_diag_ctrset.ctrset, need, ret); + return ret; +} + +static long cf_diag_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + int ret; + + debug_sprintf_event(cf_diag_dbg, 2, "%s cmd %#x arg %lx\n", __func__, + cmd, arg); + get_online_cpus(); + mutex_lock(&cf_diag_ctrset_mutex); + switch (cmd) { + case S390_HWCTR_START: + ret = cf_diag_ioctl_start(arg); + break; + case S390_HWCTR_STOP: + ret = cf_diag_ioctl_stop(); + break; + case S390_HWCTR_READ: + ret = cf_diag_ioctl_read(arg); + break; + default: + ret = -ENOTTY; + break; + } + mutex_unlock(&cf_diag_ctrset_mutex); + put_online_cpus(); + debug_sprintf_event(cf_diag_dbg, 2, "%s ret %d\n", __func__, ret); + return ret; +} + +static const struct file_operations cf_diag_fops = { + .owner = THIS_MODULE, + .open = cf_diag_open, + .release = cf_diag_release, + .unlocked_ioctl = cf_diag_ioctl, + .compat_ioctl = cf_diag_ioctl, + .llseek = no_llseek +}; + +static struct miscdevice cf_diag_dev = { + .name = S390_HWCTR_DEVICE, + .minor = MISC_DYNAMIC_MINOR, + .fops = &cf_diag_fops, +}; + +static int cf_diag_online_cpu(unsigned int cpu) +{ + struct cf_diag_call_on_cpu_parm p; + + mutex_lock(&cf_diag_ctrset_mutex); + if (!cf_diag_ctrset.ctrset) + goto out; + p.sets = cf_diag_ctrset.ctrset; + cf_diag_ioctl_on(&p); +out: + mutex_unlock(&cf_diag_ctrset_mutex); + return 0; +} + +static int cf_diag_offline_cpu(unsigned int cpu) +{ + struct cf_diag_call_on_cpu_parm p; + + mutex_lock(&cf_diag_ctrset_mutex); + if (!cf_diag_ctrset.ctrset) + goto out; + p.sets = cf_diag_ctrset.ctrset; + cf_diag_ioctl_off(&p); +out: + mutex_unlock(&cf_diag_ctrset_mutex); + return 0; +} + /* Initialize the counter set PMU to generate complete counter set data as * event raw data. This relies on the CPU Measurement Counter Facility device * already being loaded and initialized. @@ -685,21 +1165,43 @@ static int __init cf_diag_init(void) return -ENOMEM; } + rc = misc_register(&cf_diag_dev); + if (rc) { + pr_err("Registration of /dev/" S390_HWCTR_DEVICE + "failed rc=%d\n", rc); + goto out; + } + /* Setup s390dbf facility */ cf_diag_dbg = debug_register(KMSG_COMPONENT, 2, 1, 128); if (!cf_diag_dbg) { pr_err("Registration of s390dbf(cpum_cf_diag) failed\n"); - return -ENOMEM; + rc = -ENOMEM; + goto out_dbf; } debug_register_view(cf_diag_dbg, &debug_sprintf_view); rc = perf_pmu_register(&cf_diag, "cpum_cf_diag", -1); if (rc) { - debug_unregister_view(cf_diag_dbg, &debug_sprintf_view); - debug_unregister(cf_diag_dbg); pr_err("Registration of PMU(cpum_cf_diag) failed with rc=%i\n", rc); + goto out_perf; } + rc = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_S390_CFD_ONLINE, + "perf/s390/cfd:online", + cf_diag_online_cpu, cf_diag_offline_cpu); + if (!rc) + goto out; + + pr_err("Registration of CPUHP_AP_PERF_S390_CFD_ONLINE failed rc=%i\n", + rc); + perf_pmu_unregister(&cf_diag); +out_perf: + debug_unregister_view(cf_diag_dbg, &debug_sprintf_view); + debug_unregister(cf_diag_dbg); +out_dbf: + misc_deregister(&cf_diag_dev); +out: return rc; } -arch_initcall(cf_diag_init); +device_initcall(cf_diag_init); diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 367bd000f6d1..e20bed1ed34a 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -130,7 +130,7 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp, frame->sf.gprs[9] = (unsigned long)frame; /* Store access registers to kernel stack of new process. */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ memset(&frame->childregs, 0, sizeof(struct pt_regs)); frame->childregs.psw.mask = PSW_KERNEL_BITS | PSW_MASK_DAT | diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index e299892440b6..58c8afa3da65 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -62,6 +63,7 @@ enum { ec_call_function_single, ec_stop_cpu, ec_mcck_pending, + ec_irq_work, }; enum { @@ -434,10 +436,12 @@ void notrace smp_yield_cpu(int cpu) */ void notrace smp_emergency_stop(void) { - cpumask_t cpumask; + static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED; + static cpumask_t cpumask; u64 end; int cpu; + arch_spin_lock(&lock); cpumask_copy(&cpumask, cpu_online_mask); cpumask_clear_cpu(smp_processor_id(), &cpumask); @@ -458,6 +462,7 @@ void notrace smp_emergency_stop(void) break; cpu_relax(); } + arch_spin_unlock(&lock); } NOKPROBE_SYMBOL(smp_emergency_stop); @@ -505,6 +510,8 @@ static void smp_handle_ext_call(void) generic_smp_call_function_single_interrupt(); if (test_bit(ec_mcck_pending, &bits)) __s390_handle_mcck(); + if (test_bit(ec_irq_work, &bits)) + irq_work_run(); } static void do_ext_call_interrupt(struct ext_code ext_code, @@ -537,6 +544,13 @@ void smp_send_reschedule(int cpu) pcpu_ec_call(pcpu_devices + cpu, ec_schedule); } +#ifdef CONFIG_IRQ_WORK +void arch_irq_work_raise(void) +{ + pcpu_ec_call(pcpu_devices + smp_processor_id(), ec_irq_work); +} +#endif + /* * parameter area for the set/clear control bit callbacks */ @@ -775,11 +789,13 @@ static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail, static int __smp_rescan_cpus(struct sclp_core_info *info, bool early) { struct sclp_core_entry *core; - cpumask_t avail; + static cpumask_t avail; bool configured; u16 core_id; int nr, i; + get_online_cpus(); + mutex_lock(&smp_cpu_state_mutex); nr = 0; cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask); /* @@ -800,6 +816,8 @@ static int __smp_rescan_cpus(struct sclp_core_info *info, bool early) configured = i < info->configured; nr += smp_add_core(&info->core[i], &avail, configured, early); } + mutex_unlock(&smp_cpu_state_mutex); + put_online_cpus(); return nr; } @@ -847,9 +865,7 @@ void __init smp_detect_cpus(void) pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus); /* Add CPUs present at boot */ - get_online_cpus(); __smp_rescan_cpus(info, true); - put_online_cpus(); memblock_free_early((unsigned long)info, sizeof(*info)); } @@ -1178,11 +1194,7 @@ int __ref smp_rescan_cpus(void) if (!info) return -ENOMEM; smp_get_core_info(info, 0); - get_online_cpus(); - mutex_lock(&smp_cpu_state_mutex); nr = __smp_rescan_cpus(info, false); - mutex_unlock(&smp_cpu_state_mutex); - put_online_cpus(); kfree(info); if (nr) topology_schedule_update(); diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl index d443423495e5..3abef2144dac 100644 --- a/arch/s390/kernel/syscalls/syscall.tbl +++ b/arch/s390/kernel/syscalls/syscall.tbl @@ -444,3 +444,4 @@ 439 common faccessat2 sys_faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr sys_mount_setattr diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index ca47141a5be9..e7ce447651b9 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -62,16 +62,16 @@ static struct mask_info drawer_info; struct cpu_topology_s390 cpu_topology[NR_CPUS]; EXPORT_SYMBOL_GPL(cpu_topology); -static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu) +static void cpu_group_map(cpumask_t *dst, struct mask_info *info, unsigned int cpu) { - cpumask_t mask; + static cpumask_t mask; cpumask_copy(&mask, cpumask_of(cpu)); switch (topology_mode) { case TOPOLOGY_MODE_HW: while (info) { if (cpumask_test_cpu(cpu, &info->mask)) { - mask = info->mask; + cpumask_copy(&mask, &info->mask); break; } info = info->next; @@ -89,23 +89,24 @@ static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu) break; } cpumask_and(&mask, &mask, cpu_online_mask); - return mask; + cpumask_copy(dst, &mask); } -static cpumask_t cpu_thread_map(unsigned int cpu) +static void cpu_thread_map(cpumask_t *dst, unsigned int cpu) { - cpumask_t mask; + static cpumask_t mask; int i; cpumask_copy(&mask, cpumask_of(cpu)); if (topology_mode != TOPOLOGY_MODE_HW) - return mask; + goto out; cpu -= cpu % (smp_cpu_mtid + 1); for (i = 0; i <= smp_cpu_mtid; i++) if (cpu_present(cpu + i)) cpumask_set_cpu(cpu + i, &mask); cpumask_and(&mask, &mask, cpu_online_mask); - return mask; +out: + cpumask_copy(dst, &mask); } #define TOPOLOGY_CORE_BITS 64 @@ -250,10 +251,10 @@ void update_cpu_masks(void) for_each_possible_cpu(cpu) { topo = &cpu_topology[cpu]; - topo->thread_mask = cpu_thread_map(cpu); - topo->core_mask = cpu_group_map(&socket_info, cpu); - topo->book_mask = cpu_group_map(&book_info, cpu); - topo->drawer_mask = cpu_group_map(&drawer_info, cpu); + cpu_thread_map(&topo->thread_mask, cpu); + cpu_group_map(&topo->core_mask, &socket_info, cpu); + cpu_group_map(&topo->book_mask, &book_info, cpu); + cpu_group_map(&topo->drawer_mask, &drawer_info, cpu); topo->booted_cores = 0; if (topology_mode != TOPOLOGY_MODE_HW) { id = topology_mode == TOPOLOGY_MODE_PACKAGE ? 0 : cpu; diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 73a163065b95..0e76b2127dc6 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -297,6 +297,7 @@ int arch_add_memory(int nid, u64 start, u64 size, if (WARN_ON_ONCE(params->pgprot.pgprot != PAGE_KERNEL.pgprot)) return -EINVAL; + VM_BUG_ON(!mhp_range_allowed(start, size, true)); rc = vmem_add_mapping(start, size); if (rc) return rc; diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 4e87c819ddea..781965f7210e 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -58,7 +58,7 @@ unsigned long *crst_table_alloc(struct mm_struct *mm) if (!page) return NULL; arch_set_page_dat(page, 2); - return (unsigned long *) page_to_phys(page); + return (unsigned long *) page_to_virt(page); } void crst_table_free(struct mm_struct *mm, unsigned long *table) @@ -161,7 +161,7 @@ struct page *page_table_alloc_pgste(struct mm_struct *mm) page = alloc_page(GFP_KERNEL); if (page) { - table = (u64 *)page_to_phys(page); + table = (u64 *)page_to_virt(page); memset64(table, _PAGE_INVALID, PTRS_PER_PTE); memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE); } @@ -194,7 +194,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm) mask = atomic_read(&page->_refcount) >> 24; mask = (mask | (mask >> 4)) & 3; if (mask != 3) { - table = (unsigned long *) page_to_phys(page); + table = (unsigned long *) page_to_virt(page); bit = mask & 1; /* =1 -> second 2K */ if (bit) table += PTRS_PER_PTE; @@ -217,7 +217,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm) } arch_set_page_dat(page, 0); /* Initialize page table */ - table = (unsigned long *) page_to_phys(page); + table = (unsigned long *) page_to_virt(page); if (mm_alloc_pgste(mm)) { /* Return 4K page table with PGSTEs */ atomic_xor_bits(&page->_refcount, 3 << 24); @@ -239,10 +239,10 @@ void page_table_free(struct mm_struct *mm, unsigned long *table) struct page *page; unsigned int bit, mask; - page = pfn_to_page(__pa(table) >> PAGE_SHIFT); + page = virt_to_page(table); if (!mm_alloc_pgste(mm)) { /* Free 2K page table fragment of a 4K page */ - bit = (__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)); + bit = ((unsigned long) table & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)); spin_lock_bh(&mm->context.lock); mask = atomic_xor_bits(&page->_refcount, 1U << (bit + 24)); mask >>= 24; @@ -269,14 +269,14 @@ void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table, unsigned int bit, mask; mm = tlb->mm; - page = pfn_to_page(__pa(table) >> PAGE_SHIFT); + page = virt_to_page(table); if (mm_alloc_pgste(mm)) { gmap_unlink(mm, table, vmaddr); - table = (unsigned long *) (__pa(table) | 3); + table = (unsigned long *) ((unsigned long)table | 3); tlb_remove_table(tlb, table); return; } - bit = (__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)); + bit = ((unsigned long) table & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)); spin_lock_bh(&mm->context.lock); mask = atomic_xor_bits(&page->_refcount, 0x11U << (bit + 24)); mask >>= 24; @@ -285,7 +285,7 @@ void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table, else list_del(&page->lru); spin_unlock_bh(&mm->context.lock); - table = (unsigned long *) (__pa(table) | (1U << bit)); + table = (unsigned long *) ((unsigned long) table | (1U << bit)); tlb_remove_table(tlb, table); } @@ -293,7 +293,7 @@ void __tlb_remove_table(void *_table) { unsigned int mask = (unsigned long) _table & 3; void *table = (void *)((unsigned long) _table ^ mask); - struct page *page = pfn_to_page(__pa(table) >> PAGE_SHIFT); + struct page *page = virt_to_page(table); switch (mask) { case 0: /* pmd, pud, or p4d */ diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 01f3a5f58e64..96897fab89dc 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -4,6 +4,7 @@ * Author(s): Heiko Carstens */ +#include #include #include #include @@ -26,14 +27,14 @@ static void __ref *vmem_alloc_pages(unsigned int order) if (slab_is_available()) return (void *)__get_free_pages(GFP_KERNEL, order); - return (void *) memblock_phys_alloc(size, size); + return memblock_alloc(size, size); } static void vmem_free_pages(unsigned long addr, int order) { /* We don't expect boot memory to be removed ever. */ if (!slab_is_available() || - WARN_ON_ONCE(PageReserved(phys_to_page(addr)))) + WARN_ON_ONCE(PageReserved(virt_to_page(addr)))) return; free_pages(addr, order); } @@ -56,7 +57,7 @@ pte_t __ref *vmem_pte_alloc(void) if (slab_is_available()) pte = (pte_t *) page_table_alloc(&init_mm); else - pte = (pte_t *) memblock_phys_alloc(size, size); + pte = (pte_t *) memblock_alloc(size, size); if (!pte) return NULL; memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE); @@ -84,7 +85,7 @@ static void vmemmap_flush_unused_sub_pmd(void) { if (!unused_sub_pmd_start) return; - memset(__va(unused_sub_pmd_start), PAGE_UNUSED, + memset((void *)unused_sub_pmd_start, PAGE_UNUSED, ALIGN(unused_sub_pmd_start, PMD_SIZE) - unused_sub_pmd_start); unused_sub_pmd_start = 0; } @@ -97,7 +98,7 @@ static void vmemmap_mark_sub_pmd_used(unsigned long start, unsigned long end) * getting removed (just in case the memmap never gets initialized, * e.g., because the memory block never gets onlined). */ - memset(__va(start), 0, sizeof(struct page)); + memset((void *)start, 0, sizeof(struct page)); } static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end) @@ -118,7 +119,7 @@ static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end) static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) { - void *page = __va(ALIGN_DOWN(start, PMD_SIZE)); + unsigned long page = ALIGN_DOWN(start, PMD_SIZE); vmemmap_flush_unused_sub_pmd(); @@ -127,7 +128,7 @@ static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) /* Mark the unused parts of the new memmap page PAGE_UNUSED. */ if (!IS_ALIGNED(start, PMD_SIZE)) - memset(page, PAGE_UNUSED, start - __pa(page)); + memset((void *)page, PAGE_UNUSED, start - page); /* * We want to avoid memset(PAGE_UNUSED) when populating the vmemmap of * consecutive sections. Remember for the last added PMD the last @@ -140,11 +141,11 @@ static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) /* Returns true if the PMD is completely unused and can be freed. */ static bool vmemmap_unuse_sub_pmd(unsigned long start, unsigned long end) { - void *page = __va(ALIGN_DOWN(start, PMD_SIZE)); + unsigned long page = ALIGN_DOWN(start, PMD_SIZE); vmemmap_flush_unused_sub_pmd(); - memset(__va(start), PAGE_UNUSED, end - start); - return !memchr_inv(page, PAGE_UNUSED, PMD_SIZE); + memset((void *)start, PAGE_UNUSED, end - start); + return !memchr_inv((void *)page, PAGE_UNUSED, PMD_SIZE); } /* __ref: we'll only call vmemmap_alloc_block() via vmemmap_populate() */ @@ -165,7 +166,7 @@ static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr, if (pte_none(*pte)) continue; if (!direct) - vmem_free_pages(pfn_to_phys(pte_pfn(*pte)), 0); + vmem_free_pages((unsigned long) pfn_to_virt(pte_pfn(*pte)), 0); pte_clear(&init_mm, addr, pte); } else if (pte_none(*pte)) { if (!direct) { @@ -175,7 +176,7 @@ static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr, goto out; pte_val(*pte) = __pa(new_page) | prot; } else { - pte_val(*pte) = addr | prot; + pte_val(*pte) = __pa(addr) | prot; } } else { continue; @@ -200,7 +201,7 @@ static void try_free_pte_table(pmd_t *pmd, unsigned long start) if (!pte_none(*pte)) return; } - vmem_pte_free(__va(pmd_deref(*pmd))); + vmem_pte_free((unsigned long *) pmd_deref(*pmd)); pmd_clear(pmd); } @@ -241,7 +242,7 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr, IS_ALIGNED(next, PMD_SIZE) && MACHINE_HAS_EDAT1 && addr && direct && !debug_pagealloc_enabled()) { - pmd_val(*pmd) = addr | prot; + pmd_val(*pmd) = __pa(addr) | prot; pages++; continue; } else if (!direct && MACHINE_HAS_EDAT1) { @@ -337,7 +338,7 @@ static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, IS_ALIGNED(next, PUD_SIZE) && MACHINE_HAS_EDAT2 && addr && direct && !debug_pagealloc_enabled()) { - pud_val(*pud) = addr | prot; + pud_val(*pud) = __pa(addr) | prot; pages++; continue; } @@ -532,11 +533,22 @@ void vmem_remove_mapping(unsigned long start, unsigned long size) mutex_unlock(&vmem_mutex); } +struct range arch_get_mappable_range(void) +{ + struct range mhp_range; + + mhp_range.start = 0; + mhp_range.end = VMEM_MAX_PHYS - 1; + return mhp_range; +} + int vmem_add_mapping(unsigned long start, unsigned long size) { + struct range range = arch_get_mappable_range(); int ret; - if (start + size > VMEM_MAX_PHYS || + if (start < range.start || + start + size > range.end + 1 || start + size < start) return -ERANGE; diff --git a/arch/s390/tools/opcodes.txt b/arch/s390/tools/opcodes.txt index 46d8ed96cf06..0e207c46e8da 100644 --- a/arch/s390/tools/opcodes.txt +++ b/arch/s390/tools/opcodes.txt @@ -597,7 +597,7 @@ b9b3 cu42 RRE_RR b9bd trtre RRF_U0RR b9be srstu RRE_RR b9bf trte RRF_U0RR -b9c0 selhhhr RRF_RURR +b9c0 selfhr RRF_RURR b9c8 ahhhr RRF_R0RR2 b9c9 shhhr RRF_R0RR2 b9ca alhhhr RRF_R0RR2 diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 7ac847ca6356..e798e55915c2 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -54,6 +54,7 @@ config SUPERH select HAVE_PERF_EVENTS select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_UID16 + select HAVE_SOFTIRQ_ON_OWN_STACK if IRQSTACKS select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select IRQ_FORCED_THREADING diff --git a/arch/sh/boards/mach-landisk/gio.c b/arch/sh/boards/mach-landisk/gio.c index 1c0da99dfc60..ff2200fec29a 100644 --- a/arch/sh/boards/mach-landisk/gio.c +++ b/arch/sh/boards/mach-landisk/gio.c @@ -27,11 +27,10 @@ static int openCnt; static int gio_open(struct inode *inode, struct file *filp) { - int minor; + int minor = iminor(inode); int ret = -ENOENT; preempt_disable(); - minor = MINOR(inode->i_rdev); if (minor < DEVCOUNT) { if (openCnt > 0) { ret = -EALREADY; @@ -46,9 +45,8 @@ static int gio_open(struct inode *inode, struct file *filp) static int gio_close(struct inode *inode, struct file *filp) { - int minor; + int minor = iminor(inode); - minor = MINOR(inode->i_rdev); if (minor < DEVCOUNT) { openCnt--; } diff --git a/arch/sh/configs/edosk7760_defconfig b/arch/sh/configs/edosk7760_defconfig index 02ba62298576..d77f54e906fd 100644 --- a/arch/sh/configs/edosk7760_defconfig +++ b/arch/sh/configs/edosk7760_defconfig @@ -102,7 +102,6 @@ CONFIG_NLS_UTF8=y CONFIG_PRINTK_TIME=y # CONFIG_ENABLE_MUST_CHECK is not set CONFIG_MAGIC_SYSRQ=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_SHIRQ=y CONFIG_DETECT_HUNG_TASK=y diff --git a/arch/sh/configs/sdk7780_defconfig b/arch/sh/configs/sdk7780_defconfig index d00376eb044f..6c719ab4332a 100644 --- a/arch/sh/configs/sdk7780_defconfig +++ b/arch/sh/configs/sdk7780_defconfig @@ -128,7 +128,6 @@ CONFIG_NLS_ISO8859_15=y CONFIG_NLS_UTF8=y # CONFIG_ENABLE_MUST_CHECK is not set CONFIG_MAGIC_SYSRQ=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_DEBUG_KERNEL=y CONFIG_DETECT_HUNG_TASK=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/sh/include/asm/irq.h b/arch/sh/include/asm/irq.h index 6d44c32ef047..839551ce398c 100644 --- a/arch/sh/include/asm/irq.h +++ b/arch/sh/include/asm/irq.h @@ -51,7 +51,6 @@ asmlinkage int do_IRQ(unsigned int irq, struct pt_regs *regs); #ifdef CONFIG_IRQSTACKS extern void irq_ctx_init(int cpu); extern void irq_ctx_exit(int cpu); -# define __ARCH_HAS_DO_SOFTIRQ #else # define irq_ctx_init(cpu) do { } while (0) # define irq_ctx_exit(cpu) do { } while (0) diff --git a/arch/sh/kernel/irq.c b/arch/sh/kernel/irq.c index ab5f790b0cd2..ef0f0827cf57 100644 --- a/arch/sh/kernel/irq.c +++ b/arch/sh/kernel/irq.c @@ -20,6 +20,7 @@ #include #include #include +#include atomic_t irq_err_count; diff --git a/arch/sh/kernel/process_32.c b/arch/sh/kernel/process_32.c index 80a5d1c66a51..1aa508eb0823 100644 --- a/arch/sh/kernel/process_32.c +++ b/arch/sh/kernel/process_32.c @@ -114,7 +114,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, childregs = task_pt_regs(p); p->thread.sp = (unsigned long) childregs; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); p->thread.pc = (unsigned long) ret_from_kernel_thread; childregs->regs[4] = arg; diff --git a/arch/sh/kernel/syscalls/Makefile b/arch/sh/kernel/syscalls/Makefile index 659faefdcb1d..285aaba832d9 100644 --- a/arch/sh/kernel/syscalls/Makefile +++ b/arch/sh/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,18 +21,19 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_abi_$(basetarget))' \ '$(systbl_offset_$(basetarget))' -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl index 9df40ac0ebc0..d08eebad6b7f 100644 --- a/arch/sh/kernel/syscalls/syscall.tbl +++ b/arch/sh/kernel/syscalls/syscall.tbl @@ -444,3 +444,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 2c1cee9eed73..164a5254c91c 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -96,6 +96,7 @@ config SPARC64 select ARCH_HAS_PTE_SPECIAL select PCI_DOMAINS if PCI select ARCH_HAS_GIGANTIC_PAGE + select HAVE_SOFTIRQ_ON_OWN_STACK config ARCH_PROC_KCORE_TEXT def_bool y @@ -175,7 +176,7 @@ config SMP Management" code will be disabled if you say Y here. See also and the SMP-HOWTO - available at . + available at . If you don't know what to do here, say N. diff --git a/arch/sparc/boot/piggyback.c b/arch/sparc/boot/piggyback.c index a7a38fb4ece0..6d74064add0a 100644 --- a/arch/sparc/boot/piggyback.c +++ b/arch/sparc/boot/piggyback.c @@ -154,6 +154,10 @@ static off_t get_hdrs_offset(int kernelfd, const char *filename) offset -= LOOKBACK; /* skip a.out header */ offset += AOUT_TEXT_OFFSET; + if (offset < 0) { + errno = -EINVAL; + die("Calculated a negative offset, probably elftoaout generated an invalid image. Did you use a recent elftoaout ?"); + } if (lseek(kernelfd, offset, SEEK_SET) < 0) die("lseek"); if (read(kernelfd, buffer, BUFSIZE) != BUFSIZE) diff --git a/arch/sparc/configs/sparc64_defconfig b/arch/sparc/configs/sparc64_defconfig index d91eb6a76dd1..148f44b33890 100644 --- a/arch/sparc/configs/sparc64_defconfig +++ b/arch/sparc/configs/sparc64_defconfig @@ -65,9 +65,8 @@ CONFIG_CDROM_PKTCDVD=m CONFIG_CDROM_PKTCDVD_WCACHE=y CONFIG_ATA_OVER_ETH=m CONFIG_SUNVDC=m -CONFIG_IDE=y -CONFIG_BLK_DEV_IDECD=y -CONFIG_BLK_DEV_ALI15X3=y +CONFIG_ATA=y +CONFIG_PATA_ALI=y CONFIG_RAID_ATTRS=m CONFIG_SCSI=y CONFIG_BLK_DEV_SD=y @@ -235,3 +234,9 @@ CONFIG_CRYPTO_TWOFISH=m CONFIG_CRC16=m CONFIG_LIBCRC32C=m CONFIG_VCC=m +CONFIG_ATA=y +CONFIG_PATA_CMD64X=y +CONFIG_HAPPYMEAL=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_DEVTMPFS=y diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h index 8625946d8d00..597a22953bc5 100644 --- a/arch/sparc/include/asm/backoff.h +++ b/arch/sparc/include/asm/backoff.h @@ -18,7 +18,7 @@ * * When we spin, we try to use an operation that will cause the * current cpu strand to block, and therefore make the core fully - * available to any other other runnable strands. There are two + * available to any other runnable strands. There are two * options, based upon cpu capabilities. * * On all cpus prior to SPARC-T4 we do three dummy reads of the diff --git a/arch/sparc/include/asm/cmpxchg_32.h b/arch/sparc/include/asm/cmpxchg_32.h index c73b5a3ab7b9..a53d744d4212 100644 --- a/arch/sparc/include/asm/cmpxchg_32.h +++ b/arch/sparc/include/asm/cmpxchg_32.h @@ -25,7 +25,7 @@ static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, int return x; } -#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) +#define xchg(ptr,x) ({(__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));}) /* Emulate cmpxchg() the same way we emulate atomics, * by hashing the object address and indexing into an array diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h index 4d748e93b974..154df2cf19f4 100644 --- a/arch/sparc/include/asm/irq_64.h +++ b/arch/sparc/include/asm/irq_64.h @@ -93,7 +93,6 @@ void arch_trigger_cpumask_backtrace(const struct cpumask *mask, extern void *hardirq_stack[NR_CPUS]; extern void *softirq_stack[NR_CPUS]; -#define __ARCH_HAS_DO_SOFTIRQ #define NO_IRQ 0xffffffff diff --git a/arch/sparc/include/asm/mman.h b/arch/sparc/include/asm/mman.h index f94532f25db1..274217e7ed70 100644 --- a/arch/sparc/include/asm/mman.h +++ b/arch/sparc/include/asm/mman.h @@ -57,36 +57,40 @@ static inline int sparc_validate_prot(unsigned long prot, unsigned long addr) { if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM | PROT_ADI)) return 0; - if (prot & PROT_ADI) { - if (!adi_capable()) - return 0; - - if (addr) { - struct vm_area_struct *vma; - - vma = find_vma(current->mm, addr); - if (vma) { - /* ADI can not be enabled on PFN - * mapped pages - */ - if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) - return 0; - - /* Mergeable pages can become unmergeable - * if ADI is enabled on them even if they - * have identical data on them. This can be - * because ADI enabled pages with identical - * data may still not have identical ADI - * tags on them. Disallow ADI on mergeable - * pages. - */ - if (vma->vm_flags & VM_MERGEABLE) - return 0; - } - } - } return 1; } + +#define arch_validate_flags(vm_flags) arch_validate_flags(vm_flags) +/* arch_validate_flags() - Ensure combination of flags is valid for a + * VMA. + */ +static inline bool arch_validate_flags(unsigned long vm_flags) +{ + /* If ADI is being enabled on this VMA, check for ADI + * capability on the platform and ensure VMA is suitable + * for ADI + */ + if (vm_flags & VM_SPARC_ADI) { + if (!adi_capable()) + return false; + + /* ADI can not be enabled on PFN mapped pages */ + if (vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) + return false; + + /* Mergeable pages can become unmergeable + * if ADI is enabled on them even if they + * have identical data on them. This can be + * because ADI enabled pages with identical + * data may still not have identical ADI + * tags on them. Disallow ADI on mergeable + * pages. + */ + if (vm_flags & VM_MERGEABLE) + return false; + } + return true; +} #endif /* CONFIG_SPARC64 */ #endif /* __ASSEMBLY__ */ diff --git a/arch/sparc/include/asm/pgtsrmmu.h b/arch/sparc/include/asm/pgtsrmmu.h index 7708d015712b..6067925972d9 100644 --- a/arch/sparc/include/asm/pgtsrmmu.h +++ b/arch/sparc/include/asm/pgtsrmmu.h @@ -113,7 +113,7 @@ extern unsigned long last_valid_pfn; extern void *srmmu_nocache_pool; #define __nocache_pa(VADDR) (((unsigned long)VADDR) - SRMMU_NOCACHE_VADDR + __pa((unsigned long)srmmu_nocache_pool)) #define __nocache_va(PADDR) (__va((unsigned long)PADDR) - (unsigned long)srmmu_nocache_pool + SRMMU_NOCACHE_VADDR) -#define __nocache_fix(VADDR) __va(__nocache_pa(VADDR)) +#define __nocache_fix(VADDR) ((__typeof__(VADDR))__va(__nocache_pa(VADDR))) /* Accessing the MMU control register. */ unsigned int srmmu_get_mmureg(void); diff --git a/arch/sparc/include/asm/signal.h b/arch/sparc/include/asm/signal.h index 827b73a97f8a..28f81081e37d 100644 --- a/arch/sparc/include/asm/signal.h +++ b/arch/sparc/include/asm/signal.h @@ -9,18 +9,6 @@ #include #ifndef __ASSEMBLY__ -/* - * DJHR - * SA_STATIC_ALLOC is used for the sparc32 system to indicate that this - * interrupt handler's irq structure should be statically allocated - * by the request_irq routine. - * The alternative is that arch/sparc/kernel/irq.c has carnal knowledge - * of interrupt usage and that sucks. Also without a flag like this - * it may be possible for the free_irq routine to attempt to free - * statically allocated data.. which is NOT GOOD. - * - */ -#define SA_STATIC_ALLOC 0x8000 #define __ARCH_HAS_KA_RESTORER #define __ARCH_HAS_SA_RESTORER diff --git a/arch/sparc/kernel/entry.S b/arch/sparc/kernel/entry.S index d58940280f8d..a269ad2fe6df 100644 --- a/arch/sparc/kernel/entry.S +++ b/arch/sparc/kernel/entry.S @@ -994,7 +994,7 @@ do_syscall: andcc %l5, _TIF_SYSCALL_TRACE, %g0 mov %i4, %o4 bne linux_syscall_trace - mov %i0, %l5 + mov %i0, %l6 2: call %l7 mov %i5, %o5 @@ -1003,16 +1003,15 @@ do_syscall: st %o0, [%sp + STACKFRAME_SZ + PT_I0] ret_sys_call: - ld [%curptr + TI_FLAGS], %l6 + ld [%curptr + TI_FLAGS], %l5 cmp %o0, -ERESTART_RESTARTBLOCK ld [%sp + STACKFRAME_SZ + PT_PSR], %g3 set PSR_C, %g2 bgeu 1f - andcc %l6, _TIF_SYSCALL_TRACE, %g0 + andcc %l5, _TIF_SYSCALL_TRACE, %g0 /* System call success, clear Carry condition code. */ andn %g3, %g2, %g3 - clr %l6 st %g3, [%sp + STACKFRAME_SZ + PT_PSR] bne linux_syscall_trace2 ld [%sp + STACKFRAME_SZ + PT_NPC], %l1 /* pc = npc */ @@ -1027,7 +1026,6 @@ ret_sys_call: sub %g0, %o0, %o0 or %g3, %g2, %g3 st %o0, [%sp + STACKFRAME_SZ + PT_I0] - mov 1, %l6 st %g3, [%sp + STACKFRAME_SZ + PT_PSR] bne linux_syscall_trace2 ld [%sp + STACKFRAME_SZ + PT_NPC], %l1 /* pc = npc */ diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 3ec9f1402aad..c8848bb681a1 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -42,6 +42,7 @@ #include #include #include +#include #include "entry.h" #include "cpumap.h" diff --git a/arch/sparc/kernel/led.c b/arch/sparc/kernel/led.c index bd48575172c3..3a66e62eb2a0 100644 --- a/arch/sparc/kernel/led.c +++ b/arch/sparc/kernel/led.c @@ -50,6 +50,7 @@ static void led_blink(struct timer_list *unused) add_timer(&led_blink_timer); } +#ifdef CONFIG_PROC_FS static int led_proc_show(struct seq_file *m, void *v) { if (get_auxio() & AUXIO_LED) @@ -111,6 +112,7 @@ static const struct proc_ops led_proc_ops = { .proc_release = single_release, .proc_write = led_proc_write, }; +#endif static struct proc_dir_entry *led; diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c index 5d45b6d766d6..9c2b720bfd20 100644 --- a/arch/sparc/kernel/pci.c +++ b/arch/sparc/kernel/pci.c @@ -552,9 +552,8 @@ static void pci_of_scan_bus(struct pci_pbm_info *pbm, pci_info(bus, "scan_bus[%pOF] bus no %d\n", node, bus->number); - child = NULL; prev_devfn = -1; - while ((child = of_get_next_child(node, child)) != NULL) { + for_each_child_of_node(node, child) { if (ofpci_verbose) pci_info(bus, " * %pOF\n", child); reg = of_get_property(child, "reg", ®len); diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c index a02363735915..b91e88058e0c 100644 --- a/arch/sparc/kernel/process_32.c +++ b/arch/sparc/kernel/process_32.c @@ -183,7 +183,7 @@ void exit_thread(struct task_struct *tsk) #ifndef CONFIG_SMP if (last_task_used_math == tsk) { #else - if (test_ti_thread_flag(task_thread_info(tsk), TIF_USEDFPU)) { + if (test_tsk_thread_flag(tsk, TIF_USEDFPU)) { #endif /* Keep process from leaving FPU in a bogon state. */ put_psr(get_psr() | PSR_EF); @@ -309,7 +309,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, ti->ksp = (unsigned long) new_stack; p->thread.kregs = childregs; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { extern int nwindows; unsigned long psr; memset(new_stack, 0, STACKFRAME_SZ + TRACEREG_SZ); diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index 6f8c7822fc06..7afd0a859a78 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -597,7 +597,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, sizeof(struct sparc_stackf)); t->fpsaved[0] = 0; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(child_trap_frame, 0, child_stack_sz); __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = (current_pt_regs()->tstate + 1) & TSTATE_CWP; diff --git a/arch/sparc/kernel/rtrap_32.S b/arch/sparc/kernel/rtrap_32.S index dca8ed810046..8931fe266346 100644 --- a/arch/sparc/kernel/rtrap_32.S +++ b/arch/sparc/kernel/rtrap_32.S @@ -75,7 +75,7 @@ signal_p: ld [%sp + STACKFRAME_SZ + PT_PSR], %t_psr mov %g2, %o2 - mov %l5, %o1 + mov %l6, %o1 call do_notify_resume add %sp, STACKFRAME_SZ, %o0 ! pt_regs ptr diff --git a/arch/sparc/kernel/signal_32.c b/arch/sparc/kernel/signal_32.c index 11cf2281b581..02f3ad55dfe3 100644 --- a/arch/sparc/kernel/signal_32.c +++ b/arch/sparc/kernel/signal_32.c @@ -400,8 +400,8 @@ static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs, else { regs->u_regs[UREG_I7] = (unsigned long)(&(sf->insns[0]) - 2); - /* mov __NR_sigreturn, %g1 */ - err |= __put_user(0x821020d8, &sf->insns[0]); + /* mov __NR_rt_sigreturn, %g1 */ + err |= __put_user(0x82102065, &sf->insns[0]); /* t 0x10 */ err |= __put_user(0x91d02010, &sf->insns[1]); diff --git a/arch/sparc/kernel/syscalls/Makefile b/arch/sparc/kernel/syscalls/Makefile index c22a21c39f30..283f64407b07 100644 --- a/arch/sparc/kernel/syscalls/Makefile +++ b/arch/sparc/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -22,24 +22,24 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_offset_$(basetarget))' syshdr_abis_unistd_32 := common,32 -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abis_unistd_64 := common,64 -$(uapi)/unistd_64.h: $(syscall) $(syshdr) +$(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) systbl_abis_syscall_table_32 := common,32 -$(kapi)/syscall_table_32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_64 := common,64 -$(kapi)/syscall_table_64.h: $(syscall) $(systbl) +$(kapi)/syscall_table_64.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_c32 := common,32 systbl_abi_syscall_table_c32 := c32 -$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h unistd_64.h @@ -47,9 +47,10 @@ kapisyshdr-y += syscall_table_32.h \ syscall_table_64.h \ syscall_table_c32.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl index 40d8c7cd8298..84403a99039c 100644 --- a/arch/sparc/kernel/syscalls/syscall.tbl +++ b/arch/sparc/kernel/syscalls/syscall.tbl @@ -487,3 +487,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/sparc/kernel/viohs.c b/arch/sparc/kernel/viohs.c index 7db5aabe9708..e27afd233bf5 100644 --- a/arch/sparc/kernel/viohs.c +++ b/arch/sparc/kernel/viohs.c @@ -428,7 +428,7 @@ static int process_dreg_info(struct vio_driver_state *vio, struct vio_dring_register *pkt) { struct vio_dring_state *dr; - int i, len; + int i; viodbg(HS, "GOT DRING_REG INFO ident[%llx] " "ndesc[%u] dsz[%u] opt[0x%x] ncookies[%u]\n", @@ -482,9 +482,7 @@ static int process_dreg_info(struct vio_driver_state *vio, pkt->num_descr, pkt->descr_size, pkt->options, pkt->num_cookies); - len = (sizeof(*pkt) + - (dr->ncookies * sizeof(struct ldc_trans_cookie))); - if (send_ctrl(vio, &pkt->tag, len) < 0) + if (send_ctrl(vio, &pkt->tag, struct_size(pkt, cookies, dr->ncookies)) < 0) goto send_nack; vio->dr_state |= VIO_DR_STATE_RXREG; diff --git a/arch/sparc/lib/memset.S b/arch/sparc/lib/memset.S index b89d42b29e34..f427f34b8b79 100644 --- a/arch/sparc/lib/memset.S +++ b/arch/sparc/lib/memset.S @@ -142,6 +142,7 @@ __bzero: ZERO_LAST_BLOCKS(%o0, 0x48, %g2) ZERO_LAST_BLOCKS(%o0, 0x08, %g2) 13: + EXT(12b, 13b, 21f) be 8f andcc %o1, 4, %g0 diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c index eb2946b1df8a..6139c5700ccc 100644 --- a/arch/sparc/mm/init_32.c +++ b/arch/sparc/mm/init_32.c @@ -197,6 +197,9 @@ unsigned long __init bootmem_init(unsigned long *pages_avail) size = memblock_phys_mem_size() - memblock_reserved_size(); *pages_avail = (size >> PAGE_SHIFT) - high_pages; + /* Only allow low memory to be allocated via memblock allocation */ + memblock_set_current_limit(max_low_pfn << PAGE_SHIFT); + return max_pfn; } diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index a03caa5f6628..a9aa6a92c7fe 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -351,7 +351,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm) pte_t *ptep; struct page *page; - if ((ptep = pte_alloc_one_kernel(mm)) == 0) + if (!(ptep = pte_alloc_one_kernel(mm))) return NULL; page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT); spin_lock(&mm->page_table_lock); @@ -689,7 +689,7 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, pgdp = pgd_offset_k(start); p4dp = p4d_offset(pgdp, start); pudp = pud_offset(p4dp, start); - if (pud_none(*(pud_t *)__nocache_fix(pudp))) { + if (pud_none(*__nocache_fix(pudp))) { pmdp = __srmmu_get_nocache( SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) @@ -698,7 +698,7 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, pud_set(__nocache_fix(pudp), pmdp); } pmdp = pmd_offset(__nocache_fix(pudp), start); - if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + if (srmmu_pmd_none(*__nocache_fix(pmdp))) { ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); @@ -810,11 +810,11 @@ static void __init srmmu_inherit_prom_mappings(unsigned long start, p4dp = p4d_offset(pgdp, start); pudp = pud_offset(p4dp, start); if (what == 2) { - *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed); + *__nocache_fix(pgdp) = __pgd(probed); start += PGDIR_SIZE; continue; } - if (pud_none(*(pud_t *)__nocache_fix(pudp))) { + if (pud_none(*__nocache_fix(pudp))) { pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) @@ -822,13 +822,13 @@ static void __init srmmu_inherit_prom_mappings(unsigned long start, memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); pud_set(__nocache_fix(pudp), pmdp); } - pmdp = pmd_offset(__nocache_fix(pgdp), start); + pmdp = pmd_offset(__nocache_fix(pudp), start); if (what == 1) { *(pmd_t *)__nocache_fix(pmdp) = __pmd(probed); start += PMD_SIZE; continue; } - if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + if (srmmu_pmd_none(*__nocache_fix(pmdp))) { ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); @@ -836,7 +836,7 @@ static void __init srmmu_inherit_prom_mappings(unsigned long start, pmd_set(__nocache_fix(pmdp), ptep); } ptep = pte_offset_kernel(__nocache_fix(pmdp), start); - *(pte_t *)__nocache_fix(ptep) = __pte(probed); + *__nocache_fix(ptep) = __pte(probed); start += PAGE_SIZE; } } @@ -850,7 +850,7 @@ static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base unsigned long big_pte; big_pte = KERNEL_PTE(phys_base >> 4); - *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte); + *__nocache_fix(pgdp) = __pgd(big_pte); } /* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */ @@ -940,7 +940,7 @@ void __init srmmu_paging_init(void) srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa(srmmu_context_table); for (i = 0; i < num_contexts; i++) - srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); + srmmu_ctxd_set(__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); flush_cache_all(); srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys); diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index a58811dc054c..d7492e5a1bbb 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -20,6 +20,7 @@ generic-y += param.h generic-y += pci.h generic-y += percpu.h generic-y += preempt.h +generic-y += softirq_stack.h generic-y += switch_to.h generic-y += topology.h generic-y += trace_clock.h diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c index 81d508daf67c..c5011064b5dd 100644 --- a/arch/um/kernel/process.c +++ b/arch/um/kernel/process.c @@ -157,7 +157,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, struct task_struct * p, unsigned long tls) { void (*handler)(void); - int kthread = current->flags & PF_KTHREAD; + int kthread = current->flags & (PF_KTHREAD | PF_IO_WORKER); int ret = 0; p->thread = (struct thread_struct) INIT_THREAD; diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 595193bc2d31..2792879d398e 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -97,6 +97,8 @@ config X86 select ARCH_SUPPORTS_DEBUG_PAGEALLOC select ARCH_SUPPORTS_NUMA_BALANCING if X86_64 select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096 + select ARCH_SUPPORTS_LTO_CLANG if X86_64 + select ARCH_SUPPORTS_LTO_CLANG_THIN if X86_64 select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS @@ -149,6 +151,7 @@ config X86 select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN if X86_64 select HAVE_ARCH_KASAN_VMALLOC if X86_64 + select HAVE_ARCH_KFENCE select HAVE_ARCH_KGDB select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_ARCH_MMAP_RND_COMPAT_BITS if MMU && COMPAT @@ -169,6 +172,7 @@ config X86 select HAVE_CONTEXT_TRACKING if X86_64 select HAVE_CONTEXT_TRACKING_OFFSTACK if HAVE_CONTEXT_TRACKING select HAVE_C_RECORDMCOUNT + select HAVE_OBJTOOL_MCOUNT if STACK_VALIDATION select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS select HAVE_DYNAMIC_FTRACE @@ -188,6 +192,7 @@ config X86 select HAVE_HW_BREAKPOINT select HAVE_IDE select HAVE_IOREMAP_PROT + select HAVE_IRQ_EXIT_ON_IRQ_STACK if X86_64 select HAVE_IRQ_TIME_ACCOUNTING select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP @@ -220,6 +225,7 @@ config X86 select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION select HAVE_FUNCTION_ARG_ACCESS_API + select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 select HAVE_STATIC_CALL @@ -445,7 +451,7 @@ config X86_X2APIC If you don't know what to do here, say N. config X86_MPPARSE - bool "Enable MPS table" if ACPI || SFI + bool "Enable MPS table" if ACPI default y depends on X86_LOCAL_APIC help @@ -604,7 +610,6 @@ config X86_INTEL_MID depends on PCI depends on X86_64 || (PCI_GOANY && X86_32) depends on X86_IO_APIC - select SFI select I2C select DW_APB_TIMER select APB_TIMER @@ -893,18 +898,6 @@ config HPET_EMULATE_RTC def_bool y depends on HPET_TIMER && (RTC_DRV_CMOS=m || RTC_DRV_CMOS=y) -config APB_TIMER - def_bool y if X86_INTEL_MID - prompt "Intel MID APB Timer Support" if X86_INTEL_MID - select DW_APB_TIMER - depends on X86_INTEL_MID && SFI - help - APB timer is the replacement for 8254, HPET on X86 MID platforms. - The APBT provides a stable time base on SMP - systems, unlike the TSC, but it is more expensive to access, - as it is off-chip. APB timers are always running regardless of CPU - C states, they are used as per CPU clockevent device when possible. - # Mark as expert because too many people got it wrong. # The code disables itself when not needed. config DMI @@ -2466,8 +2459,6 @@ source "kernel/power/Kconfig" source "drivers/acpi/Kconfig" -source "drivers/sfi/Kconfig" - config X86_APM_BOOT def_bool y depends on APM @@ -2654,7 +2645,7 @@ config PCI_DIRECT config PCI_MMCONFIG bool "Support mmconfig PCI config space access" if X86_64 default y - depends on PCI && (ACPI || SFI || JAILHOUSE_GUEST) + depends on PCI && (ACPI || JAILHOUSE_GUEST) depends on X86_64 || (PCI_GOANY || PCI_GOMMCONFIG) config PCI_OLPC diff --git a/arch/x86/Makefile b/arch/x86/Makefile index b797f1561943..2d6d5a28c3bf 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -169,6 +169,11 @@ ifeq ($(ACCUMULATE_OUTGOING_ARGS), 1) KBUILD_CFLAGS += $(call cc-option,-maccumulate-outgoing-args,) endif +ifdef CONFIG_LTO_CLANG +KBUILD_LDFLAGS += -plugin-opt=-code-model=kernel \ + -plugin-opt=-stack-alignment=$(if $(CONFIG_X86_32),4,8) +endif + # Workaround for a gcc prelease that unfortunately was shipped in a suse release KBUILD_CFLAGS += -Wno-sign-compare # diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig index 78210793d357..9c9c4a888b1d 100644 --- a/arch/x86/configs/i386_defconfig +++ b/arch/x86/configs/i386_defconfig @@ -50,7 +50,6 @@ CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y -# CONFIG_UNUSED_SYMBOLS is not set CONFIG_BINFMT_MISC=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig index 9936528e1939..b60bd2d86034 100644 --- a/arch/x86/configs/x86_64_defconfig +++ b/arch/x86/configs/x86_64_defconfig @@ -48,7 +48,6 @@ CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y -# CONFIG_UNUSED_SYMBOLS is not set CONFIG_BINFMT_MISC=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index a31de0c6ccde..b28e36b7c96b 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -4,8 +4,6 @@ OBJECT_FILES_NON_STANDARD := y -obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o - obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index d1436c37008b..4e3972570916 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -43,10 +43,6 @@ #ifdef __x86_64__ # constants in mergeable sections, linker can reorder and merge -.section .rodata.cst16.gf128mul_x_ble_mask, "aM", @progbits, 16 -.align 16 -.Lgf128mul_x_ble_mask: - .octa 0x00000000000000010000000000000087 .section .rodata.cst16.POLY, "aM", @progbits, 16 .align 16 POLY: .octa 0xC2000000000000000000000000000001 @@ -146,7 +142,7 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff #define CTR %xmm11 #define INC %xmm12 -#define GF128MUL_MASK %xmm10 +#define GF128MUL_MASK %xmm7 #ifdef __x86_64__ #define AREG %rax @@ -2577,13 +2573,140 @@ SYM_FUNC_START(aesni_cbc_dec) ret SYM_FUNC_END(aesni_cbc_dec) -#ifdef __x86_64__ +/* + * void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, + * size_t len, u8 *iv) + */ +SYM_FUNC_START(aesni_cts_cbc_enc) + FRAME_BEGIN +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl (FRAME_OFFSET+20)(%esp), KEYP # ctx + movl (FRAME_OFFSET+24)(%esp), OUTP # dst + movl (FRAME_OFFSET+28)(%esp), INP # src + movl (FRAME_OFFSET+32)(%esp), LEN # len + movl (FRAME_OFFSET+36)(%esp), IVP # iv + lea .Lcts_permute_table, T1 +#else + lea .Lcts_permute_table(%rip), T1 +#endif + mov 480(KEYP), KLEN + movups (IVP), STATE + sub $16, LEN + mov T1, IVP + add $32, IVP + add LEN, T1 + sub LEN, IVP + movups (T1), %xmm4 + movups (IVP), %xmm5 + + movups (INP), IN1 + add LEN, INP + movups (INP), IN2 + + pxor IN1, STATE + call _aesni_enc1 + + pshufb %xmm5, IN2 + pxor STATE, IN2 + pshufb %xmm4, STATE + add OUTP, LEN + movups STATE, (LEN) + + movaps IN2, STATE + call _aesni_enc1 + movups STATE, (OUTP) + +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif + FRAME_END + ret +SYM_FUNC_END(aesni_cts_cbc_enc) + +/* + * void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, + * size_t len, u8 *iv) + */ +SYM_FUNC_START(aesni_cts_cbc_dec) + FRAME_BEGIN +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl (FRAME_OFFSET+20)(%esp), KEYP # ctx + movl (FRAME_OFFSET+24)(%esp), OUTP # dst + movl (FRAME_OFFSET+28)(%esp), INP # src + movl (FRAME_OFFSET+32)(%esp), LEN # len + movl (FRAME_OFFSET+36)(%esp), IVP # iv + lea .Lcts_permute_table, T1 +#else + lea .Lcts_permute_table(%rip), T1 +#endif + mov 480(KEYP), KLEN + add $240, KEYP + movups (IVP), IV + sub $16, LEN + mov T1, IVP + add $32, IVP + add LEN, T1 + sub LEN, IVP + movups (T1), %xmm4 + + movups (INP), STATE + add LEN, INP + movups (INP), IN1 + + call _aesni_dec1 + movaps STATE, IN2 + pshufb %xmm4, STATE + pxor IN1, STATE + + add OUTP, LEN + movups STATE, (LEN) + + movups (IVP), %xmm0 + pshufb %xmm0, IN1 + pblendvb IN2, IN1 + movaps IN1, STATE + call _aesni_dec1 + + pxor IV, STATE + movups STATE, (OUTP) + +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif + FRAME_END + ret +SYM_FUNC_END(aesni_cts_cbc_dec) + .pushsection .rodata .align 16 +.Lcts_permute_table: + .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 + .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 + .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 + .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 +#ifdef __x86_64__ .Lbswap_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +#endif .popsection +#ifdef __x86_64__ /* * _aesni_inc_init: internal ABI * setup registers used by _aesni_inc @@ -2696,6 +2819,14 @@ SYM_FUNC_START(aesni_ctr_enc) ret SYM_FUNC_END(aesni_ctr_enc) +#endif + +.section .rodata.cst16.gf128mul_x_ble_mask, "aM", @progbits, 16 +.align 16 +.Lgf128mul_x_ble_mask: + .octa 0x00000000000000010000000000000087 +.previous + /* * _aesni_gf128mul_x_ble: internal ABI * Multiply in GF(2^128) for XTS IVs @@ -2708,120 +2839,325 @@ SYM_FUNC_END(aesni_ctr_enc) * CTR: == temporary value */ #define _aesni_gf128mul_x_ble() \ - pshufd $0x13, IV, CTR; \ + pshufd $0x13, IV, KEY; \ paddq IV, IV; \ - psrad $31, CTR; \ - pand GF128MUL_MASK, CTR; \ - pxor CTR, IV; + psrad $31, KEY; \ + pand GF128MUL_MASK, KEY; \ + pxor KEY, IV; /* - * void aesni_xts_crypt8(const struct crypto_aes_ctx *ctx, u8 *dst, - * const u8 *src, bool enc, le128 *iv) + * void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *dst, + * const u8 *src, unsigned int len, le128 *iv) */ -SYM_FUNC_START(aesni_xts_crypt8) +SYM_FUNC_START(aesni_xts_encrypt) FRAME_BEGIN - testb %cl, %cl - movl $0, %ecx - movl $240, %r10d - leaq _aesni_enc4, %r11 - leaq _aesni_dec4, %rax - cmovel %r10d, %ecx - cmoveq %rax, %r11 - +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl (FRAME_OFFSET+20)(%esp), KEYP # ctx + movl (FRAME_OFFSET+24)(%esp), OUTP # dst + movl (FRAME_OFFSET+28)(%esp), INP # src + movl (FRAME_OFFSET+32)(%esp), LEN # len + movl (FRAME_OFFSET+36)(%esp), IVP # iv movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK +#else + movdqa .Lgf128mul_x_ble_mask(%rip), GF128MUL_MASK +#endif movups (IVP), IV mov 480(KEYP), KLEN - addq %rcx, KEYP + +.Lxts_enc_loop4: + sub $64, LEN + jl .Lxts_enc_1x movdqa IV, STATE1 - movdqu 0x00(INP), INC - pxor INC, STATE1 + movdqu 0x00(INP), IN + pxor IN, STATE1 movdqu IV, 0x00(OUTP) _aesni_gf128mul_x_ble() movdqa IV, STATE2 - movdqu 0x10(INP), INC - pxor INC, STATE2 + movdqu 0x10(INP), IN + pxor IN, STATE2 movdqu IV, 0x10(OUTP) _aesni_gf128mul_x_ble() movdqa IV, STATE3 - movdqu 0x20(INP), INC - pxor INC, STATE3 + movdqu 0x20(INP), IN + pxor IN, STATE3 movdqu IV, 0x20(OUTP) _aesni_gf128mul_x_ble() movdqa IV, STATE4 - movdqu 0x30(INP), INC - pxor INC, STATE4 + movdqu 0x30(INP), IN + pxor IN, STATE4 movdqu IV, 0x30(OUTP) - CALL_NOSPEC r11 + call _aesni_enc4 - movdqu 0x00(OUTP), INC - pxor INC, STATE1 + movdqu 0x00(OUTP), IN + pxor IN, STATE1 movdqu STATE1, 0x00(OUTP) - _aesni_gf128mul_x_ble() - movdqa IV, STATE1 - movdqu 0x40(INP), INC - pxor INC, STATE1 - movdqu IV, 0x40(OUTP) - - movdqu 0x10(OUTP), INC - pxor INC, STATE2 + movdqu 0x10(OUTP), IN + pxor IN, STATE2 movdqu STATE2, 0x10(OUTP) - _aesni_gf128mul_x_ble() - movdqa IV, STATE2 - movdqu 0x50(INP), INC - pxor INC, STATE2 - movdqu IV, 0x50(OUTP) - - movdqu 0x20(OUTP), INC - pxor INC, STATE3 + movdqu 0x20(OUTP), IN + pxor IN, STATE3 movdqu STATE3, 0x20(OUTP) - _aesni_gf128mul_x_ble() - movdqa IV, STATE3 - movdqu 0x60(INP), INC - pxor INC, STATE3 - movdqu IV, 0x60(OUTP) - - movdqu 0x30(OUTP), INC - pxor INC, STATE4 + movdqu 0x30(OUTP), IN + pxor IN, STATE4 movdqu STATE4, 0x30(OUTP) _aesni_gf128mul_x_ble() - movdqa IV, STATE4 - movdqu 0x70(INP), INC - pxor INC, STATE4 - movdqu IV, 0x70(OUTP) - _aesni_gf128mul_x_ble() + add $64, INP + add $64, OUTP + test LEN, LEN + jnz .Lxts_enc_loop4 + +.Lxts_enc_ret_iv: movups IV, (IVP) - CALL_NOSPEC r11 - - movdqu 0x40(OUTP), INC - pxor INC, STATE1 - movdqu STATE1, 0x40(OUTP) - - movdqu 0x50(OUTP), INC - pxor INC, STATE2 - movdqu STATE2, 0x50(OUTP) - - movdqu 0x60(OUTP), INC - pxor INC, STATE3 - movdqu STATE3, 0x60(OUTP) - - movdqu 0x70(OUTP), INC - pxor INC, STATE4 - movdqu STATE4, 0x70(OUTP) - +.Lxts_enc_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif FRAME_END ret -SYM_FUNC_END(aesni_xts_crypt8) +.Lxts_enc_1x: + add $64, LEN + jz .Lxts_enc_ret_iv + sub $16, LEN + jl .Lxts_enc_cts4 + +.Lxts_enc_loop1: + movdqu (INP), STATE + pxor IV, STATE + call _aesni_enc1 + pxor IV, STATE + _aesni_gf128mul_x_ble() + + test LEN, LEN + jz .Lxts_enc_out + + add $16, INP + sub $16, LEN + jl .Lxts_enc_cts1 + + movdqu STATE, (OUTP) + add $16, OUTP + jmp .Lxts_enc_loop1 + +.Lxts_enc_out: + movdqu STATE, (OUTP) + jmp .Lxts_enc_ret_iv + +.Lxts_enc_cts4: + movdqa STATE4, STATE + sub $16, OUTP + +.Lxts_enc_cts1: +#ifndef __x86_64__ + lea .Lcts_permute_table, T1 +#else + lea .Lcts_permute_table(%rip), T1 #endif + add LEN, INP /* rewind input pointer */ + add $16, LEN /* # bytes in final block */ + movups (INP), IN1 + + mov T1, IVP + add $32, IVP + add LEN, T1 + sub LEN, IVP + add OUTP, LEN + + movups (T1), %xmm4 + movaps STATE, IN2 + pshufb %xmm4, STATE + movups STATE, (LEN) + + movups (IVP), %xmm0 + pshufb %xmm0, IN1 + pblendvb IN2, IN1 + movaps IN1, STATE + + pxor IV, STATE + call _aesni_enc1 + pxor IV, STATE + + movups STATE, (OUTP) + jmp .Lxts_enc_ret +SYM_FUNC_END(aesni_xts_encrypt) + +/* + * void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *dst, + * const u8 *src, unsigned int len, le128 *iv) + */ +SYM_FUNC_START(aesni_xts_decrypt) + FRAME_BEGIN +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl (FRAME_OFFSET+20)(%esp), KEYP # ctx + movl (FRAME_OFFSET+24)(%esp), OUTP # dst + movl (FRAME_OFFSET+28)(%esp), INP # src + movl (FRAME_OFFSET+32)(%esp), LEN # len + movl (FRAME_OFFSET+36)(%esp), IVP # iv + movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK +#else + movdqa .Lgf128mul_x_ble_mask(%rip), GF128MUL_MASK +#endif + movups (IVP), IV + + mov 480(KEYP), KLEN + add $240, KEYP + + test $15, LEN + jz .Lxts_dec_loop4 + sub $16, LEN + +.Lxts_dec_loop4: + sub $64, LEN + jl .Lxts_dec_1x + + movdqa IV, STATE1 + movdqu 0x00(INP), IN + pxor IN, STATE1 + movdqu IV, 0x00(OUTP) + + _aesni_gf128mul_x_ble() + movdqa IV, STATE2 + movdqu 0x10(INP), IN + pxor IN, STATE2 + movdqu IV, 0x10(OUTP) + + _aesni_gf128mul_x_ble() + movdqa IV, STATE3 + movdqu 0x20(INP), IN + pxor IN, STATE3 + movdqu IV, 0x20(OUTP) + + _aesni_gf128mul_x_ble() + movdqa IV, STATE4 + movdqu 0x30(INP), IN + pxor IN, STATE4 + movdqu IV, 0x30(OUTP) + + call _aesni_dec4 + + movdqu 0x00(OUTP), IN + pxor IN, STATE1 + movdqu STATE1, 0x00(OUTP) + + movdqu 0x10(OUTP), IN + pxor IN, STATE2 + movdqu STATE2, 0x10(OUTP) + + movdqu 0x20(OUTP), IN + pxor IN, STATE3 + movdqu STATE3, 0x20(OUTP) + + movdqu 0x30(OUTP), IN + pxor IN, STATE4 + movdqu STATE4, 0x30(OUTP) + + _aesni_gf128mul_x_ble() + + add $64, INP + add $64, OUTP + test LEN, LEN + jnz .Lxts_dec_loop4 + +.Lxts_dec_ret_iv: + movups IV, (IVP) + +.Lxts_dec_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif + FRAME_END + ret + +.Lxts_dec_1x: + add $64, LEN + jz .Lxts_dec_ret_iv + +.Lxts_dec_loop1: + movdqu (INP), STATE + + add $16, INP + sub $16, LEN + jl .Lxts_dec_cts1 + + pxor IV, STATE + call _aesni_dec1 + pxor IV, STATE + _aesni_gf128mul_x_ble() + + test LEN, LEN + jz .Lxts_dec_out + + movdqu STATE, (OUTP) + add $16, OUTP + jmp .Lxts_dec_loop1 + +.Lxts_dec_out: + movdqu STATE, (OUTP) + jmp .Lxts_dec_ret_iv + +.Lxts_dec_cts1: + movdqa IV, STATE4 + _aesni_gf128mul_x_ble() + + pxor IV, STATE + call _aesni_dec1 + pxor IV, STATE + +#ifndef __x86_64__ + lea .Lcts_permute_table, T1 +#else + lea .Lcts_permute_table(%rip), T1 +#endif + add LEN, INP /* rewind input pointer */ + add $16, LEN /* # bytes in final block */ + movups (INP), IN1 + + mov T1, IVP + add $32, IVP + add LEN, T1 + sub LEN, IVP + add OUTP, LEN + + movups (T1), %xmm4 + movaps STATE, IN2 + pshufb %xmm4, STATE + movups STATE, (LEN) + + movups (IVP), %xmm0 + pshufb %xmm0, IN1 + pblendvb IN2, IN1 + movaps IN1, STATE + + pxor STATE4, STATE + call _aesni_dec1 + pxor STATE4, STATE + + movups STATE, (OUTP) + jmp .Lxts_dec_ret +SYM_FUNC_END(aesni_xts_decrypt) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index ad8a7188a2bf..2144e54a6c89 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -31,11 +31,10 @@ #include #include #include +#include #include #include -#ifdef CONFIG_X86_64 -#include -#endif +#include #define AESNI_ALIGN 16 @@ -93,62 +92,25 @@ asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); +asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, + const u8 *in, unsigned int len, u8 *iv); +asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, + const u8 *in, unsigned int len, u8 *iv); #define AVX_GEN2_OPTSIZE 640 #define AVX_GEN4_OPTSIZE 4096 +asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, + const u8 *in, unsigned int len, u8 *iv); + +asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, + const u8 *in, unsigned int len, u8 *iv); + #ifdef CONFIG_X86_64 -static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out, - const u8 *in, unsigned int len, u8 *iv); asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); - -asmlinkage void aesni_xts_crypt8(const struct crypto_aes_ctx *ctx, u8 *out, - const u8 *in, bool enc, le128 *iv); - -/* asmlinkage void aesni_gcm_enc() - * void *ctx, AES Key schedule. Starts on a 16 byte boundary. - * struct gcm_context_data. May be uninitialized. - * u8 *out, Ciphertext output. Encrypt in-place is allowed. - * const u8 *in, Plaintext input - * unsigned long plaintext_len, Length of data in bytes for encryption. - * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. - * 16-byte aligned pointer. - * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. - * const u8 *aad, Additional Authentication Data (AAD) - * unsigned long aad_len, Length of AAD in bytes. - * u8 *auth_tag, Authenticated Tag output. - * unsigned long auth_tag_len), Authenticated Tag Length in bytes. - * Valid values are 16 (most likely), 12 or 8. - */ -asmlinkage void aesni_gcm_enc(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long plaintext_len, u8 *iv, - u8 *hash_subkey, const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); - -/* asmlinkage void aesni_gcm_dec() - * void *ctx, AES Key schedule. Starts on a 16 byte boundary. - * struct gcm_context_data. May be uninitialized. - * u8 *out, Plaintext output. Decrypt in-place is allowed. - * const u8 *in, Ciphertext input - * unsigned long ciphertext_len, Length of data in bytes for decryption. - * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. - * 16-byte aligned pointer. - * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. - * const u8 *aad, Additional Authentication Data (AAD) - * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going - * to be 8 or 12 bytes - * u8 *auth_tag, Authenticated Tag output. - * unsigned long auth_tag_len) Authenticated Tag Length in bytes. - * Valid values are 16 (most likely), 12 or 8. - */ -asmlinkage void aesni_gcm_dec(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long ciphertext_len, u8 *iv, - u8 *hash_subkey, const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); +DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc); /* Scatter / Gather routines, with args similar to above */ asmlinkage void aesni_gcm_init(void *ctx, @@ -167,24 +129,6 @@ asmlinkage void aesni_gcm_finalize(void *ctx, struct gcm_context_data *gdata, u8 *auth_tag, unsigned long auth_tag_len); -static const struct aesni_gcm_tfm_s { - void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv, - u8 *hash_subkey, const u8 *aad, unsigned long aad_len); - void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long plaintext_len); - void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long ciphertext_len); - void (*finalize)(void *ctx, struct gcm_context_data *gdata, - u8 *auth_tag, unsigned long auth_tag_len); -} *aesni_gcm_tfm; - -static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = { - .init = &aesni_gcm_init, - .enc_update = &aesni_gcm_enc_update, - .dec_update = &aesni_gcm_dec_update, - .finalize = &aesni_gcm_finalize, -}; - asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, void *keys, u8 *out, unsigned int num_bytes); asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, @@ -214,25 +158,6 @@ asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx, struct gcm_context_data *gdata, u8 *auth_tag, unsigned long auth_tag_len); -asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long plaintext_len, u8 *iv, - const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); - -asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long ciphertext_len, u8 *iv, - const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); - -static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = { - .init = &aesni_gcm_init_avx_gen2, - .enc_update = &aesni_gcm_enc_update_avx_gen2, - .dec_update = &aesni_gcm_dec_update_avx_gen2, - .finalize = &aesni_gcm_finalize_avx_gen2, -}; - /* * asmlinkage void aesni_gcm_init_avx_gen4() * gcm_data *my_ctx_data, context data @@ -256,24 +181,8 @@ asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx, struct gcm_context_data *gdata, u8 *auth_tag, unsigned long auth_tag_len); -asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long plaintext_len, u8 *iv, - const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); - -asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long ciphertext_len, u8 *iv, - const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); - -static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = { - .init = &aesni_gcm_init_avx_gen4, - .enc_update = &aesni_gcm_enc_update_avx_gen4, - .dec_update = &aesni_gcm_dec_update_avx_gen4, - .finalize = &aesni_gcm_finalize_avx_gen4, -}; +static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx2); static inline struct aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) @@ -374,16 +283,16 @@ static int ecb_encrypt(struct skcipher_request *req) unsigned int nbytes; int err; - err = skcipher_walk_virt(&walk, req, true); + err = skcipher_walk_virt(&walk, req, false); - kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { + kernel_fpu_begin(); aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & AES_BLOCK_MASK); + kernel_fpu_end(); nbytes &= AES_BLOCK_SIZE - 1; err = skcipher_walk_done(&walk, nbytes); } - kernel_fpu_end(); return err; } @@ -396,16 +305,16 @@ static int ecb_decrypt(struct skcipher_request *req) unsigned int nbytes; int err; - err = skcipher_walk_virt(&walk, req, true); + err = skcipher_walk_virt(&walk, req, false); - kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { + kernel_fpu_begin(); aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & AES_BLOCK_MASK); + kernel_fpu_end(); nbytes &= AES_BLOCK_SIZE - 1; err = skcipher_walk_done(&walk, nbytes); } - kernel_fpu_end(); return err; } @@ -418,16 +327,16 @@ static int cbc_encrypt(struct skcipher_request *req) unsigned int nbytes; int err; - err = skcipher_walk_virt(&walk, req, true); + err = skcipher_walk_virt(&walk, req, false); - kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { + kernel_fpu_begin(); aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & AES_BLOCK_MASK, walk.iv); + kernel_fpu_end(); nbytes &= AES_BLOCK_SIZE - 1; err = skcipher_walk_done(&walk, nbytes); } - kernel_fpu_end(); return err; } @@ -440,36 +349,133 @@ static int cbc_decrypt(struct skcipher_request *req) unsigned int nbytes; int err; - err = skcipher_walk_virt(&walk, req, true); + err = skcipher_walk_virt(&walk, req, false); - kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { + kernel_fpu_begin(); aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & AES_BLOCK_MASK, walk.iv); + kernel_fpu_end(); nbytes &= AES_BLOCK_SIZE - 1; err = skcipher_walk_done(&walk, nbytes); } - kernel_fpu_end(); return err; } -#ifdef CONFIG_X86_64 -static void ctr_crypt_final(struct crypto_aes_ctx *ctx, - struct skcipher_walk *walk) +static int cts_cbc_encrypt(struct skcipher_request *req) { - u8 *ctrblk = walk->iv; - u8 keystream[AES_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); + int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; + struct scatterlist *src = req->src, *dst = req->dst; + struct scatterlist sg_src[2], sg_dst[2]; + struct skcipher_request subreq; + struct skcipher_walk walk; + int err; - aesni_enc(ctx, keystream, ctrblk); - crypto_xor_cpy(dst, keystream, src, nbytes); + skcipher_request_set_tfm(&subreq, tfm); + skcipher_request_set_callback(&subreq, skcipher_request_flags(req), + NULL, NULL); - crypto_inc(ctrblk, AES_BLOCK_SIZE); + if (req->cryptlen <= AES_BLOCK_SIZE) { + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + cbc_blocks = 1; + } + + if (cbc_blocks > 0) { + skcipher_request_set_crypt(&subreq, req->src, req->dst, + cbc_blocks * AES_BLOCK_SIZE, + req->iv); + + err = cbc_encrypt(&subreq); + if (err) + return err; + + if (req->cryptlen == AES_BLOCK_SIZE) + return 0; + + dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); + if (req->dst != req->src) + dst = scatterwalk_ffwd(sg_dst, req->dst, + subreq.cryptlen); + } + + /* handle ciphertext stealing */ + skcipher_request_set_crypt(&subreq, src, dst, + req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, + req->iv); + + err = skcipher_walk_virt(&walk, &subreq, false); + if (err) + return err; + + kernel_fpu_begin(); + aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes, walk.iv); + kernel_fpu_end(); + + return skcipher_walk_done(&walk, 0); } +static int cts_cbc_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); + int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; + struct scatterlist *src = req->src, *dst = req->dst; + struct scatterlist sg_src[2], sg_dst[2]; + struct skcipher_request subreq; + struct skcipher_walk walk; + int err; + + skcipher_request_set_tfm(&subreq, tfm); + skcipher_request_set_callback(&subreq, skcipher_request_flags(req), + NULL, NULL); + + if (req->cryptlen <= AES_BLOCK_SIZE) { + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + cbc_blocks = 1; + } + + if (cbc_blocks > 0) { + skcipher_request_set_crypt(&subreq, req->src, req->dst, + cbc_blocks * AES_BLOCK_SIZE, + req->iv); + + err = cbc_decrypt(&subreq); + if (err) + return err; + + if (req->cryptlen == AES_BLOCK_SIZE) + return 0; + + dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); + if (req->dst != req->src) + dst = scatterwalk_ffwd(sg_dst, req->dst, + subreq.cryptlen); + } + + /* handle ciphertext stealing */ + skcipher_request_set_crypt(&subreq, src, dst, + req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, + req->iv); + + err = skcipher_walk_virt(&walk, &subreq, false); + if (err) + return err; + + kernel_fpu_begin(); + aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes, walk.iv); + kernel_fpu_end(); + + return skcipher_walk_done(&walk, 0); +} + +#ifdef CONFIG_X86_64 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv) { @@ -491,120 +497,36 @@ static int ctr_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); + u8 keystream[AES_BLOCK_SIZE]; struct skcipher_walk walk; unsigned int nbytes; int err; - err = skcipher_walk_virt(&walk, req, true); + err = skcipher_walk_virt(&walk, req, false); - kernel_fpu_begin(); - while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { - aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, - nbytes & AES_BLOCK_MASK, walk.iv); - nbytes &= AES_BLOCK_SIZE - 1; + while ((nbytes = walk.nbytes) > 0) { + kernel_fpu_begin(); + if (nbytes & AES_BLOCK_MASK) + static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr, + walk.src.virt.addr, + nbytes & AES_BLOCK_MASK, + walk.iv); + nbytes &= ~AES_BLOCK_MASK; + + if (walk.nbytes == walk.total && nbytes > 0) { + aesni_enc(ctx, keystream, walk.iv); + crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes, + walk.src.virt.addr + walk.nbytes - nbytes, + keystream, nbytes); + crypto_inc(walk.iv, AES_BLOCK_SIZE); + nbytes = 0; + } + kernel_fpu_end(); err = skcipher_walk_done(&walk, nbytes); } - if (walk.nbytes) { - ctr_crypt_final(ctx, &walk); - err = skcipher_walk_done(&walk, 0); - } - kernel_fpu_end(); - return err; } -static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = xts_verify_key(tfm, key, keylen); - if (err) - return err; - - keylen /= 2; - - /* first half of xts-key is for crypt */ - err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, - key, keylen); - if (err) - return err; - - /* second half of xts-key is for tweak */ - return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, - key + keylen, keylen); -} - - -static void aesni_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, aesni_enc); -} - -static void aesni_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, aesni_dec); -} - -static void aesni_xts_enc8(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - aesni_xts_crypt8(ctx, dst, src, true, iv); -} - -static void aesni_xts_dec8(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - aesni_xts_crypt8(ctx, dst, src, false, iv); -} - -static const struct common_glue_ctx aesni_enc_xts = { - .num_funcs = 2, - .fpu_blocks_limit = 1, - - .funcs = { { - .num_blocks = 8, - .fn_u = { .xts = aesni_xts_enc8 } - }, { - .num_blocks = 1, - .fn_u = { .xts = aesni_xts_enc } - } } -}; - -static const struct common_glue_ctx aesni_dec_xts = { - .num_funcs = 2, - .fpu_blocks_limit = 1, - - .funcs = { { - .num_blocks = 8, - .fn_u = { .xts = aesni_xts_dec8 } - }, { - .num_blocks = 1, - .fn_u = { .xts = aesni_xts_dec } - } } -}; - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&aesni_enc_xts, req, aesni_enc, - aes_ctx(ctx->raw_tweak_ctx), - aes_ctx(ctx->raw_crypt_ctx), - false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&aesni_dec_xts, req, aesni_enc, - aes_ctx(ctx->raw_tweak_ctx), - aes_ctx(ctx->raw_crypt_ctx), - true); -} - static int rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) { @@ -681,42 +603,35 @@ static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, unsigned int assoclen, u8 *hash_subkey, - u8 *iv, void *aes_ctx) + u8 *iv, void *aes_ctx, u8 *auth_tag, + unsigned long auth_tag_len) { - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - unsigned long auth_tag_len = crypto_aead_authsize(tfm); - const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm; - struct gcm_context_data data AESNI_ALIGN_ATTR; - struct scatter_walk dst_sg_walk = {}; + u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8); + struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN); unsigned long left = req->cryptlen; - unsigned long len, srclen, dstlen; struct scatter_walk assoc_sg_walk; - struct scatter_walk src_sg_walk; - struct scatterlist src_start[2]; - struct scatterlist dst_start[2]; - struct scatterlist *src_sg; - struct scatterlist *dst_sg; - u8 *src, *dst, *assoc; + struct skcipher_walk walk; + bool do_avx, do_avx2; u8 *assocmem = NULL; - u8 authTag[16]; + u8 *assoc; + int err; if (!enc) left -= auth_tag_len; - if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4) - gcm_tfm = &aesni_gcm_tfm_avx_gen2; - if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2) - gcm_tfm = &aesni_gcm_tfm_sse; + do_avx = (left >= AVX_GEN2_OPTSIZE); + do_avx2 = (left >= AVX_GEN4_OPTSIZE); /* Linearize assoc, if not already linear */ - if (req->src->length >= assoclen && req->src->length && - (!PageHighMem(sg_page(req->src)) || - req->src->offset + req->src->length <= PAGE_SIZE)) { + if (req->src->length >= assoclen && req->src->length) { scatterwalk_start(&assoc_sg_walk, req->src); assoc = scatterwalk_map(&assoc_sg_walk); } else { + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + /* assoc can be any length, so must be on heap */ - assocmem = kmalloc(assoclen, GFP_ATOMIC); + assocmem = kmalloc(assoclen, flags); if (unlikely(!assocmem)) return -ENOMEM; assoc = assocmem; @@ -724,62 +639,15 @@ static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); } - if (left) { - src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); - scatterwalk_start(&src_sg_walk, src_sg); - if (req->src != req->dst) { - dst_sg = scatterwalk_ffwd(dst_start, req->dst, - req->assoclen); - scatterwalk_start(&dst_sg_walk, dst_sg); - } - } - kernel_fpu_begin(); - gcm_tfm->init(aes_ctx, &data, iv, - hash_subkey, assoc, assoclen); - if (req->src != req->dst) { - while (left) { - src = scatterwalk_map(&src_sg_walk); - dst = scatterwalk_map(&dst_sg_walk); - srclen = scatterwalk_clamp(&src_sg_walk, left); - dstlen = scatterwalk_clamp(&dst_sg_walk, left); - len = min(srclen, dstlen); - if (len) { - if (enc) - gcm_tfm->enc_update(aes_ctx, &data, - dst, src, len); - else - gcm_tfm->dec_update(aes_ctx, &data, - dst, src, len); - } - left -= len; - - scatterwalk_unmap(src); - scatterwalk_unmap(dst); - scatterwalk_advance(&src_sg_walk, len); - scatterwalk_advance(&dst_sg_walk, len); - scatterwalk_done(&src_sg_walk, 0, left); - scatterwalk_done(&dst_sg_walk, 1, left); - } - } else { - while (left) { - dst = src = scatterwalk_map(&src_sg_walk); - len = scatterwalk_clamp(&src_sg_walk, left); - if (len) { - if (enc) - gcm_tfm->enc_update(aes_ctx, &data, - src, src, len); - else - gcm_tfm->dec_update(aes_ctx, &data, - src, src, len); - } - left -= len; - scatterwalk_unmap(src); - scatterwalk_advance(&src_sg_walk, len); - scatterwalk_done(&src_sg_walk, 1, left); - } - } - gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len); + if (static_branch_likely(&gcm_use_avx2) && do_avx2) + aesni_gcm_init_avx_gen4(aes_ctx, data, iv, hash_subkey, assoc, + assoclen); + else if (static_branch_likely(&gcm_use_avx) && do_avx) + aesni_gcm_init_avx_gen2(aes_ctx, data, iv, hash_subkey, assoc, + assoclen); + else + aesni_gcm_init(aes_ctx, data, iv, hash_subkey, assoc, assoclen); kernel_fpu_end(); if (!assocmem) @@ -787,24 +655,58 @@ static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, else kfree(assocmem); - if (!enc) { - u8 authTagMsg[16]; + err = enc ? skcipher_walk_aead_encrypt(&walk, req, false) + : skcipher_walk_aead_decrypt(&walk, req, false); - /* Copy out original authTag */ - scatterwalk_map_and_copy(authTagMsg, req->src, - req->assoclen + req->cryptlen - - auth_tag_len, - auth_tag_len, 0); + while (walk.nbytes > 0) { + kernel_fpu_begin(); + if (static_branch_likely(&gcm_use_avx2) && do_avx2) { + if (enc) + aesni_gcm_enc_update_avx_gen4(aes_ctx, data, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes); + else + aesni_gcm_dec_update_avx_gen4(aes_ctx, data, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes); + } else if (static_branch_likely(&gcm_use_avx) && do_avx) { + if (enc) + aesni_gcm_enc_update_avx_gen2(aes_ctx, data, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes); + else + aesni_gcm_dec_update_avx_gen2(aes_ctx, data, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes); + } else if (enc) { + aesni_gcm_enc_update(aes_ctx, data, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + } else { + aesni_gcm_dec_update(aes_ctx, data, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + } + kernel_fpu_end(); - /* Compare generated tag with passed in tag. */ - return crypto_memneq(authTagMsg, authTag, auth_tag_len) ? - -EBADMSG : 0; + err = skcipher_walk_done(&walk, 0); } - /* Copy in the authTag */ - scatterwalk_map_and_copy(authTag, req->dst, - req->assoclen + req->cryptlen, - auth_tag_len, 1); + if (err) + return err; + + kernel_fpu_begin(); + if (static_branch_likely(&gcm_use_avx2) && do_avx2) + aesni_gcm_finalize_avx_gen4(aes_ctx, data, auth_tag, + auth_tag_len); + else if (static_branch_likely(&gcm_use_avx) && do_avx) + aesni_gcm_finalize_avx_gen2(aes_ctx, data, auth_tag, + auth_tag_len); + else + aesni_gcm_finalize(aes_ctx, data, auth_tag, auth_tag_len); + kernel_fpu_end(); return 0; } @@ -812,15 +714,47 @@ static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen, u8 *hash_subkey, u8 *iv, void *aes_ctx) { - return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, - aes_ctx); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned long auth_tag_len = crypto_aead_authsize(tfm); + u8 auth_tag[16]; + int err; + + err = gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, aes_ctx, + auth_tag, auth_tag_len); + if (err) + return err; + + scatterwalk_map_and_copy(auth_tag, req->dst, + req->assoclen + req->cryptlen, + auth_tag_len, 1); + return 0; } static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen, u8 *hash_subkey, u8 *iv, void *aes_ctx) { - return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, - aes_ctx); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned long auth_tag_len = crypto_aead_authsize(tfm); + u8 auth_tag_msg[16]; + u8 auth_tag[16]; + int err; + + err = gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, aes_ctx, + auth_tag, auth_tag_len); + if (err) + return err; + + /* Copy out original auth_tag */ + scatterwalk_map_and_copy(auth_tag_msg, req->src, + req->assoclen + req->cryptlen - auth_tag_len, + auth_tag_len, 0); + + /* Compare generated tag with passed in tag. */ + if (crypto_memneq(auth_tag_msg, auth_tag, auth_tag_len)) { + memzero_explicit(auth_tag, sizeof(auth_tag)); + return -EBADMSG; + } + return 0; } static int helper_rfc4106_encrypt(struct aead_request *req) @@ -828,7 +762,8 @@ static int helper_rfc4106_encrypt(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); void *aes_ctx = &(ctx->aes_key_expanded); - u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); + u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); unsigned int i; __be32 counter = cpu_to_be32(1); @@ -855,7 +790,8 @@ static int helper_rfc4106_decrypt(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); void *aes_ctx = &(ctx->aes_key_expanded); - u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); + u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); unsigned int i; if (unlikely(req->assoclen != 16 && req->assoclen != 20)) @@ -877,6 +813,128 @@ static int helper_rfc4106_decrypt(struct aead_request *req) } #endif +static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + int err; + + err = xts_verify_key(tfm, key, keylen); + if (err) + return err; + + keylen /= 2; + + /* first half of xts-key is for crypt */ + err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, + key, keylen); + if (err) + return err; + + /* second half of xts-key is for tweak */ + return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, + key + keylen, keylen); +} + +static int xts_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + int tail = req->cryptlen % AES_BLOCK_SIZE; + struct skcipher_request subreq; + struct skcipher_walk walk; + int err; + + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(tail > 0 && walk.nbytes < walk.total)) { + int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; + + skcipher_walk_abort(&walk); + + skcipher_request_set_tfm(&subreq, tfm); + skcipher_request_set_callback(&subreq, + skcipher_request_flags(req), + NULL, NULL); + skcipher_request_set_crypt(&subreq, req->src, req->dst, + blocks * AES_BLOCK_SIZE, req->iv); + req = &subreq; + err = skcipher_walk_virt(&walk, req, false); + } else { + tail = 0; + } + + kernel_fpu_begin(); + + /* calculate first value of T */ + aesni_enc(aes_ctx(ctx->raw_tweak_ctx), walk.iv, walk.iv); + + while (walk.nbytes > 0) { + int nbytes = walk.nbytes; + + if (nbytes < walk.total) + nbytes &= ~(AES_BLOCK_SIZE - 1); + + if (encrypt) + aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx), + walk.dst.virt.addr, walk.src.virt.addr, + nbytes, walk.iv); + else + aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx), + walk.dst.virt.addr, walk.src.virt.addr, + nbytes, walk.iv); + kernel_fpu_end(); + + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); + + if (walk.nbytes > 0) + kernel_fpu_begin(); + } + + if (unlikely(tail > 0 && !err)) { + struct scatterlist sg_src[2], sg_dst[2]; + struct scatterlist *src, *dst; + + dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); + if (req->dst != req->src) + dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); + + skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, + req->iv); + + err = skcipher_walk_virt(&walk, &subreq, false); + if (err) + return err; + + kernel_fpu_begin(); + if (encrypt) + aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx), + walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes, walk.iv); + else + aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx), + walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes, walk.iv); + kernel_fpu_end(); + + err = skcipher_walk_done(&walk, 0); + } + return err; +} + +static int xts_encrypt(struct skcipher_request *req) +{ + return xts_crypt(req, true); +} + +static int xts_decrypt(struct skcipher_request *req) +{ + return xts_crypt(req, false); +} + static struct crypto_alg aesni_cipher_alg = { .cra_name = "aes", .cra_driver_name = "aes-aesni", @@ -928,6 +986,23 @@ static struct skcipher_alg aesni_skciphers[] = { .setkey = aesni_skcipher_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, + }, { + .base = { + .cra_name = "__cts(cbc(aes))", + .cra_driver_name = "__cts-cbc-aes-aesni", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_INTERNAL, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = CRYPTO_AES_CTX_SIZE, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .walksize = 2 * AES_BLOCK_SIZE, + .setkey = aesni_skcipher_setkey, + .encrypt = cts_cbc_encrypt, + .decrypt = cts_cbc_decrypt, #ifdef CONFIG_X86_64 }, { .base = { @@ -946,6 +1021,7 @@ static struct skcipher_alg aesni_skciphers[] = { .setkey = aesni_skcipher_setkey, .encrypt = ctr_crypt, .decrypt = ctr_crypt, +#endif }, { .base = { .cra_name = "__xts(aes)", @@ -959,10 +1035,10 @@ static struct skcipher_alg aesni_skciphers[] = { .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, + .walksize = 2 * AES_BLOCK_SIZE, .setkey = xts_aesni_setkey, .encrypt = xts_encrypt, .decrypt = xts_decrypt, -#endif } }; @@ -985,7 +1061,8 @@ static int generic_gcmaes_encrypt(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); void *aes_ctx = &(ctx->aes_key_expanded); - u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); + u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); __be32 counter = cpu_to_be32(1); memcpy(iv, req->iv, 12); @@ -1001,7 +1078,8 @@ static int generic_gcmaes_decrypt(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); void *aes_ctx = &(ctx->aes_key_expanded); - u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); + u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); memcpy(iv, req->iv, 12); *((__be32 *)(iv+12)) = counter; @@ -1066,19 +1144,18 @@ static int __init aesni_init(void) #ifdef CONFIG_X86_64 if (boot_cpu_has(X86_FEATURE_AVX2)) { pr_info("AVX2 version of gcm_enc/dec engaged.\n"); - aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4; + static_branch_enable(&gcm_use_avx); + static_branch_enable(&gcm_use_avx2); } else if (boot_cpu_has(X86_FEATURE_AVX)) { pr_info("AVX version of gcm_enc/dec engaged.\n"); - aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2; + static_branch_enable(&gcm_use_avx); } else { pr_info("SSE version of gcm_enc/dec engaged.\n"); - aesni_gcm_tfm = &aesni_gcm_tfm_sse; } - aesni_ctr_enc_tfm = aesni_ctr_enc; if (boot_cpu_has(X86_FEATURE_AVX)) { /* optimize performance of ctr mode encryption transform */ - aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm; + static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm); pr_info("AES CTR mode by8 optimization enabled\n"); } #endif diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c index c025a01cf708..a40365ab301e 100644 --- a/arch/x86/crypto/blake2s-glue.c +++ b/arch/x86/crypto/blake2s-glue.c @@ -58,138 +58,40 @@ void blake2s_compress_arch(struct blake2s_state *state, } EXPORT_SYMBOL(blake2s_compress_arch); -static int crypto_blake2s_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) +static int crypto_blake2s_update_x86(struct shash_desc *desc, + const u8 *in, unsigned int inlen) { - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen == 0 || keylen > BLAKE2S_KEY_SIZE) - return -EINVAL; - - memcpy(tctx->key, key, keylen); - tctx->keylen = keylen; - - return 0; + return crypto_blake2s_update(desc, in, inlen, blake2s_compress_arch); } -static int crypto_blake2s_init(struct shash_desc *desc) +static int crypto_blake2s_final_x86(struct shash_desc *desc, u8 *out) { - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct blake2s_state *state = shash_desc_ctx(desc); - const int outlen = crypto_shash_digestsize(desc->tfm); - - if (tctx->keylen) - blake2s_init_key(state, outlen, tctx->key, tctx->keylen); - else - blake2s_init(state, outlen); - - return 0; + return crypto_blake2s_final(desc, out, blake2s_compress_arch); } -static int crypto_blake2s_update(struct shash_desc *desc, const u8 *in, - unsigned int inlen) -{ - struct blake2s_state *state = shash_desc_ctx(desc); - const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; - - if (unlikely(!inlen)) - return 0; - if (inlen > fill) { - memcpy(state->buf + state->buflen, in, fill); - blake2s_compress_arch(state, state->buf, 1, BLAKE2S_BLOCK_SIZE); - state->buflen = 0; - in += fill; - inlen -= fill; +#define BLAKE2S_ALG(name, driver_name, digest_size) \ + { \ + .base.cra_name = name, \ + .base.cra_driver_name = driver_name, \ + .base.cra_priority = 200, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \ + .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \ + .base.cra_module = THIS_MODULE, \ + .digestsize = digest_size, \ + .setkey = crypto_blake2s_setkey, \ + .init = crypto_blake2s_init, \ + .update = crypto_blake2s_update_x86, \ + .final = crypto_blake2s_final_x86, \ + .descsize = sizeof(struct blake2s_state), \ } - if (inlen > BLAKE2S_BLOCK_SIZE) { - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); - /* Hash one less (full) block than strictly possible */ - blake2s_compress_arch(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE); - in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); - inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); - } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; - return 0; -} - -static int crypto_blake2s_final(struct shash_desc *desc, u8 *out) -{ - struct blake2s_state *state = shash_desc_ctx(desc); - - blake2s_set_lastblock(state); - memset(state->buf + state->buflen, 0, - BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ - blake2s_compress_arch(state, state->buf, 1, state->buflen); - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - memcpy(out, state->h, state->outlen); - memzero_explicit(state, sizeof(*state)); - - return 0; -} - -static struct shash_alg blake2s_algs[] = {{ - .base.cra_name = "blake2s-128", - .base.cra_driver_name = "blake2s-128-x86", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_128_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-160", - .base.cra_driver_name = "blake2s-160-x86", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_160_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-224", - .base.cra_driver_name = "blake2s-224-x86", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_224_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-256", - .base.cra_driver_name = "blake2s-256-x86", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_256_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}}; +static struct shash_alg blake2s_algs[] = { + BLAKE2S_ALG("blake2s-128", "blake2s-128-x86", BLAKE2S_128_HASH_SIZE), + BLAKE2S_ALG("blake2s-160", "blake2s-160-x86", BLAKE2S_160_HASH_SIZE), + BLAKE2S_ALG("blake2s-224", "blake2s-224-x86", BLAKE2S_224_HASH_SIZE), + BLAKE2S_ALG("blake2s-256", "blake2s-256-x86", BLAKE2S_256_HASH_SIZE), +}; static int __init blake2s_mod_init(void) { diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c index cedfdba69ce3..a880e0b1c255 100644 --- a/arch/x86/crypto/blowfish_glue.c +++ b/arch/x86/crypto/blowfish_glue.c @@ -6,8 +6,6 @@ * * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: * Copyright (c) 2006 Herbert Xu - * CTR part based on code (crypto/ctr.c) by: - * (C) Copyright IBM Corp. 2007 - Joy Latten */ #include @@ -247,97 +245,6 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } -static void ctr_crypt_final(struct bf_ctx *ctx, struct skcipher_walk *walk) -{ - u8 *ctrblk = walk->iv; - u8 keystream[BF_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; - - blowfish_enc_blk(ctx, keystream, ctrblk); - crypto_xor_cpy(dst, keystream, src, nbytes); - - crypto_inc(ctrblk, BF_BLOCK_SIZE); -} - -static unsigned int __ctr_crypt(struct bf_ctx *ctx, struct skcipher_walk *walk) -{ - unsigned int bsize = BF_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u64 *src = (u64 *)walk->src.virt.addr; - u64 *dst = (u64 *)walk->dst.virt.addr; - u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv); - __be64 ctrblocks[4]; - - /* Process four block batch */ - if (nbytes >= bsize * 4) { - do { - if (dst != src) { - dst[0] = src[0]; - dst[1] = src[1]; - dst[2] = src[2]; - dst[3] = src[3]; - } - - /* create ctrblks for parallel encrypt */ - ctrblocks[0] = cpu_to_be64(ctrblk++); - ctrblocks[1] = cpu_to_be64(ctrblk++); - ctrblocks[2] = cpu_to_be64(ctrblk++); - ctrblocks[3] = cpu_to_be64(ctrblk++); - - blowfish_enc_blk_xor_4way(ctx, (u8 *)dst, - (u8 *)ctrblocks); - - src += 4; - dst += 4; - } while ((nbytes -= bsize * 4) >= bsize * 4); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - if (dst != src) - *dst = *src; - - ctrblocks[0] = cpu_to_be64(ctrblk++); - - blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks); - - src += 1; - dst += 1; - } while ((nbytes -= bsize) >= bsize); - -done: - *(__be64 *)walk->iv = cpu_to_be64(ctrblk); - return nbytes; -} - -static int ctr_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct bf_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) { - nbytes = __ctr_crypt(ctx, &walk); - err = skcipher_walk_done(&walk, nbytes); - } - - if (nbytes) { - ctr_crypt_final(ctx, &walk); - err = skcipher_walk_done(&walk, 0); - } - - return err; -} - static struct crypto_alg bf_cipher_alg = { .cra_name = "blowfish", .cra_driver_name = "blowfish-asm", @@ -384,20 +291,6 @@ static struct skcipher_alg bf_skcipher_algs[] = { .setkey = blowfish_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "ctr(blowfish)", - .base.cra_driver_name = "ctr-blowfish-asm", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct bf_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = BF_MIN_KEY_SIZE, - .max_keysize = BF_MAX_KEY_SIZE, - .ivsize = BF_BLOCK_SIZE, - .chunksize = BF_BLOCK_SIZE, - .setkey = blowfish_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, }, }; diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S index ecc0a9a905c4..e2a0e0f4bf9d 100644 --- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S +++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S @@ -17,7 +17,6 @@ #include #include -#include #define CAMELLIA_TABLE_BYTE_LEN 272 @@ -589,14 +588,6 @@ SYM_FUNC_END(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) .long 0x80808080 .long 0x80808080 -/* For CTR-mode IV byteswap */ -.Lbswap128_mask: - .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 - -/* For XTS mode IV generation */ -.Lxts_gf128mul_and_shl1_mask: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 - /* * pre-SubByte transform * @@ -998,292 +989,3 @@ SYM_FUNC_START(camellia_cbc_dec_16way) FRAME_END ret; SYM_FUNC_END(camellia_cbc_dec_16way) - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -SYM_FUNC_START(camellia_ctr_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - - subq $(16 * 16), %rsp; - movq %rsp, %rax; - - vmovdqa .Lbswap128_mask, %xmm14; - - /* load IV and byteswap */ - vmovdqu (%rcx), %xmm0; - vpshufb %xmm14, %xmm0, %xmm15; - vmovdqu %xmm15, 15 * 16(%rax); - - vpcmpeqd %xmm15, %xmm15, %xmm15; - vpsrldq $8, %xmm15, %xmm15; /* low: -1, high: 0 */ - - /* construct IVs */ - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm13; - vmovdqu %xmm13, 14 * 16(%rax); - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm13; - vmovdqu %xmm13, 13 * 16(%rax); - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm12; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm11; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm10; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm9; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm8; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm7; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm6; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm5; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm4; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm3; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm2; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm1; - inc_le128(%xmm0, %xmm15, %xmm13); - vmovdqa %xmm0, %xmm13; - vpshufb %xmm14, %xmm0, %xmm0; - inc_le128(%xmm13, %xmm15, %xmm14); - vmovdqu %xmm13, (%rcx); - - /* inpack16_pre: */ - vmovq (key_table)(CTX), %xmm15; - vpshufb .Lpack_bswap, %xmm15, %xmm15; - vpxor %xmm0, %xmm15, %xmm0; - vpxor %xmm1, %xmm15, %xmm1; - vpxor %xmm2, %xmm15, %xmm2; - vpxor %xmm3, %xmm15, %xmm3; - vpxor %xmm4, %xmm15, %xmm4; - vpxor %xmm5, %xmm15, %xmm5; - vpxor %xmm6, %xmm15, %xmm6; - vpxor %xmm7, %xmm15, %xmm7; - vpxor %xmm8, %xmm15, %xmm8; - vpxor %xmm9, %xmm15, %xmm9; - vpxor %xmm10, %xmm15, %xmm10; - vpxor %xmm11, %xmm15, %xmm11; - vpxor %xmm12, %xmm15, %xmm12; - vpxor 13 * 16(%rax), %xmm15, %xmm13; - vpxor 14 * 16(%rax), %xmm15, %xmm14; - vpxor 15 * 16(%rax), %xmm15, %xmm15; - - call __camellia_enc_blk16; - - addq $(16 * 16), %rsp; - - vpxor 0 * 16(%rdx), %xmm7, %xmm7; - vpxor 1 * 16(%rdx), %xmm6, %xmm6; - vpxor 2 * 16(%rdx), %xmm5, %xmm5; - vpxor 3 * 16(%rdx), %xmm4, %xmm4; - vpxor 4 * 16(%rdx), %xmm3, %xmm3; - vpxor 5 * 16(%rdx), %xmm2, %xmm2; - vpxor 6 * 16(%rdx), %xmm1, %xmm1; - vpxor 7 * 16(%rdx), %xmm0, %xmm0; - vpxor 8 * 16(%rdx), %xmm15, %xmm15; - vpxor 9 * 16(%rdx), %xmm14, %xmm14; - vpxor 10 * 16(%rdx), %xmm13, %xmm13; - vpxor 11 * 16(%rdx), %xmm12, %xmm12; - vpxor 12 * 16(%rdx), %xmm11, %xmm11; - vpxor 13 * 16(%rdx), %xmm10, %xmm10; - vpxor 14 * 16(%rdx), %xmm9, %xmm9; - vpxor 15 * 16(%rdx), %xmm8, %xmm8; - write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0, - %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9, - %xmm8, %rsi); - - FRAME_END - ret; -SYM_FUNC_END(camellia_ctr_16way) - -#define gf128mul_x_ble(iv, mask, tmp) \ - vpsrad $31, iv, tmp; \ - vpaddq iv, iv, iv; \ - vpshufd $0x13, tmp, tmp; \ - vpand mask, tmp, tmp; \ - vpxor tmp, iv, iv; - -.align 8 -SYM_FUNC_START_LOCAL(camellia_xts_crypt_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - * %r8: index for input whitening key - * %r9: pointer to __camellia_enc_blk16 or __camellia_dec_blk16 - */ - FRAME_BEGIN - - subq $(16 * 16), %rsp; - movq %rsp, %rax; - - vmovdqa .Lxts_gf128mul_and_shl1_mask, %xmm14; - - /* load IV */ - vmovdqu (%rcx), %xmm0; - vpxor 0 * 16(%rdx), %xmm0, %xmm15; - vmovdqu %xmm15, 15 * 16(%rax); - vmovdqu %xmm0, 0 * 16(%rsi); - - /* construct IVs */ - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 1 * 16(%rdx), %xmm0, %xmm15; - vmovdqu %xmm15, 14 * 16(%rax); - vmovdqu %xmm0, 1 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 2 * 16(%rdx), %xmm0, %xmm13; - vmovdqu %xmm0, 2 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 3 * 16(%rdx), %xmm0, %xmm12; - vmovdqu %xmm0, 3 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 4 * 16(%rdx), %xmm0, %xmm11; - vmovdqu %xmm0, 4 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 5 * 16(%rdx), %xmm0, %xmm10; - vmovdqu %xmm0, 5 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 6 * 16(%rdx), %xmm0, %xmm9; - vmovdqu %xmm0, 6 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 7 * 16(%rdx), %xmm0, %xmm8; - vmovdqu %xmm0, 7 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 8 * 16(%rdx), %xmm0, %xmm7; - vmovdqu %xmm0, 8 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 9 * 16(%rdx), %xmm0, %xmm6; - vmovdqu %xmm0, 9 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 10 * 16(%rdx), %xmm0, %xmm5; - vmovdqu %xmm0, 10 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 11 * 16(%rdx), %xmm0, %xmm4; - vmovdqu %xmm0, 11 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 12 * 16(%rdx), %xmm0, %xmm3; - vmovdqu %xmm0, 12 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 13 * 16(%rdx), %xmm0, %xmm2; - vmovdqu %xmm0, 13 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 14 * 16(%rdx), %xmm0, %xmm1; - vmovdqu %xmm0, 14 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 15 * 16(%rdx), %xmm0, %xmm15; - vmovdqu %xmm15, 0 * 16(%rax); - vmovdqu %xmm0, 15 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vmovdqu %xmm0, (%rcx); - - /* inpack16_pre: */ - vmovq (key_table)(CTX, %r8, 8), %xmm15; - vpshufb .Lpack_bswap, %xmm15, %xmm15; - vpxor 0 * 16(%rax), %xmm15, %xmm0; - vpxor %xmm1, %xmm15, %xmm1; - vpxor %xmm2, %xmm15, %xmm2; - vpxor %xmm3, %xmm15, %xmm3; - vpxor %xmm4, %xmm15, %xmm4; - vpxor %xmm5, %xmm15, %xmm5; - vpxor %xmm6, %xmm15, %xmm6; - vpxor %xmm7, %xmm15, %xmm7; - vpxor %xmm8, %xmm15, %xmm8; - vpxor %xmm9, %xmm15, %xmm9; - vpxor %xmm10, %xmm15, %xmm10; - vpxor %xmm11, %xmm15, %xmm11; - vpxor %xmm12, %xmm15, %xmm12; - vpxor %xmm13, %xmm15, %xmm13; - vpxor 14 * 16(%rax), %xmm15, %xmm14; - vpxor 15 * 16(%rax), %xmm15, %xmm15; - - CALL_NOSPEC r9; - - addq $(16 * 16), %rsp; - - vpxor 0 * 16(%rsi), %xmm7, %xmm7; - vpxor 1 * 16(%rsi), %xmm6, %xmm6; - vpxor 2 * 16(%rsi), %xmm5, %xmm5; - vpxor 3 * 16(%rsi), %xmm4, %xmm4; - vpxor 4 * 16(%rsi), %xmm3, %xmm3; - vpxor 5 * 16(%rsi), %xmm2, %xmm2; - vpxor 6 * 16(%rsi), %xmm1, %xmm1; - vpxor 7 * 16(%rsi), %xmm0, %xmm0; - vpxor 8 * 16(%rsi), %xmm15, %xmm15; - vpxor 9 * 16(%rsi), %xmm14, %xmm14; - vpxor 10 * 16(%rsi), %xmm13, %xmm13; - vpxor 11 * 16(%rsi), %xmm12, %xmm12; - vpxor 12 * 16(%rsi), %xmm11, %xmm11; - vpxor 13 * 16(%rsi), %xmm10, %xmm10; - vpxor 14 * 16(%rsi), %xmm9, %xmm9; - vpxor 15 * 16(%rsi), %xmm8, %xmm8; - write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0, - %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9, - %xmm8, %rsi); - - FRAME_END - ret; -SYM_FUNC_END(camellia_xts_crypt_16way) - -SYM_FUNC_START(camellia_xts_enc_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - xorl %r8d, %r8d; /* input whitening key, 0 for enc */ - - leaq __camellia_enc_blk16, %r9; - - jmp camellia_xts_crypt_16way; -SYM_FUNC_END(camellia_xts_enc_16way) - -SYM_FUNC_START(camellia_xts_dec_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - - cmpl $16, key_length(CTX); - movl $32, %r8d; - movl $24, %eax; - cmovel %eax, %r8d; /* input whitening key, last for dec */ - - leaq __camellia_dec_blk16, %r9; - - jmp camellia_xts_crypt_16way; -SYM_FUNC_END(camellia_xts_dec_16way) diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S index 0907243c501c..782e9712a1ec 100644 --- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S +++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S @@ -7,7 +7,6 @@ #include #include -#include #define CAMELLIA_TABLE_BYTE_LEN 272 @@ -625,16 +624,6 @@ SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) .section .rodata.cst16, "aM", @progbits, 16 .align 16 -/* For CTR-mode IV byteswap */ -.Lbswap128_mask: - .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 - -/* For XTS mode */ -.Lxts_gf128mul_and_shl1_mask_0: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 -.Lxts_gf128mul_and_shl1_mask_1: - .byte 0x0e, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0 - /* * pre-SubByte transform * @@ -1061,343 +1050,3 @@ SYM_FUNC_START(camellia_cbc_dec_32way) FRAME_END ret; SYM_FUNC_END(camellia_cbc_dec_32way) - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ - vpcmpeqq minus_one, x, tmp1; \ - vpcmpeqq minus_two, x, tmp2; \ - vpsubq minus_two, x, x; \ - vpor tmp2, tmp1, tmp1; \ - vpslldq $8, tmp1, tmp1; \ - vpsubq tmp1, x, x; - -SYM_FUNC_START(camellia_ctr_32way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (32 blocks) - * %rdx: src (32 blocks) - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - - vzeroupper; - - movq %rsp, %r10; - cmpq %rsi, %rdx; - je .Lctr_use_stack; - - /* dst can be used as temporary storage, src is not overwritten. */ - movq %rsi, %rax; - jmp .Lctr_continue; - -.Lctr_use_stack: - subq $(16 * 32), %rsp; - movq %rsp, %rax; - -.Lctr_continue: - vpcmpeqd %ymm15, %ymm15, %ymm15; - vpsrldq $8, %ymm15, %ymm15; /* ab: -1:0 ; cd: -1:0 */ - vpaddq %ymm15, %ymm15, %ymm12; /* ab: -2:0 ; cd: -2:0 */ - - /* load IV and byteswap */ - vmovdqu (%rcx), %xmm0; - vmovdqa %xmm0, %xmm1; - inc_le128(%xmm0, %xmm15, %xmm14); - vbroadcasti128 .Lbswap128_mask, %ymm14; - vinserti128 $1, %xmm0, %ymm1, %ymm0; - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 15 * 32(%rax); - - /* construct IVs */ - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); /* ab:le2 ; cd:le3 */ - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 14 * 32(%rax); - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 13 * 32(%rax); - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 12 * 32(%rax); - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 11 * 32(%rax); - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm10; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm9; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm8; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm7; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm6; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm5; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm4; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm3; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm2; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm1; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vextracti128 $1, %ymm0, %xmm13; - vpshufb %ymm14, %ymm0, %ymm0; - inc_le128(%xmm13, %xmm15, %xmm14); - vmovdqu %xmm13, (%rcx); - - /* inpack32_pre: */ - vpbroadcastq (key_table)(CTX), %ymm15; - vpshufb .Lpack_bswap, %ymm15, %ymm15; - vpxor %ymm0, %ymm15, %ymm0; - vpxor %ymm1, %ymm15, %ymm1; - vpxor %ymm2, %ymm15, %ymm2; - vpxor %ymm3, %ymm15, %ymm3; - vpxor %ymm4, %ymm15, %ymm4; - vpxor %ymm5, %ymm15, %ymm5; - vpxor %ymm6, %ymm15, %ymm6; - vpxor %ymm7, %ymm15, %ymm7; - vpxor %ymm8, %ymm15, %ymm8; - vpxor %ymm9, %ymm15, %ymm9; - vpxor %ymm10, %ymm15, %ymm10; - vpxor 11 * 32(%rax), %ymm15, %ymm11; - vpxor 12 * 32(%rax), %ymm15, %ymm12; - vpxor 13 * 32(%rax), %ymm15, %ymm13; - vpxor 14 * 32(%rax), %ymm15, %ymm14; - vpxor 15 * 32(%rax), %ymm15, %ymm15; - - call __camellia_enc_blk32; - - movq %r10, %rsp; - - vpxor 0 * 32(%rdx), %ymm7, %ymm7; - vpxor 1 * 32(%rdx), %ymm6, %ymm6; - vpxor 2 * 32(%rdx), %ymm5, %ymm5; - vpxor 3 * 32(%rdx), %ymm4, %ymm4; - vpxor 4 * 32(%rdx), %ymm3, %ymm3; - vpxor 5 * 32(%rdx), %ymm2, %ymm2; - vpxor 6 * 32(%rdx), %ymm1, %ymm1; - vpxor 7 * 32(%rdx), %ymm0, %ymm0; - vpxor 8 * 32(%rdx), %ymm15, %ymm15; - vpxor 9 * 32(%rdx), %ymm14, %ymm14; - vpxor 10 * 32(%rdx), %ymm13, %ymm13; - vpxor 11 * 32(%rdx), %ymm12, %ymm12; - vpxor 12 * 32(%rdx), %ymm11, %ymm11; - vpxor 13 * 32(%rdx), %ymm10, %ymm10; - vpxor 14 * 32(%rdx), %ymm9, %ymm9; - vpxor 15 * 32(%rdx), %ymm8, %ymm8; - write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, - %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, - %ymm8, %rsi); - - vzeroupper; - - FRAME_END - ret; -SYM_FUNC_END(camellia_ctr_32way) - -#define gf128mul_x_ble(iv, mask, tmp) \ - vpsrad $31, iv, tmp; \ - vpaddq iv, iv, iv; \ - vpshufd $0x13, tmp, tmp; \ - vpand mask, tmp, tmp; \ - vpxor tmp, iv, iv; - -#define gf128mul_x2_ble(iv, mask1, mask2, tmp0, tmp1) \ - vpsrad $31, iv, tmp0; \ - vpaddq iv, iv, tmp1; \ - vpsllq $2, iv, iv; \ - vpshufd $0x13, tmp0, tmp0; \ - vpsrad $31, tmp1, tmp1; \ - vpand mask2, tmp0, tmp0; \ - vpshufd $0x13, tmp1, tmp1; \ - vpxor tmp0, iv, iv; \ - vpand mask1, tmp1, tmp1; \ - vpxor tmp1, iv, iv; - -.align 8 -SYM_FUNC_START_LOCAL(camellia_xts_crypt_32way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (32 blocks) - * %rdx: src (32 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - * %r8: index for input whitening key - * %r9: pointer to __camellia_enc_blk32 or __camellia_dec_blk32 - */ - FRAME_BEGIN - - vzeroupper; - - subq $(16 * 32), %rsp; - movq %rsp, %rax; - - vbroadcasti128 .Lxts_gf128mul_and_shl1_mask_0, %ymm12; - - /* load IV and construct second IV */ - vmovdqu (%rcx), %xmm0; - vmovdqa %xmm0, %xmm15; - gf128mul_x_ble(%xmm0, %xmm12, %xmm13); - vbroadcasti128 .Lxts_gf128mul_and_shl1_mask_1, %ymm13; - vinserti128 $1, %xmm0, %ymm15, %ymm0; - vpxor 0 * 32(%rdx), %ymm0, %ymm15; - vmovdqu %ymm15, 15 * 32(%rax); - vmovdqu %ymm0, 0 * 32(%rsi); - - /* construct IVs */ - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 1 * 32(%rdx), %ymm0, %ymm15; - vmovdqu %ymm15, 14 * 32(%rax); - vmovdqu %ymm0, 1 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 2 * 32(%rdx), %ymm0, %ymm15; - vmovdqu %ymm15, 13 * 32(%rax); - vmovdqu %ymm0, 2 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 3 * 32(%rdx), %ymm0, %ymm15; - vmovdqu %ymm15, 12 * 32(%rax); - vmovdqu %ymm0, 3 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 4 * 32(%rdx), %ymm0, %ymm11; - vmovdqu %ymm0, 4 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 5 * 32(%rdx), %ymm0, %ymm10; - vmovdqu %ymm0, 5 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 6 * 32(%rdx), %ymm0, %ymm9; - vmovdqu %ymm0, 6 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 7 * 32(%rdx), %ymm0, %ymm8; - vmovdqu %ymm0, 7 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 8 * 32(%rdx), %ymm0, %ymm7; - vmovdqu %ymm0, 8 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 9 * 32(%rdx), %ymm0, %ymm6; - vmovdqu %ymm0, 9 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 10 * 32(%rdx), %ymm0, %ymm5; - vmovdqu %ymm0, 10 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 11 * 32(%rdx), %ymm0, %ymm4; - vmovdqu %ymm0, 11 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 12 * 32(%rdx), %ymm0, %ymm3; - vmovdqu %ymm0, 12 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 13 * 32(%rdx), %ymm0, %ymm2; - vmovdqu %ymm0, 13 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 14 * 32(%rdx), %ymm0, %ymm1; - vmovdqu %ymm0, 14 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 15 * 32(%rdx), %ymm0, %ymm15; - vmovdqu %ymm15, 0 * 32(%rax); - vmovdqu %ymm0, 15 * 32(%rsi); - - vextracti128 $1, %ymm0, %xmm0; - gf128mul_x_ble(%xmm0, %xmm12, %xmm15); - vmovdqu %xmm0, (%rcx); - - /* inpack32_pre: */ - vpbroadcastq (key_table)(CTX, %r8, 8), %ymm15; - vpshufb .Lpack_bswap, %ymm15, %ymm15; - vpxor 0 * 32(%rax), %ymm15, %ymm0; - vpxor %ymm1, %ymm15, %ymm1; - vpxor %ymm2, %ymm15, %ymm2; - vpxor %ymm3, %ymm15, %ymm3; - vpxor %ymm4, %ymm15, %ymm4; - vpxor %ymm5, %ymm15, %ymm5; - vpxor %ymm6, %ymm15, %ymm6; - vpxor %ymm7, %ymm15, %ymm7; - vpxor %ymm8, %ymm15, %ymm8; - vpxor %ymm9, %ymm15, %ymm9; - vpxor %ymm10, %ymm15, %ymm10; - vpxor %ymm11, %ymm15, %ymm11; - vpxor 12 * 32(%rax), %ymm15, %ymm12; - vpxor 13 * 32(%rax), %ymm15, %ymm13; - vpxor 14 * 32(%rax), %ymm15, %ymm14; - vpxor 15 * 32(%rax), %ymm15, %ymm15; - - CALL_NOSPEC r9; - - addq $(16 * 32), %rsp; - - vpxor 0 * 32(%rsi), %ymm7, %ymm7; - vpxor 1 * 32(%rsi), %ymm6, %ymm6; - vpxor 2 * 32(%rsi), %ymm5, %ymm5; - vpxor 3 * 32(%rsi), %ymm4, %ymm4; - vpxor 4 * 32(%rsi), %ymm3, %ymm3; - vpxor 5 * 32(%rsi), %ymm2, %ymm2; - vpxor 6 * 32(%rsi), %ymm1, %ymm1; - vpxor 7 * 32(%rsi), %ymm0, %ymm0; - vpxor 8 * 32(%rsi), %ymm15, %ymm15; - vpxor 9 * 32(%rsi), %ymm14, %ymm14; - vpxor 10 * 32(%rsi), %ymm13, %ymm13; - vpxor 11 * 32(%rsi), %ymm12, %ymm12; - vpxor 12 * 32(%rsi), %ymm11, %ymm11; - vpxor 13 * 32(%rsi), %ymm10, %ymm10; - vpxor 14 * 32(%rsi), %ymm9, %ymm9; - vpxor 15 * 32(%rsi), %ymm8, %ymm8; - write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, - %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, - %ymm8, %rsi); - - vzeroupper; - - FRAME_END - ret; -SYM_FUNC_END(camellia_xts_crypt_32way) - -SYM_FUNC_START(camellia_xts_enc_32way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (32 blocks) - * %rdx: src (32 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - - xorl %r8d, %r8d; /* input whitening key, 0 for enc */ - - leaq __camellia_enc_blk32, %r9; - - jmp camellia_xts_crypt_32way; -SYM_FUNC_END(camellia_xts_enc_32way) - -SYM_FUNC_START(camellia_xts_dec_32way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (32 blocks) - * %rdx: src (32 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - - cmpl $16, key_length(CTX); - movl $32, %r8d; - movl $24, %eax; - cmovel %eax, %r8d; /* input whitening key, last for dec */ - - leaq __camellia_dec_blk32, %r9; - - jmp camellia_xts_crypt_32way; -SYM_FUNC_END(camellia_xts_dec_32way) diff --git a/arch/x86/include/asm/crypto/camellia.h b/arch/x86/crypto/camellia.h similarity index 69% rename from arch/x86/include/asm/crypto/camellia.h rename to arch/x86/crypto/camellia.h index f6d91861cb14..1dcea79e8f8e 100644 --- a/arch/x86/include/asm/crypto/camellia.h +++ b/arch/x86/crypto/camellia.h @@ -19,18 +19,10 @@ struct camellia_ctx { u32 key_length; }; -struct camellia_xts_ctx { - struct camellia_ctx tweak_ctx; - struct camellia_ctx crypt_ctx; -}; - extern int __camellia_setkey(struct camellia_ctx *cctx, const unsigned char *key, unsigned int key_len); -extern int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen); - /* regular block cipher functions */ asmlinkage void __camellia_enc_blk(const void *ctx, u8 *dst, const u8 *src, bool xor); @@ -46,13 +38,6 @@ asmlinkage void camellia_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void camellia_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void camellia_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src); -asmlinkage void camellia_ctr_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -asmlinkage void camellia_xts_enc_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void camellia_xts_dec_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); static inline void camellia_enc_blk(const void *ctx, u8 *dst, const u8 *src) { @@ -78,14 +63,5 @@ static inline void camellia_enc_blk_xor_2way(const void *ctx, u8 *dst, /* glue helpers */ extern void camellia_decrypt_cbc_2way(const void *ctx, u8 *dst, const u8 *src); -extern void camellia_crypt_ctr(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -extern void camellia_crypt_ctr_2way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -extern void camellia_xts_enc(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -extern void camellia_xts_dec(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); #endif /* ASM_X86_CAMELLIA_H */ diff --git a/arch/x86/crypto/camellia_aesni_avx2_glue.c b/arch/x86/crypto/camellia_aesni_avx2_glue.c index ccda647422d6..e7e4d64e9577 100644 --- a/arch/x86/crypto/camellia_aesni_avx2_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx2_glue.c @@ -5,16 +5,16 @@ * Copyright © 2013 Jussi Kivilinna */ -#include -#include #include #include -#include #include #include #include #include +#include "camellia.h" +#include "ecb_cbc_helpers.h" + #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32 @@ -23,121 +23,6 @@ asmlinkage void camellia_ecb_enc_32way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void camellia_ecb_dec_32way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void camellia_cbc_dec_32way(const void *ctx, u8 *dst, const u8 *src); -asmlinkage void camellia_ctr_32way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -asmlinkage void camellia_xts_enc_32way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void camellia_xts_dec_32way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -static const struct common_glue_ctx camellia_enc = { - .num_funcs = 4, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_enc_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_enc_16way } - }, { - .num_blocks = 2, - .fn_u = { .ecb = camellia_enc_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_enc_blk } - } } -}; - -static const struct common_glue_ctx camellia_ctr = { - .num_funcs = 4, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .ctr = camellia_ctr_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ctr = camellia_ctr_16way } - }, { - .num_blocks = 2, - .fn_u = { .ctr = camellia_crypt_ctr_2way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = camellia_crypt_ctr } - } } -}; - -static const struct common_glue_ctx camellia_enc_xts = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_enc_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_enc_16way } - }, { - .num_blocks = 1, - .fn_u = { .xts = camellia_xts_enc } - } } -}; - -static const struct common_glue_ctx camellia_dec = { - .num_funcs = 4, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_dec_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_dec_16way } - }, { - .num_blocks = 2, - .fn_u = { .ecb = camellia_dec_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_dec_blk } - } } -}; - -static const struct common_glue_ctx camellia_dec_cbc = { - .num_funcs = 4, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .cbc = camellia_cbc_dec_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .cbc = camellia_cbc_dec_16way } - }, { - .num_blocks = 2, - .fn_u = { .cbc = camellia_decrypt_cbc_2way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = camellia_dec_blk } - } } -}; - -static const struct common_glue_ctx camellia_dec_xts = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_dec_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_dec_16way } - }, { - .num_blocks = 1, - .fn_u = { .xts = camellia_xts_dec } - } } -}; static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -147,45 +32,39 @@ static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_enc, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_ecb_enc_32way); + ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_enc_16way); + ECB_BLOCK(2, camellia_enc_blk_2way); + ECB_BLOCK(1, camellia_enc_blk); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_dec, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_ecb_dec_32way); + ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_dec_16way); + ECB_BLOCK(2, camellia_dec_blk_2way); + ECB_BLOCK(1, camellia_dec_blk); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(camellia_enc_blk); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&camellia_ctr, req); -} - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, true); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_cbc_dec_32way); + CBC_DEC_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_cbc_dec_16way); + CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way); + CBC_DEC_BLOCK(1, camellia_dec_blk); + CBC_WALK_END(); } static struct skcipher_alg camellia_algs[] = { @@ -216,35 +95,6 @@ static struct skcipher_alg camellia_algs[] = { .setkey = camellia_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(camellia)", - .base.cra_driver_name = "__ctr-camellia-aesni-avx2", - .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct camellia_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = CAMELLIA_MIN_KEY_SIZE, - .max_keysize = CAMELLIA_MAX_KEY_SIZE, - .ivsize = CAMELLIA_BLOCK_SIZE, - .chunksize = CAMELLIA_BLOCK_SIZE, - .setkey = camellia_setkey, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(camellia)", - .base.cra_driver_name = "__xts-camellia-aesni-avx2", - .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct camellia_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * CAMELLIA_MIN_KEY_SIZE, - .max_keysize = 2 * CAMELLIA_MAX_KEY_SIZE, - .ivsize = CAMELLIA_BLOCK_SIZE, - .setkey = xts_camellia_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, }, }; diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c index 4e5de6ef206e..c7ccf63e741e 100644 --- a/arch/x86/crypto/camellia_aesni_avx_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx_glue.c @@ -5,16 +5,16 @@ * Copyright © 2012-2013 Jussi Kivilinna */ -#include -#include #include #include -#include #include #include #include #include +#include "camellia.h" +#include "ecb_cbc_helpers.h" + #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 /* 16-way parallel cipher functions (avx/aes-ni) */ @@ -27,120 +27,6 @@ EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way); asmlinkage void camellia_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src); EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way); -asmlinkage void camellia_ctr_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -EXPORT_SYMBOL_GPL(camellia_ctr_16way); - -asmlinkage void camellia_xts_enc_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -EXPORT_SYMBOL_GPL(camellia_xts_enc_16way); - -asmlinkage void camellia_xts_dec_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -EXPORT_SYMBOL_GPL(camellia_xts_dec_16way); - -void camellia_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, camellia_enc_blk); -} -EXPORT_SYMBOL_GPL(camellia_xts_enc); - -void camellia_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, camellia_dec_blk); -} -EXPORT_SYMBOL_GPL(camellia_xts_dec); - -static const struct common_glue_ctx camellia_enc = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_enc_16way } - }, { - .num_blocks = 2, - .fn_u = { .ecb = camellia_enc_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_enc_blk } - } } -}; - -static const struct common_glue_ctx camellia_ctr = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ctr = camellia_ctr_16way } - }, { - .num_blocks = 2, - .fn_u = { .ctr = camellia_crypt_ctr_2way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = camellia_crypt_ctr } - } } -}; - -static const struct common_glue_ctx camellia_enc_xts = { - .num_funcs = 2, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_enc_16way } - }, { - .num_blocks = 1, - .fn_u = { .xts = camellia_xts_enc } - } } -}; - -static const struct common_glue_ctx camellia_dec = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_dec_16way } - }, { - .num_blocks = 2, - .fn_u = { .ecb = camellia_dec_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_dec_blk } - } } -}; - -static const struct common_glue_ctx camellia_dec_cbc = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .cbc = camellia_cbc_dec_16way } - }, { - .num_blocks = 2, - .fn_u = { .cbc = camellia_decrypt_cbc_2way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = camellia_dec_blk } - } } -}; - -static const struct common_glue_ctx camellia_dec_xts = { - .num_funcs = 2, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_dec_16way } - }, { - .num_blocks = 1, - .fn_u = { .xts = camellia_xts_dec } - } } -}; - static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { @@ -149,65 +35,36 @@ static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_enc, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_enc_16way); + ECB_BLOCK(2, camellia_enc_blk_2way); + ECB_BLOCK(1, camellia_enc_blk); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_dec, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_dec_16way); + ECB_BLOCK(2, camellia_dec_blk_2way); + ECB_BLOCK(1, camellia_dec_blk); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(camellia_enc_blk); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&camellia_ctr, req); -} - -int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = xts_verify_key(tfm, key, keylen); - if (err) - return err; - - /* first half of xts-key is for crypt */ - err = __camellia_setkey(&ctx->crypt_ctx, key, keylen / 2); - if (err) - return err; - - /* second half of xts-key is for tweak */ - return __camellia_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); -} -EXPORT_SYMBOL_GPL(xts_camellia_setkey); - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, true); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_cbc_dec_16way); + CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way); + CBC_DEC_BLOCK(1, camellia_dec_blk); + CBC_WALK_END(); } static struct skcipher_alg camellia_algs[] = { @@ -238,36 +95,7 @@ static struct skcipher_alg camellia_algs[] = { .setkey = camellia_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(camellia)", - .base.cra_driver_name = "__ctr-camellia-aesni", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct camellia_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = CAMELLIA_MIN_KEY_SIZE, - .max_keysize = CAMELLIA_MAX_KEY_SIZE, - .ivsize = CAMELLIA_BLOCK_SIZE, - .chunksize = CAMELLIA_BLOCK_SIZE, - .setkey = camellia_setkey, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(camellia)", - .base.cra_driver_name = "__xts-camellia-aesni", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct camellia_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * CAMELLIA_MIN_KEY_SIZE, - .max_keysize = 2 * CAMELLIA_MAX_KEY_SIZE, - .ivsize = CAMELLIA_BLOCK_SIZE, - .setkey = xts_camellia_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, - }, + } }; static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c index 242c056e5fa8..66c435ba9d3d 100644 --- a/arch/x86/crypto/camellia_glue.c +++ b/arch/x86/crypto/camellia_glue.c @@ -14,8 +14,9 @@ #include #include #include -#include -#include + +#include "camellia.h" +#include "ecb_cbc_helpers.h" /* regular block cipher functions */ asmlinkage void __camellia_enc_blk(const void *ctx, u8 *dst, const u8 *src, @@ -1262,129 +1263,47 @@ static int camellia_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, return camellia_setkey(&tfm->base, key, key_len); } -void camellia_decrypt_cbc_2way(const void *ctx, u8 *d, const u8 *s) +void camellia_decrypt_cbc_2way(const void *ctx, u8 *dst, const u8 *src) { - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - u128 iv = *src; + u8 buf[CAMELLIA_BLOCK_SIZE]; + const u8 *iv = src; - camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src); - - u128_xor(&dst[1], &dst[1], &iv); + if (dst == src) + iv = memcpy(buf, iv, sizeof(buf)); + camellia_dec_blk_2way(ctx, dst, src); + crypto_xor(dst + CAMELLIA_BLOCK_SIZE, iv, CAMELLIA_BLOCK_SIZE); } EXPORT_SYMBOL_GPL(camellia_decrypt_cbc_2way); -void camellia_crypt_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblk; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - if (dst != src) - *dst = *src; - - le128_to_be128(&ctrblk, iv); - le128_inc(iv); - - camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk); -} -EXPORT_SYMBOL_GPL(camellia_crypt_ctr); - -void camellia_crypt_ctr_2way(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblks[2]; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - if (dst != src) { - dst[0] = src[0]; - dst[1] = src[1]; - } - - le128_to_be128(&ctrblks[0], iv); - le128_inc(iv); - le128_to_be128(&ctrblks[1], iv); - le128_inc(iv); - - camellia_enc_blk_xor_2way(ctx, (u8 *)dst, (u8 *)ctrblks); -} -EXPORT_SYMBOL_GPL(camellia_crypt_ctr_2way); - -static const struct common_glue_ctx camellia_enc = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 2, - .fn_u = { .ecb = camellia_enc_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_enc_blk } - } } -}; - -static const struct common_glue_ctx camellia_ctr = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 2, - .fn_u = { .ctr = camellia_crypt_ctr_2way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = camellia_crypt_ctr } - } } -}; - -static const struct common_glue_ctx camellia_dec = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 2, - .fn_u = { .ecb = camellia_dec_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_dec_blk } - } } -}; - -static const struct common_glue_ctx camellia_dec_cbc = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 2, - .fn_u = { .cbc = camellia_decrypt_cbc_2way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = camellia_dec_blk } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_enc, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + ECB_BLOCK(2, camellia_enc_blk_2way); + ECB_BLOCK(1, camellia_enc_blk); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_dec, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + ECB_BLOCK(2, camellia_dec_blk_2way); + ECB_BLOCK(1, camellia_dec_blk); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(camellia_enc_blk); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&camellia_ctr, req); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way); + CBC_DEC_BLOCK(1, camellia_dec_blk); + CBC_WALK_END(); } static struct crypto_alg camellia_cipher_alg = { @@ -1433,20 +1352,6 @@ static struct skcipher_alg camellia_skcipher_algs[] = { .setkey = camellia_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "ctr(camellia)", - .base.cra_driver_name = "ctr-camellia-asm", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct camellia_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = CAMELLIA_MIN_KEY_SIZE, - .max_keysize = CAMELLIA_MAX_KEY_SIZE, - .ivsize = CAMELLIA_BLOCK_SIZE, - .chunksize = CAMELLIA_BLOCK_SIZE, - .setkey = camellia_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, } }; diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c index 384ccb00f9e1..3976a87f92ad 100644 --- a/arch/x86/crypto/cast5_avx_glue.c +++ b/arch/x86/crypto/cast5_avx_glue.c @@ -6,7 +6,6 @@ * */ -#include #include #include #include @@ -15,6 +14,8 @@ #include #include +#include "ecb_cbc_helpers.h" + #define CAST5_PARALLEL_BLOCKS 16 asmlinkage void cast5_ecb_enc_16way(struct cast5_ctx *ctx, u8 *dst, @@ -23,8 +24,6 @@ asmlinkage void cast5_ecb_dec_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src); asmlinkage void cast5_cbc_dec_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src); -asmlinkage void cast5_ctr_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src, - __be64 *iv); static int cast5_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -32,272 +31,35 @@ static int cast5_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, return cast5_setkey(&tfm->base, key, keylen); } -static inline bool cast5_fpu_begin(bool fpu_enabled, struct skcipher_walk *walk, - unsigned int nbytes) -{ - return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS, - walk, fpu_enabled, nbytes); -} - -static inline void cast5_fpu_end(bool fpu_enabled) -{ - return glue_fpu_end(fpu_enabled); -} - -static int ecb_crypt(struct skcipher_request *req, bool enc) -{ - bool fpu_enabled = false; - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - const unsigned int bsize = CAST5_BLOCK_SIZE; - unsigned int nbytes; - void (*fn)(struct cast5_ctx *ctx, u8 *dst, const u8 *src); - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - u8 *wsrc = walk.src.virt.addr; - u8 *wdst = walk.dst.virt.addr; - - fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); - - /* Process multi-block batch */ - if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { - fn = (enc) ? cast5_ecb_enc_16way : cast5_ecb_dec_16way; - do { - fn(ctx, wdst, wsrc); - - wsrc += bsize * CAST5_PARALLEL_BLOCKS; - wdst += bsize * CAST5_PARALLEL_BLOCKS; - nbytes -= bsize * CAST5_PARALLEL_BLOCKS; - } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS); - - if (nbytes < bsize) - goto done; - } - - fn = (enc) ? __cast5_encrypt : __cast5_decrypt; - - /* Handle leftovers */ - do { - fn(ctx, wdst, wsrc); - - wsrc += bsize; - wdst += bsize; - nbytes -= bsize; - } while (nbytes >= bsize); - -done: - err = skcipher_walk_done(&walk, nbytes); - } - - cast5_fpu_end(fpu_enabled); - return err; -} - static int ecb_encrypt(struct skcipher_request *req) { - return ecb_crypt(req, true); + ECB_WALK_START(req, CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS); + ECB_BLOCK(CAST5_PARALLEL_BLOCKS, cast5_ecb_enc_16way); + ECB_BLOCK(1, __cast5_encrypt); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return ecb_crypt(req, false); + ECB_WALK_START(req, CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS); + ECB_BLOCK(CAST5_PARALLEL_BLOCKS, cast5_ecb_dec_16way); + ECB_BLOCK(1, __cast5_decrypt); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - const unsigned int bsize = CAST5_BLOCK_SIZE; - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - u64 *src = (u64 *)walk.src.virt.addr; - u64 *dst = (u64 *)walk.dst.virt.addr; - u64 *iv = (u64 *)walk.iv; - - do { - *dst = *src ^ *iv; - __cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst); - iv = dst; - src++; - dst++; - nbytes -= bsize; - } while (nbytes >= bsize); - - *(u64 *)walk.iv = *iv; - err = skcipher_walk_done(&walk, nbytes); - } - - return err; -} - -static unsigned int __cbc_decrypt(struct cast5_ctx *ctx, - struct skcipher_walk *walk) -{ - const unsigned int bsize = CAST5_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u64 *src = (u64 *)walk->src.virt.addr; - u64 *dst = (u64 *)walk->dst.virt.addr; - u64 last_iv; - - /* Start of the last block. */ - src += nbytes / bsize - 1; - dst += nbytes / bsize - 1; - - last_iv = *src; - - /* Process multi-block batch */ - if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { - do { - nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1); - src -= CAST5_PARALLEL_BLOCKS - 1; - dst -= CAST5_PARALLEL_BLOCKS - 1; - - cast5_cbc_dec_16way(ctx, (u8 *)dst, (u8 *)src); - - nbytes -= bsize; - if (nbytes < bsize) - goto done; - - *dst ^= *(src - 1); - src -= 1; - dst -= 1; - } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS); - } - - /* Handle leftovers */ - for (;;) { - __cast5_decrypt(ctx, (u8 *)dst, (u8 *)src); - - nbytes -= bsize; - if (nbytes < bsize) - break; - - *dst ^= *(src - 1); - src -= 1; - dst -= 1; - } - -done: - *dst ^= *(u64 *)walk->iv; - *(u64 *)walk->iv = last_iv; - - return nbytes; + CBC_WALK_START(req, CAST5_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(__cast5_encrypt); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm); - bool fpu_enabled = false; - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); - nbytes = __cbc_decrypt(ctx, &walk); - err = skcipher_walk_done(&walk, nbytes); - } - - cast5_fpu_end(fpu_enabled); - return err; -} - -static void ctr_crypt_final(struct skcipher_walk *walk, struct cast5_ctx *ctx) -{ - u8 *ctrblk = walk->iv; - u8 keystream[CAST5_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; - - __cast5_encrypt(ctx, keystream, ctrblk); - crypto_xor_cpy(dst, keystream, src, nbytes); - - crypto_inc(ctrblk, CAST5_BLOCK_SIZE); -} - -static unsigned int __ctr_crypt(struct skcipher_walk *walk, - struct cast5_ctx *ctx) -{ - const unsigned int bsize = CAST5_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u64 *src = (u64 *)walk->src.virt.addr; - u64 *dst = (u64 *)walk->dst.virt.addr; - - /* Process multi-block batch */ - if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { - do { - cast5_ctr_16way(ctx, (u8 *)dst, (u8 *)src, - (__be64 *)walk->iv); - - src += CAST5_PARALLEL_BLOCKS; - dst += CAST5_PARALLEL_BLOCKS; - nbytes -= bsize * CAST5_PARALLEL_BLOCKS; - } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - u64 ctrblk; - - if (dst != src) - *dst = *src; - - ctrblk = *(u64 *)walk->iv; - be64_add_cpu((__be64 *)walk->iv, 1); - - __cast5_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); - *dst ^= ctrblk; - - src += 1; - dst += 1; - nbytes -= bsize; - } while (nbytes >= bsize); - -done: - return nbytes; -} - -static int ctr_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm); - bool fpu_enabled = false; - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) { - fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); - nbytes = __ctr_crypt(&walk, ctx); - err = skcipher_walk_done(&walk, nbytes); - } - - cast5_fpu_end(fpu_enabled); - - if (walk.nbytes) { - ctr_crypt_final(&walk, ctx); - err = skcipher_walk_done(&walk, 0); - } - - return err; + CBC_WALK_START(req, CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(CAST5_PARALLEL_BLOCKS, cast5_cbc_dec_16way); + CBC_DEC_BLOCK(1, __cast5_decrypt); + CBC_WALK_END(); } static struct skcipher_alg cast5_algs[] = { @@ -328,21 +90,6 @@ static struct skcipher_alg cast5_algs[] = { .setkey = cast5_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(cast5)", - .base.cra_driver_name = "__ctr-cast5-avx", - .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct cast5_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = CAST5_MIN_KEY_SIZE, - .max_keysize = CAST5_MAX_KEY_SIZE, - .ivsize = CAST5_BLOCK_SIZE, - .chunksize = CAST5_BLOCK_SIZE, - .setkey = cast5_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, } }; diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S index 932a3ce32a88..fbddcecc3e3f 100644 --- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S @@ -212,8 +212,6 @@ .section .rodata.cst16, "aM", @progbits, 16 .align 16 -.Lxts_gf128mul_and_shl1_mask: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 .Lbswap_mask: .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 .Lbswap128_mask: @@ -412,85 +410,3 @@ SYM_FUNC_START(cast6_cbc_dec_8way) FRAME_END ret; SYM_FUNC_END(cast6_cbc_dec_8way) - -SYM_FUNC_START(cast6_ctr_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - pushq %r12; - pushq %r15 - - movq %rdi, CTX; - movq %rsi, %r11; - movq %rdx, %r12; - - load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RX, RKR, RKM); - - call __cast6_enc_blk8; - - store_ctr_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - popq %r15; - popq %r12; - FRAME_END - ret; -SYM_FUNC_END(cast6_ctr_8way) - -SYM_FUNC_START(cast6_xts_enc_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - pushq %r15; - - movq %rdi, CTX - movq %rsi, %r11; - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, - RX, RKR, RKM, .Lxts_gf128mul_and_shl1_mask); - - call __cast6_enc_blk8; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - popq %r15; - FRAME_END - ret; -SYM_FUNC_END(cast6_xts_enc_8way) - -SYM_FUNC_START(cast6_xts_dec_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - pushq %r15; - - movq %rdi, CTX - movq %rsi, %r11; - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, - RX, RKR, RKM, .Lxts_gf128mul_and_shl1_mask); - - call __cast6_dec_blk8; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - popq %r15; - FRAME_END - ret; -SYM_FUNC_END(cast6_xts_dec_8way) diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c index 48e0f37796fa..7e2aea372349 100644 --- a/arch/x86/crypto/cast6_avx_glue.c +++ b/arch/x86/crypto/cast6_avx_glue.c @@ -15,8 +15,8 @@ #include #include #include -#include -#include + +#include "ecb_cbc_helpers.h" #define CAST6_PARALLEL_BLOCKS 8 @@ -24,13 +24,6 @@ asmlinkage void cast6_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void cast6_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void cast6_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src); -asmlinkage void cast6_ctr_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -asmlinkage void cast6_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void cast6_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); static int cast6_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -38,172 +31,35 @@ static int cast6_setkey_skcipher(struct crypto_skcipher *tfm, return cast6_setkey(&tfm->base, key, keylen); } -static void cast6_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, __cast6_encrypt); -} - -static void cast6_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, __cast6_decrypt); -} - -static void cast6_crypt_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblk; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - le128_to_be128(&ctrblk, iv); - le128_inc(iv); - - __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); - u128_xor(dst, src, (u128 *)&ctrblk); -} - -static const struct common_glue_ctx cast6_enc = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .ecb = cast6_ecb_enc_8way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __cast6_encrypt } - } } -}; - -static const struct common_glue_ctx cast6_ctr = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .ctr = cast6_ctr_8way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = cast6_crypt_ctr } - } } -}; - -static const struct common_glue_ctx cast6_enc_xts = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .xts = cast6_xts_enc_8way } - }, { - .num_blocks = 1, - .fn_u = { .xts = cast6_xts_enc } - } } -}; - -static const struct common_glue_ctx cast6_dec = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .ecb = cast6_ecb_dec_8way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __cast6_decrypt } - } } -}; - -static const struct common_glue_ctx cast6_dec_cbc = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .cbc = cast6_cbc_dec_8way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = __cast6_decrypt } - } } -}; - -static const struct common_glue_ctx cast6_dec_xts = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .xts = cast6_xts_dec_8way } - }, { - .num_blocks = 1, - .fn_u = { .xts = cast6_xts_dec } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&cast6_enc, req); + ECB_WALK_START(req, CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS); + ECB_BLOCK(CAST6_PARALLEL_BLOCKS, cast6_ecb_enc_8way); + ECB_BLOCK(1, __cast6_encrypt); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&cast6_dec, req); + ECB_WALK_START(req, CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS); + ECB_BLOCK(CAST6_PARALLEL_BLOCKS, cast6_ecb_dec_8way); + ECB_BLOCK(1, __cast6_decrypt); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(__cast6_encrypt, req); + CBC_WALK_START(req, CAST6_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(__cast6_encrypt); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&cast6_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&cast6_ctr, req); -} - -struct cast6_xts_ctx { - struct cast6_ctx tweak_ctx; - struct cast6_ctx crypt_ctx; -}; - -static int xts_cast6_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = xts_verify_key(tfm, key, keylen); - if (err) - return err; - - /* first half of xts-key is for crypt */ - err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2); - if (err) - return err; - - /* second half of xts-key is for tweak */ - return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); -} - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&cast6_enc_xts, req, __cast6_encrypt, - &ctx->tweak_ctx, &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&cast6_dec_xts, req, __cast6_encrypt, - &ctx->tweak_ctx, &ctx->crypt_ctx, true); + CBC_WALK_START(req, CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(CAST6_PARALLEL_BLOCKS, cast6_cbc_dec_8way); + CBC_DEC_BLOCK(1, __cast6_decrypt); + CBC_WALK_END(); } static struct skcipher_alg cast6_algs[] = { @@ -234,35 +90,6 @@ static struct skcipher_alg cast6_algs[] = { .setkey = cast6_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(cast6)", - .base.cra_driver_name = "__ctr-cast6-avx", - .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct cast6_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = CAST6_MIN_KEY_SIZE, - .max_keysize = CAST6_MAX_KEY_SIZE, - .ivsize = CAST6_BLOCK_SIZE, - .chunksize = CAST6_BLOCK_SIZE, - .setkey = cast6_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(cast6)", - .base.cra_driver_name = "__xts-cast6-avx", - .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = CAST6_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct cast6_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * CAST6_MIN_KEY_SIZE, - .max_keysize = 2 * CAST6_MAX_KEY_SIZE, - .ivsize = CAST6_BLOCK_SIZE, - .setkey = xts_cast6_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, }, }; diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c index 89830e531350..e7cb68a3db3b 100644 --- a/arch/x86/crypto/des3_ede_glue.c +++ b/arch/x86/crypto/des3_ede_glue.c @@ -6,8 +6,6 @@ * * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: * Copyright (c) 2006 Herbert Xu - * CTR part based on code (crypto/ctr.c) by: - * (C) Copyright IBM Corp. 2007 - Joy Latten */ #include @@ -253,94 +251,6 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } -static void ctr_crypt_final(struct des3_ede_x86_ctx *ctx, - struct skcipher_walk *walk) -{ - u8 *ctrblk = walk->iv; - u8 keystream[DES3_EDE_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; - - des3_ede_enc_blk(ctx, keystream, ctrblk); - crypto_xor_cpy(dst, keystream, src, nbytes); - - crypto_inc(ctrblk, DES3_EDE_BLOCK_SIZE); -} - -static unsigned int __ctr_crypt(struct des3_ede_x86_ctx *ctx, - struct skcipher_walk *walk) -{ - unsigned int bsize = DES3_EDE_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - __be64 *src = (__be64 *)walk->src.virt.addr; - __be64 *dst = (__be64 *)walk->dst.virt.addr; - u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv); - __be64 ctrblocks[3]; - - /* Process four block batch */ - if (nbytes >= bsize * 3) { - do { - /* create ctrblks for parallel encrypt */ - ctrblocks[0] = cpu_to_be64(ctrblk++); - ctrblocks[1] = cpu_to_be64(ctrblk++); - ctrblocks[2] = cpu_to_be64(ctrblk++); - - des3_ede_enc_blk_3way(ctx, (u8 *)ctrblocks, - (u8 *)ctrblocks); - - dst[0] = src[0] ^ ctrblocks[0]; - dst[1] = src[1] ^ ctrblocks[1]; - dst[2] = src[2] ^ ctrblocks[2]; - - src += 3; - dst += 3; - } while ((nbytes -= bsize * 3) >= bsize * 3); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - ctrblocks[0] = cpu_to_be64(ctrblk++); - - des3_ede_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks); - - dst[0] = src[0] ^ ctrblocks[0]; - - src += 1; - dst += 1; - } while ((nbytes -= bsize) >= bsize); - -done: - *(__be64 *)walk->iv = cpu_to_be64(ctrblk); - return nbytes; -} - -static int ctr_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) >= DES3_EDE_BLOCK_SIZE) { - nbytes = __ctr_crypt(ctx, &walk); - err = skcipher_walk_done(&walk, nbytes); - } - - if (nbytes) { - ctr_crypt_final(ctx, &walk); - err = skcipher_walk_done(&walk, 0); - } - - return err; -} - static int des3_ede_x86_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { @@ -428,20 +338,6 @@ static struct skcipher_alg des3_ede_skciphers[] = { .setkey = des3_ede_x86_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "ctr(des3_ede)", - .base.cra_driver_name = "ctr-des3_ede-asm", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct des3_ede_x86_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .chunksize = DES3_EDE_BLOCK_SIZE, - .setkey = des3_ede_x86_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, } }; diff --git a/arch/x86/crypto/ecb_cbc_helpers.h b/arch/x86/crypto/ecb_cbc_helpers.h new file mode 100644 index 000000000000..eaa15c7b29d6 --- /dev/null +++ b/arch/x86/crypto/ecb_cbc_helpers.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _CRYPTO_ECB_CBC_HELPER_H +#define _CRYPTO_ECB_CBC_HELPER_H + +#include +#include + +/* + * Mode helpers to instantiate parameterized skcipher ECB/CBC modes without + * having to rely on indirect calls and retpolines. + */ + +#define ECB_WALK_START(req, bsize, fpu_blocks) do { \ + void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); \ + const int __bsize = (bsize); \ + struct skcipher_walk walk; \ + int err = skcipher_walk_virt(&walk, (req), false); \ + while (walk.nbytes > 0) { \ + unsigned int nbytes = walk.nbytes; \ + bool do_fpu = (fpu_blocks) != -1 && \ + nbytes >= (fpu_blocks) * __bsize; \ + const u8 *src = walk.src.virt.addr; \ + u8 *dst = walk.dst.virt.addr; \ + u8 __maybe_unused buf[(bsize)]; \ + if (do_fpu) kernel_fpu_begin() + +#define CBC_WALK_START(req, bsize, fpu_blocks) \ + ECB_WALK_START(req, bsize, fpu_blocks) + +#define ECB_WALK_ADVANCE(blocks) do { \ + dst += (blocks) * __bsize; \ + src += (blocks) * __bsize; \ + nbytes -= (blocks) * __bsize; \ +} while (0) + +#define ECB_BLOCK(blocks, func) do { \ + while (nbytes >= (blocks) * __bsize) { \ + (func)(ctx, dst, src); \ + ECB_WALK_ADVANCE(blocks); \ + } \ +} while (0) + +#define CBC_ENC_BLOCK(func) do { \ + const u8 *__iv = walk.iv; \ + while (nbytes >= __bsize) { \ + crypto_xor_cpy(dst, src, __iv, __bsize); \ + (func)(ctx, dst, dst); \ + __iv = dst; \ + ECB_WALK_ADVANCE(1); \ + } \ + memcpy(walk.iv, __iv, __bsize); \ +} while (0) + +#define CBC_DEC_BLOCK(blocks, func) do { \ + while (nbytes >= (blocks) * __bsize) { \ + const u8 *__iv = src + ((blocks) - 1) * __bsize; \ + if (dst == src) \ + __iv = memcpy(buf, __iv, __bsize); \ + (func)(ctx, dst, src); \ + crypto_xor(dst, walk.iv, __bsize); \ + memcpy(walk.iv, __iv, __bsize); \ + ECB_WALK_ADVANCE(blocks); \ + } \ +} while (0) + +#define ECB_WALK_END() \ + if (do_fpu) kernel_fpu_end(); \ + err = skcipher_walk_done(&walk, nbytes); \ + } \ + return err; \ +} while (0) + +#define CBC_WALK_END() ECB_WALK_END() + +#endif diff --git a/arch/x86/crypto/glue_helper-asm-avx.S b/arch/x86/crypto/glue_helper-asm-avx.S index d08fc575ef7f..3da385271227 100644 --- a/arch/x86/crypto/glue_helper-asm-avx.S +++ b/arch/x86/crypto/glue_helper-asm-avx.S @@ -34,107 +34,3 @@ vpxor (5*16)(src), x6, x6; \ vpxor (6*16)(src), x7, x7; \ store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7); - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -#define load_ctr_8way(iv, bswap, x0, x1, x2, x3, x4, x5, x6, x7, t0, t1, t2) \ - vpcmpeqd t0, t0, t0; \ - vpsrldq $8, t0, t0; /* low: -1, high: 0 */ \ - vmovdqa bswap, t1; \ - \ - /* load IV and byteswap */ \ - vmovdqu (iv), x7; \ - vpshufb t1, x7, x0; \ - \ - /* construct IVs */ \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x1; \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x2; \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x3; \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x4; \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x5; \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x6; \ - inc_le128(x7, t0, t2); \ - vmovdqa x7, t2; \ - vpshufb t1, x7, x7; \ - inc_le128(t2, t0, t1); \ - vmovdqu t2, (iv); - -#define store_ctr_8way(src, dst, x0, x1, x2, x3, x4, x5, x6, x7) \ - vpxor (0*16)(src), x0, x0; \ - vpxor (1*16)(src), x1, x1; \ - vpxor (2*16)(src), x2, x2; \ - vpxor (3*16)(src), x3, x3; \ - vpxor (4*16)(src), x4, x4; \ - vpxor (5*16)(src), x5, x5; \ - vpxor (6*16)(src), x6, x6; \ - vpxor (7*16)(src), x7, x7; \ - store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7); - -#define gf128mul_x_ble(iv, mask, tmp) \ - vpsrad $31, iv, tmp; \ - vpaddq iv, iv, iv; \ - vpshufd $0x13, tmp, tmp; \ - vpand mask, tmp, tmp; \ - vpxor tmp, iv, iv; - -#define load_xts_8way(iv, src, dst, x0, x1, x2, x3, x4, x5, x6, x7, tiv, t0, \ - t1, xts_gf128mul_and_shl1_mask) \ - vmovdqa xts_gf128mul_and_shl1_mask, t0; \ - \ - /* load IV */ \ - vmovdqu (iv), tiv; \ - vpxor (0*16)(src), tiv, x0; \ - vmovdqu tiv, (0*16)(dst); \ - \ - /* construct and store IVs, also xor with source */ \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (1*16)(src), tiv, x1; \ - vmovdqu tiv, (1*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (2*16)(src), tiv, x2; \ - vmovdqu tiv, (2*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (3*16)(src), tiv, x3; \ - vmovdqu tiv, (3*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (4*16)(src), tiv, x4; \ - vmovdqu tiv, (4*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (5*16)(src), tiv, x5; \ - vmovdqu tiv, (5*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (6*16)(src), tiv, x6; \ - vmovdqu tiv, (6*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (7*16)(src), tiv, x7; \ - vmovdqu tiv, (7*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vmovdqu tiv, (iv); - -#define store_xts_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7) \ - vpxor (0*16)(dst), x0, x0; \ - vpxor (1*16)(dst), x1, x1; \ - vpxor (2*16)(dst), x2, x2; \ - vpxor (3*16)(dst), x3, x3; \ - vpxor (4*16)(dst), x4, x4; \ - vpxor (5*16)(dst), x5, x5; \ - vpxor (6*16)(dst), x6, x6; \ - vpxor (7*16)(dst), x7, x7; \ - store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7); diff --git a/arch/x86/crypto/glue_helper-asm-avx2.S b/arch/x86/crypto/glue_helper-asm-avx2.S index d84508c85c13..c77e9049431f 100644 --- a/arch/x86/crypto/glue_helper-asm-avx2.S +++ b/arch/x86/crypto/glue_helper-asm-avx2.S @@ -37,139 +37,3 @@ vpxor (5*32+16)(src), x6, x6; \ vpxor (6*32+16)(src), x7, x7; \ store_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7); - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ - vpcmpeqq minus_one, x, tmp1; \ - vpcmpeqq minus_two, x, tmp2; \ - vpsubq minus_two, x, x; \ - vpor tmp2, tmp1, tmp1; \ - vpslldq $8, tmp1, tmp1; \ - vpsubq tmp1, x, x; - -#define load_ctr_16way(iv, bswap, x0, x1, x2, x3, x4, x5, x6, x7, t0, t0x, t1, \ - t1x, t2, t2x, t3, t3x, t4, t5) \ - vpcmpeqd t0, t0, t0; \ - vpsrldq $8, t0, t0; /* ab: -1:0 ; cd: -1:0 */ \ - vpaddq t0, t0, t4; /* ab: -2:0 ; cd: -2:0 */\ - \ - /* load IV and byteswap */ \ - vmovdqu (iv), t2x; \ - vmovdqa t2x, t3x; \ - inc_le128(t2x, t0x, t1x); \ - vbroadcasti128 bswap, t1; \ - vinserti128 $1, t2x, t3, t2; /* ab: le0 ; cd: le1 */ \ - vpshufb t1, t2, x0; \ - \ - /* construct IVs */ \ - add2_le128(t2, t0, t4, t3, t5); /* ab: le2 ; cd: le3 */ \ - vpshufb t1, t2, x1; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x2; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x3; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x4; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x5; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x6; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x7; \ - vextracti128 $1, t2, t2x; \ - inc_le128(t2x, t0x, t3x); \ - vmovdqu t2x, (iv); - -#define store_ctr_16way(src, dst, x0, x1, x2, x3, x4, x5, x6, x7) \ - vpxor (0*32)(src), x0, x0; \ - vpxor (1*32)(src), x1, x1; \ - vpxor (2*32)(src), x2, x2; \ - vpxor (3*32)(src), x3, x3; \ - vpxor (4*32)(src), x4, x4; \ - vpxor (5*32)(src), x5, x5; \ - vpxor (6*32)(src), x6, x6; \ - vpxor (7*32)(src), x7, x7; \ - store_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7); - -#define gf128mul_x_ble(iv, mask, tmp) \ - vpsrad $31, iv, tmp; \ - vpaddq iv, iv, iv; \ - vpshufd $0x13, tmp, tmp; \ - vpand mask, tmp, tmp; \ - vpxor tmp, iv, iv; - -#define gf128mul_x2_ble(iv, mask1, mask2, tmp0, tmp1) \ - vpsrad $31, iv, tmp0; \ - vpaddq iv, iv, tmp1; \ - vpsllq $2, iv, iv; \ - vpshufd $0x13, tmp0, tmp0; \ - vpsrad $31, tmp1, tmp1; \ - vpand mask2, tmp0, tmp0; \ - vpshufd $0x13, tmp1, tmp1; \ - vpxor tmp0, iv, iv; \ - vpand mask1, tmp1, tmp1; \ - vpxor tmp1, iv, iv; - -#define load_xts_16way(iv, src, dst, x0, x1, x2, x3, x4, x5, x6, x7, tiv, \ - tivx, t0, t0x, t1, t1x, t2, t2x, t3, \ - xts_gf128mul_and_shl1_mask_0, \ - xts_gf128mul_and_shl1_mask_1) \ - vbroadcasti128 xts_gf128mul_and_shl1_mask_0, t1; \ - \ - /* load IV and construct second IV */ \ - vmovdqu (iv), tivx; \ - vmovdqa tivx, t0x; \ - gf128mul_x_ble(tivx, t1x, t2x); \ - vbroadcasti128 xts_gf128mul_and_shl1_mask_1, t2; \ - vinserti128 $1, tivx, t0, tiv; \ - vpxor (0*32)(src), tiv, x0; \ - vmovdqu tiv, (0*32)(dst); \ - \ - /* construct and store IVs, also xor with source */ \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (1*32)(src), tiv, x1; \ - vmovdqu tiv, (1*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (2*32)(src), tiv, x2; \ - vmovdqu tiv, (2*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (3*32)(src), tiv, x3; \ - vmovdqu tiv, (3*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (4*32)(src), tiv, x4; \ - vmovdqu tiv, (4*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (5*32)(src), tiv, x5; \ - vmovdqu tiv, (5*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (6*32)(src), tiv, x6; \ - vmovdqu tiv, (6*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (7*32)(src), tiv, x7; \ - vmovdqu tiv, (7*32)(dst); \ - \ - vextracti128 $1, tiv, tivx; \ - gf128mul_x_ble(tivx, t1x, t2x); \ - vmovdqu tivx, (iv); - -#define store_xts_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7) \ - vpxor (0*32)(dst), x0, x0; \ - vpxor (1*32)(dst), x1, x1; \ - vpxor (2*32)(dst), x2, x2; \ - vpxor (3*32)(dst), x3, x3; \ - vpxor (4*32)(dst), x4, x4; \ - vpxor (5*32)(dst), x5, x5; \ - vpxor (6*32)(dst), x6, x6; \ - vpxor (7*32)(dst), x7, x7; \ - store_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7); diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c deleted file mode 100644 index d3d91a0abf88..000000000000 --- a/arch/x86/crypto/glue_helper.c +++ /dev/null @@ -1,381 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Shared glue code for 128bit block ciphers - * - * Copyright © 2012-2013 Jussi Kivilinna - * - * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: - * Copyright (c) 2006 Herbert Xu - * CTR part based on code (crypto/ctr.c) by: - * (C) Copyright IBM Corp. 2007 - Joy Latten - */ - -#include -#include -#include -#include -#include -#include -#include - -int glue_ecb_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req) -{ - void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); - const unsigned int bsize = 128 / 8; - struct skcipher_walk walk; - bool fpu_enabled = false; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - unsigned int func_bytes; - unsigned int i; - - fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, - &walk, fpu_enabled, nbytes); - for (i = 0; i < gctx->num_funcs; i++) { - func_bytes = bsize * gctx->funcs[i].num_blocks; - - if (nbytes < func_bytes) - continue; - - /* Process multi-block batch */ - do { - gctx->funcs[i].fn_u.ecb(ctx, dst, src); - src += func_bytes; - dst += func_bytes; - nbytes -= func_bytes; - } while (nbytes >= func_bytes); - - if (nbytes < bsize) - break; - } - err = skcipher_walk_done(&walk, nbytes); - } - - glue_fpu_end(fpu_enabled); - return err; -} -EXPORT_SYMBOL_GPL(glue_ecb_req_128bit); - -int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn, - struct skcipher_request *req) -{ - void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); - const unsigned int bsize = 128 / 8; - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - const u128 *src = (u128 *)walk.src.virt.addr; - u128 *dst = (u128 *)walk.dst.virt.addr; - u128 *iv = (u128 *)walk.iv; - - do { - u128_xor(dst, src, iv); - fn(ctx, (u8 *)dst, (u8 *)dst); - iv = dst; - src++; - dst++; - nbytes -= bsize; - } while (nbytes >= bsize); - - *(u128 *)walk.iv = *iv; - err = skcipher_walk_done(&walk, nbytes); - } - return err; -} -EXPORT_SYMBOL_GPL(glue_cbc_encrypt_req_128bit); - -int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req) -{ - void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); - const unsigned int bsize = 128 / 8; - struct skcipher_walk walk; - bool fpu_enabled = false; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - const u128 *src = walk.src.virt.addr; - u128 *dst = walk.dst.virt.addr; - unsigned int func_bytes, num_blocks; - unsigned int i; - u128 last_iv; - - fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, - &walk, fpu_enabled, nbytes); - /* Start of the last block. */ - src += nbytes / bsize - 1; - dst += nbytes / bsize - 1; - - last_iv = *src; - - for (i = 0; i < gctx->num_funcs; i++) { - num_blocks = gctx->funcs[i].num_blocks; - func_bytes = bsize * num_blocks; - - if (nbytes < func_bytes) - continue; - - /* Process multi-block batch */ - do { - src -= num_blocks - 1; - dst -= num_blocks - 1; - - gctx->funcs[i].fn_u.cbc(ctx, (u8 *)dst, - (const u8 *)src); - - nbytes -= func_bytes; - if (nbytes < bsize) - goto done; - - u128_xor(dst, dst, --src); - dst--; - } while (nbytes >= func_bytes); - } -done: - u128_xor(dst, dst, (u128 *)walk.iv); - *(u128 *)walk.iv = last_iv; - err = skcipher_walk_done(&walk, nbytes); - } - - glue_fpu_end(fpu_enabled); - return err; -} -EXPORT_SYMBOL_GPL(glue_cbc_decrypt_req_128bit); - -int glue_ctr_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req) -{ - void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); - const unsigned int bsize = 128 / 8; - struct skcipher_walk walk; - bool fpu_enabled = false; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) >= bsize) { - const u128 *src = walk.src.virt.addr; - u128 *dst = walk.dst.virt.addr; - unsigned int func_bytes, num_blocks; - unsigned int i; - le128 ctrblk; - - fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, - &walk, fpu_enabled, nbytes); - - be128_to_le128(&ctrblk, (be128 *)walk.iv); - - for (i = 0; i < gctx->num_funcs; i++) { - num_blocks = gctx->funcs[i].num_blocks; - func_bytes = bsize * num_blocks; - - if (nbytes < func_bytes) - continue; - - /* Process multi-block batch */ - do { - gctx->funcs[i].fn_u.ctr(ctx, (u8 *)dst, - (const u8 *)src, - &ctrblk); - src += num_blocks; - dst += num_blocks; - nbytes -= func_bytes; - } while (nbytes >= func_bytes); - - if (nbytes < bsize) - break; - } - - le128_to_be128((be128 *)walk.iv, &ctrblk); - err = skcipher_walk_done(&walk, nbytes); - } - - glue_fpu_end(fpu_enabled); - - if (nbytes) { - le128 ctrblk; - u128 tmp; - - be128_to_le128(&ctrblk, (be128 *)walk.iv); - memcpy(&tmp, walk.src.virt.addr, nbytes); - gctx->funcs[gctx->num_funcs - 1].fn_u.ctr(ctx, (u8 *)&tmp, - (const u8 *)&tmp, - &ctrblk); - memcpy(walk.dst.virt.addr, &tmp, nbytes); - le128_to_be128((be128 *)walk.iv, &ctrblk); - - err = skcipher_walk_done(&walk, 0); - } - - return err; -} -EXPORT_SYMBOL_GPL(glue_ctr_req_128bit); - -static unsigned int __glue_xts_req_128bit(const struct common_glue_ctx *gctx, - void *ctx, - struct skcipher_walk *walk) -{ - const unsigned int bsize = 128 / 8; - unsigned int nbytes = walk->nbytes; - u128 *src = walk->src.virt.addr; - u128 *dst = walk->dst.virt.addr; - unsigned int num_blocks, func_bytes; - unsigned int i; - - /* Process multi-block batch */ - for (i = 0; i < gctx->num_funcs; i++) { - num_blocks = gctx->funcs[i].num_blocks; - func_bytes = bsize * num_blocks; - - if (nbytes >= func_bytes) { - do { - gctx->funcs[i].fn_u.xts(ctx, (u8 *)dst, - (const u8 *)src, - walk->iv); - - src += num_blocks; - dst += num_blocks; - nbytes -= func_bytes; - } while (nbytes >= func_bytes); - - if (nbytes < bsize) - goto done; - } - } - -done: - return nbytes; -} - -int glue_xts_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req, - common_glue_func_t tweak_fn, void *tweak_ctx, - void *crypt_ctx, bool decrypt) -{ - const bool cts = (req->cryptlen % XTS_BLOCK_SIZE); - const unsigned int bsize = 128 / 8; - struct skcipher_request subreq; - struct skcipher_walk walk; - bool fpu_enabled = false; - unsigned int nbytes, tail; - int err; - - if (req->cryptlen < XTS_BLOCK_SIZE) - return -EINVAL; - - if (unlikely(cts)) { - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - - tail = req->cryptlen % XTS_BLOCK_SIZE + XTS_BLOCK_SIZE; - - skcipher_request_set_tfm(&subreq, tfm); - skcipher_request_set_callback(&subreq, - crypto_skcipher_get_flags(tfm), - NULL, NULL); - skcipher_request_set_crypt(&subreq, req->src, req->dst, - req->cryptlen - tail, req->iv); - req = &subreq; - } - - err = skcipher_walk_virt(&walk, req, false); - nbytes = walk.nbytes; - if (err) - return err; - - /* set minimum length to bsize, for tweak_fn */ - fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, - &walk, fpu_enabled, - nbytes < bsize ? bsize : nbytes); - - /* calculate first value of T */ - tweak_fn(tweak_ctx, walk.iv, walk.iv); - - while (nbytes) { - nbytes = __glue_xts_req_128bit(gctx, crypt_ctx, &walk); - - err = skcipher_walk_done(&walk, nbytes); - nbytes = walk.nbytes; - } - - if (unlikely(cts)) { - u8 *next_tweak, *final_tweak = req->iv; - struct scatterlist *src, *dst; - struct scatterlist s[2], d[2]; - le128 b[2]; - - dst = src = scatterwalk_ffwd(s, req->src, req->cryptlen); - if (req->dst != req->src) - dst = scatterwalk_ffwd(d, req->dst, req->cryptlen); - - if (decrypt) { - next_tweak = memcpy(b, req->iv, XTS_BLOCK_SIZE); - gf128mul_x_ble(b, b); - } else { - next_tweak = req->iv; - } - - skcipher_request_set_crypt(&subreq, src, dst, XTS_BLOCK_SIZE, - next_tweak); - - err = skcipher_walk_virt(&walk, req, false) ?: - skcipher_walk_done(&walk, - __glue_xts_req_128bit(gctx, crypt_ctx, &walk)); - if (err) - goto out; - - scatterwalk_map_and_copy(b, dst, 0, XTS_BLOCK_SIZE, 0); - memcpy(b + 1, b, tail - XTS_BLOCK_SIZE); - scatterwalk_map_and_copy(b, src, XTS_BLOCK_SIZE, - tail - XTS_BLOCK_SIZE, 0); - scatterwalk_map_and_copy(b, dst, 0, tail, 1); - - skcipher_request_set_crypt(&subreq, dst, dst, XTS_BLOCK_SIZE, - final_tweak); - - err = skcipher_walk_virt(&walk, req, false) ?: - skcipher_walk_done(&walk, - __glue_xts_req_128bit(gctx, crypt_ctx, &walk)); - } - -out: - glue_fpu_end(fpu_enabled); - - return err; -} -EXPORT_SYMBOL_GPL(glue_xts_req_128bit); - -void glue_xts_crypt_128bit_one(const void *ctx, u8 *dst, const u8 *src, - le128 *iv, common_glue_func_t fn) -{ - le128 ivblk = *iv; - - /* generate next IV */ - gf128mul_x_ble(iv, &ivblk); - - /* CC <- T xor C */ - u128_xor((u128 *)dst, (const u128 *)src, (u128 *)&ivblk); - - /* PP <- D(Key2,CC) */ - fn(ctx, dst, dst); - - /* P <- T xor PP */ - u128_xor((u128 *)dst, (u128 *)dst, (u128 *)&ivblk); -} -EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one); - -MODULE_LICENSE("GPL"); diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S index ba9e4c1e7f5c..b7ee24df7fba 100644 --- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S @@ -18,10 +18,6 @@ .align 16 .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 -.section .rodata.cst16.xts_gf128mul_and_shl1_mask, "aM", @progbits, 16 -.align 16 -.Lxts_gf128mul_and_shl1_mask: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 .text @@ -715,67 +711,3 @@ SYM_FUNC_START(serpent_cbc_dec_8way_avx) FRAME_END ret; SYM_FUNC_END(serpent_cbc_dec_8way_avx) - -SYM_FUNC_START(serpent_ctr_8way_avx) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - - load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RK0, RK1, RK2); - - call __serpent_enc_blk8_avx; - - store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - FRAME_END - ret; -SYM_FUNC_END(serpent_ctr_8way_avx) - -SYM_FUNC_START(serpent_xts_enc_8way_avx) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, - RK0, RK1, RK2, .Lxts_gf128mul_and_shl1_mask); - - call __serpent_enc_blk8_avx; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - FRAME_END - ret; -SYM_FUNC_END(serpent_xts_enc_8way_avx) - -SYM_FUNC_START(serpent_xts_dec_8way_avx) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, - RK0, RK1, RK2, .Lxts_gf128mul_and_shl1_mask); - - call __serpent_dec_blk8_avx; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); - - FRAME_END - ret; -SYM_FUNC_END(serpent_xts_dec_8way_avx) diff --git a/arch/x86/crypto/serpent-avx.h b/arch/x86/crypto/serpent-avx.h new file mode 100644 index 000000000000..23f3361a0e72 --- /dev/null +++ b/arch/x86/crypto/serpent-avx.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ASM_X86_SERPENT_AVX_H +#define ASM_X86_SERPENT_AVX_H + +#include +#include +#include + +struct crypto_skcipher; + +#define SERPENT_PARALLEL_BLOCKS 8 + +asmlinkage void serpent_ecb_enc_8way_avx(const void *ctx, u8 *dst, + const u8 *src); +asmlinkage void serpent_ecb_dec_8way_avx(const void *ctx, u8 *dst, + const u8 *src); + +asmlinkage void serpent_cbc_dec_8way_avx(const void *ctx, u8 *dst, + const u8 *src); + +#endif diff --git a/arch/x86/crypto/serpent-avx2-asm_64.S b/arch/x86/crypto/serpent-avx2-asm_64.S index c9648aeae705..9161b6e441f3 100644 --- a/arch/x86/crypto/serpent-avx2-asm_64.S +++ b/arch/x86/crypto/serpent-avx2-asm_64.S @@ -20,16 +20,6 @@ .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 -.section .rodata.cst16.xts_gf128mul_and_shl1_mask_0, "aM", @progbits, 16 -.align 16 -.Lxts_gf128mul_and_shl1_mask_0: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 - -.section .rodata.cst16.xts_gf128mul_and_shl1_mask_1, "aM", @progbits, 16 -.align 16 -.Lxts_gf128mul_and_shl1_mask_1: - .byte 0x0e, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0 - .text #define CTX %rdi @@ -734,80 +724,3 @@ SYM_FUNC_START(serpent_cbc_dec_16way) FRAME_END ret; SYM_FUNC_END(serpent_cbc_dec_16way) - -SYM_FUNC_START(serpent_ctr_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - - vzeroupper; - - load_ctr_16way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RK0, RK0x, RK1, RK1x, RK2, RK2x, RK3, RK3x, RNOT, - tp); - - call __serpent_enc_blk16; - - store_ctr_16way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - vzeroupper; - - FRAME_END - ret; -SYM_FUNC_END(serpent_ctr_16way) - -SYM_FUNC_START(serpent_xts_enc_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - vzeroupper; - - load_xts_16way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RK0, RK0x, RK1, RK1x, RK2, RK2x, RK3, RK3x, RNOT, - .Lxts_gf128mul_and_shl1_mask_0, - .Lxts_gf128mul_and_shl1_mask_1); - - call __serpent_enc_blk16; - - store_xts_16way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - vzeroupper; - - FRAME_END - ret; -SYM_FUNC_END(serpent_xts_enc_16way) - -SYM_FUNC_START(serpent_xts_dec_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - vzeroupper; - - load_xts_16way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RK0, RK0x, RK1, RK1x, RK2, RK2x, RK3, RK3x, RNOT, - .Lxts_gf128mul_and_shl1_mask_0, - .Lxts_gf128mul_and_shl1_mask_1); - - call __serpent_dec_blk16; - - store_xts_16way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); - - vzeroupper; - - FRAME_END - ret; -SYM_FUNC_END(serpent_xts_dec_16way) diff --git a/arch/x86/include/asm/crypto/serpent-sse2.h b/arch/x86/crypto/serpent-sse2.h similarity index 100% rename from arch/x86/include/asm/crypto/serpent-sse2.h rename to arch/x86/crypto/serpent-sse2.h diff --git a/arch/x86/crypto/serpent_avx2_glue.c b/arch/x86/crypto/serpent_avx2_glue.c index f973ace44ad3..ccf0b5fa4933 100644 --- a/arch/x86/crypto/serpent_avx2_glue.c +++ b/arch/x86/crypto/serpent_avx2_glue.c @@ -12,9 +12,9 @@ #include #include #include -#include -#include -#include + +#include "serpent-avx.h" +#include "ecb_cbc_helpers.h" #define SERPENT_AVX2_PARALLEL_BLOCKS 16 @@ -23,158 +23,44 @@ asmlinkage void serpent_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void serpent_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void serpent_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src); -asmlinkage void serpent_ctr_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void serpent_xts_enc_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void serpent_xts_dec_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); } -static const struct common_glue_ctx serpent_enc = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .ecb = serpent_ecb_enc_16way } - }, { - .num_blocks = 8, - .fn_u = { .ecb = serpent_ecb_enc_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_encrypt } - } } -}; - -static const struct common_glue_ctx serpent_ctr = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .ctr = serpent_ctr_16way } - }, { - .num_blocks = 8, - .fn_u = { .ctr = serpent_ctr_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ctr = __serpent_crypt_ctr } - } } -}; - -static const struct common_glue_ctx serpent_enc_xts = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .xts = serpent_xts_enc_16way } - }, { - .num_blocks = 8, - .fn_u = { .xts = serpent_xts_enc_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .xts = serpent_xts_enc } - } } -}; - -static const struct common_glue_ctx serpent_dec = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .ecb = serpent_ecb_dec_16way } - }, { - .num_blocks = 8, - .fn_u = { .ecb = serpent_ecb_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_decrypt } - } } -}; - -static const struct common_glue_ctx serpent_dec_cbc = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .cbc = serpent_cbc_dec_16way } - }, { - .num_blocks = 8, - .fn_u = { .cbc = serpent_cbc_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .cbc = __serpent_decrypt } - } } -}; - -static const struct common_glue_ctx serpent_dec_xts = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .xts = serpent_xts_dec_16way } - }, { - .num_blocks = 8, - .fn_u = { .xts = serpent_xts_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .xts = serpent_xts_dec } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_enc, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_enc_16way); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx); + ECB_BLOCK(1, __serpent_encrypt); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_dec, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_dec_16way); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx); + ECB_BLOCK(1, __serpent_decrypt); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(__serpent_encrypt, req); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(__serpent_encrypt); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&serpent_ctr, req); -} - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&serpent_enc_xts, req, - __serpent_encrypt, &ctx->tweak_ctx, - &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&serpent_dec_xts, req, - __serpent_encrypt, &ctx->tweak_ctx, - &ctx->crypt_ctx, true); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_cbc_dec_16way); + CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx); + CBC_DEC_BLOCK(1, __serpent_decrypt); + CBC_WALK_END(); } static struct skcipher_alg serpent_algs[] = { @@ -205,35 +91,6 @@ static struct skcipher_alg serpent_algs[] = { .setkey = serpent_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(serpent)", - .base.cra_driver_name = "__ctr-serpent-avx2", - .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct serpent_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = SERPENT_MIN_KEY_SIZE, - .max_keysize = SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .chunksize = SERPENT_BLOCK_SIZE, - .setkey = serpent_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(serpent)", - .base.cra_driver_name = "__xts-serpent-avx2", - .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = SERPENT_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct serpent_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * SERPENT_MIN_KEY_SIZE, - .max_keysize = 2 * SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .setkey = xts_serpent_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, }, }; diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c index 7806d1cbe854..6c248e1ea4ef 100644 --- a/arch/x86/crypto/serpent_avx_glue.c +++ b/arch/x86/crypto/serpent_avx_glue.c @@ -15,9 +15,9 @@ #include #include #include -#include -#include -#include + +#include "serpent-avx.h" +#include "ecb_cbc_helpers.h" /* 8-way parallel cipher functions */ asmlinkage void serpent_ecb_enc_8way_avx(const void *ctx, u8 *dst, @@ -32,191 +32,41 @@ asmlinkage void serpent_cbc_dec_8way_avx(const void *ctx, u8 *dst, const u8 *src); EXPORT_SYMBOL_GPL(serpent_cbc_dec_8way_avx); -asmlinkage void serpent_ctr_8way_avx(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -EXPORT_SYMBOL_GPL(serpent_ctr_8way_avx); - -asmlinkage void serpent_xts_enc_8way_avx(const void *ctx, u8 *dst, - const u8 *src, le128 *iv); -EXPORT_SYMBOL_GPL(serpent_xts_enc_8way_avx); - -asmlinkage void serpent_xts_dec_8way_avx(const void *ctx, u8 *dst, - const u8 *src, le128 *iv); -EXPORT_SYMBOL_GPL(serpent_xts_dec_8way_avx); - -void __serpent_crypt_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblk; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - le128_to_be128(&ctrblk, iv); - le128_inc(iv); - - __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); - u128_xor(dst, src, (u128 *)&ctrblk); -} -EXPORT_SYMBOL_GPL(__serpent_crypt_ctr); - -void serpent_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, __serpent_encrypt); -} -EXPORT_SYMBOL_GPL(serpent_xts_enc); - -void serpent_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, __serpent_decrypt); -} -EXPORT_SYMBOL_GPL(serpent_xts_dec); - static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); } -int xts_serpent_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = xts_verify_key(tfm, key, keylen); - if (err) - return err; - - /* first half of xts-key is for crypt */ - err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2); - if (err) - return err; - - /* second half of xts-key is for tweak */ - return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); -} -EXPORT_SYMBOL_GPL(xts_serpent_setkey); - -static const struct common_glue_ctx serpent_enc = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ecb = serpent_ecb_enc_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_encrypt } - } } -}; - -static const struct common_glue_ctx serpent_ctr = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ctr = serpent_ctr_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ctr = __serpent_crypt_ctr } - } } -}; - -static const struct common_glue_ctx serpent_enc_xts = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .xts = serpent_xts_enc_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .xts = serpent_xts_enc } - } } -}; - -static const struct common_glue_ctx serpent_dec = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ecb = serpent_ecb_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_decrypt } - } } -}; - -static const struct common_glue_ctx serpent_dec_cbc = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .cbc = serpent_cbc_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .cbc = __serpent_decrypt } - } } -}; - -static const struct common_glue_ctx serpent_dec_xts = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .xts = serpent_xts_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .xts = serpent_xts_dec } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_enc, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx); + ECB_BLOCK(1, __serpent_encrypt); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_dec, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx); + ECB_BLOCK(1, __serpent_decrypt); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(__serpent_encrypt, req); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(__serpent_encrypt); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&serpent_ctr, req); -} - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&serpent_enc_xts, req, - __serpent_encrypt, &ctx->tweak_ctx, - &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&serpent_dec_xts, req, - __serpent_encrypt, &ctx->tweak_ctx, - &ctx->crypt_ctx, true); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx); + CBC_DEC_BLOCK(1, __serpent_decrypt); + CBC_WALK_END(); } static struct skcipher_alg serpent_algs[] = { @@ -247,35 +97,6 @@ static struct skcipher_alg serpent_algs[] = { .setkey = serpent_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(serpent)", - .base.cra_driver_name = "__ctr-serpent-avx", - .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct serpent_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = SERPENT_MIN_KEY_SIZE, - .max_keysize = SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .chunksize = SERPENT_BLOCK_SIZE, - .setkey = serpent_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(serpent)", - .base.cra_driver_name = "__xts-serpent-avx", - .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = SERPENT_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct serpent_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * SERPENT_MIN_KEY_SIZE, - .max_keysize = 2 * SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .setkey = xts_serpent_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, }, }; diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index 4fed8d26b91a..d78f37e9b2cf 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c @@ -10,8 +10,6 @@ * * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: * Copyright (c) 2006 Herbert Xu - * CTR part based on code (crypto/ctr.c) by: - * (C) Copyright IBM Corp. 2007 - Joy Latten */ #include @@ -22,8 +20,9 @@ #include #include #include -#include -#include + +#include "serpent-sse2.h" +#include "ecb_cbc_helpers.h" static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -31,130 +30,46 @@ static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); } -static void serpent_decrypt_cbc_xway(const void *ctx, u8 *d, const u8 *s) +static void serpent_decrypt_cbc_xway(const void *ctx, u8 *dst, const u8 *src) { - u128 ivs[SERPENT_PARALLEL_BLOCKS - 1]; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - unsigned int j; + u8 buf[SERPENT_PARALLEL_BLOCKS - 1][SERPENT_BLOCK_SIZE]; + const u8 *s = src; - for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) - ivs[j] = src[j]; - - serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); - - for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) - u128_xor(dst + (j + 1), dst + (j + 1), ivs + j); + if (dst == src) + s = memcpy(buf, src, sizeof(buf)); + serpent_dec_blk_xway(ctx, dst, src); + crypto_xor(dst + SERPENT_BLOCK_SIZE, s, sizeof(buf)); } -static void serpent_crypt_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblk; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - le128_to_be128(&ctrblk, iv); - le128_inc(iv); - - __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); - u128_xor(dst, src, (u128 *)&ctrblk); -} - -static void serpent_crypt_ctr_xway(const void *ctx, u8 *d, const u8 *s, - le128 *iv) -{ - be128 ctrblks[SERPENT_PARALLEL_BLOCKS]; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - unsigned int i; - - for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) { - if (dst != src) - dst[i] = src[i]; - - le128_to_be128(&ctrblks[i], iv); - le128_inc(iv); - } - - serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks); -} - -static const struct common_glue_ctx serpent_enc = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ecb = serpent_enc_blk_xway } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_encrypt } - } } -}; - -static const struct common_glue_ctx serpent_ctr = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ctr = serpent_crypt_ctr_xway } - }, { - .num_blocks = 1, - .fn_u = { .ctr = serpent_crypt_ctr } - } } -}; - -static const struct common_glue_ctx serpent_dec = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ecb = serpent_dec_blk_xway } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_decrypt } - } } -}; - -static const struct common_glue_ctx serpent_dec_cbc = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .cbc = serpent_decrypt_cbc_xway } - }, { - .num_blocks = 1, - .fn_u = { .cbc = __serpent_decrypt } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_enc, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_enc_blk_xway); + ECB_BLOCK(1, __serpent_encrypt); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_dec, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_dec_blk_xway); + ECB_BLOCK(1, __serpent_decrypt); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(__serpent_encrypt, - req); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(__serpent_encrypt); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&serpent_ctr, req); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_decrypt_cbc_xway); + CBC_DEC_BLOCK(1, __serpent_decrypt); + CBC_WALK_END(); } static struct skcipher_alg serpent_algs[] = { @@ -185,21 +100,6 @@ static struct skcipher_alg serpent_algs[] = { .setkey = serpent_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(serpent)", - .base.cra_driver_name = "__ctr-serpent-sse2", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct serpent_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = SERPENT_MIN_KEY_SIZE, - .max_keysize = SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .chunksize = SERPENT_BLOCK_SIZE, - .setkey = serpent_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, }, }; diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S index a5151393bb2f..37e63b3c664e 100644 --- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S @@ -19,11 +19,6 @@ .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 -.section .rodata.cst16.xts_gf128mul_and_shl1_mask, "aM", @progbits, 16 -.align 16 -.Lxts_gf128mul_and_shl1_mask: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 - .text /* structure of crypto context */ @@ -379,78 +374,3 @@ SYM_FUNC_START(twofish_cbc_dec_8way) FRAME_END ret; SYM_FUNC_END(twofish_cbc_dec_8way) - -SYM_FUNC_START(twofish_ctr_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - - pushq %r12; - - movq %rsi, %r11; - movq %rdx, %r12; - - load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RX0, RX1, RY0); - - call __twofish_enc_blk8; - - store_ctr_8way(%r12, %r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); - - popq %r12; - - FRAME_END - ret; -SYM_FUNC_END(twofish_ctr_8way) - -SYM_FUNC_START(twofish_xts_enc_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - movq %rsi, %r11; - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, - RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask); - - call __twofish_enc_blk8; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); - - FRAME_END - ret; -SYM_FUNC_END(twofish_xts_enc_8way) - -SYM_FUNC_START(twofish_xts_dec_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - movq %rsi, %r11; - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2, - RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask); - - call __twofish_dec_blk8; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - FRAME_END - ret; -SYM_FUNC_END(twofish_xts_dec_8way) diff --git a/arch/x86/include/asm/crypto/twofish.h b/arch/x86/crypto/twofish.h similarity index 80% rename from arch/x86/include/asm/crypto/twofish.h rename to arch/x86/crypto/twofish.h index 2c377a8042e1..12df400e6d53 100644 --- a/arch/x86/include/asm/crypto/twofish.h +++ b/arch/x86/crypto/twofish.h @@ -17,9 +17,5 @@ asmlinkage void twofish_dec_blk_3way(const void *ctx, u8 *dst, const u8 *src); /* helpers from twofish_x86_64-3way module */ extern void twofish_dec_blk_cbc_3way(const void *ctx, u8 *dst, const u8 *src); -extern void twofish_enc_blk_ctr(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -extern void twofish_enc_blk_ctr_3way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); #endif /* ASM_X86_TWOFISH_H */ diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c index 2dbc8ce3730e..3eb3440b477a 100644 --- a/arch/x86/crypto/twofish_avx_glue.c +++ b/arch/x86/crypto/twofish_avx_glue.c @@ -15,9 +15,9 @@ #include #include #include -#include -#include -#include + +#include "twofish.h" +#include "ecb_cbc_helpers.h" #define TWOFISH_PARALLEL_BLOCKS 8 @@ -26,13 +26,6 @@ asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src); -asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); static int twofish_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -45,171 +38,38 @@ static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src) __twofish_enc_blk_3way(ctx, dst, src, false); } -static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk); -} - -static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk); -} - -struct twofish_xts_ctx { - struct twofish_ctx tweak_ctx; - struct twofish_ctx crypt_ctx; -}; - -static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = xts_verify_key(tfm, key, keylen); - if (err) - return err; - - /* first half of xts-key is for crypt */ - err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2); - if (err) - return err; - - /* second half of xts-key is for tweak */ - return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); -} - -static const struct common_glue_ctx twofish_enc = { - .num_funcs = 3, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .ecb = twofish_ecb_enc_8way } - }, { - .num_blocks = 3, - .fn_u = { .ecb = twofish_enc_blk_3way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = twofish_enc_blk } - } } -}; - -static const struct common_glue_ctx twofish_ctr = { - .num_funcs = 3, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .ctr = twofish_ctr_8way } - }, { - .num_blocks = 3, - .fn_u = { .ctr = twofish_enc_blk_ctr_3way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = twofish_enc_blk_ctr } - } } -}; - -static const struct common_glue_ctx twofish_enc_xts = { - .num_funcs = 2, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .xts = twofish_xts_enc_8way } - }, { - .num_blocks = 1, - .fn_u = { .xts = twofish_xts_enc } - } } -}; - -static const struct common_glue_ctx twofish_dec = { - .num_funcs = 3, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .ecb = twofish_ecb_dec_8way } - }, { - .num_blocks = 3, - .fn_u = { .ecb = twofish_dec_blk_3way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = twofish_dec_blk } - } } -}; - -static const struct common_glue_ctx twofish_dec_cbc = { - .num_funcs = 3, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .cbc = twofish_cbc_dec_8way } - }, { - .num_blocks = 3, - .fn_u = { .cbc = twofish_dec_blk_cbc_3way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = twofish_dec_blk } - } } -}; - -static const struct common_glue_ctx twofish_dec_xts = { - .num_funcs = 2, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .xts = twofish_xts_dec_8way } - }, { - .num_blocks = 1, - .fn_u = { .xts = twofish_xts_dec } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&twofish_enc, req); + ECB_WALK_START(req, TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS); + ECB_BLOCK(TWOFISH_PARALLEL_BLOCKS, twofish_ecb_enc_8way); + ECB_BLOCK(3, twofish_enc_blk_3way); + ECB_BLOCK(1, twofish_enc_blk); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&twofish_dec, req); + ECB_WALK_START(req, TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS); + ECB_BLOCK(TWOFISH_PARALLEL_BLOCKS, twofish_ecb_dec_8way); + ECB_BLOCK(3, twofish_dec_blk_3way); + ECB_BLOCK(1, twofish_dec_blk); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req); + CBC_WALK_START(req, TF_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(twofish_enc_blk); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&twofish_ctr, req); -} - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, true); + CBC_WALK_START(req, TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(TWOFISH_PARALLEL_BLOCKS, twofish_cbc_dec_8way); + CBC_DEC_BLOCK(3, twofish_dec_blk_cbc_3way); + CBC_DEC_BLOCK(1, twofish_dec_blk); + CBC_WALK_END(); } static struct skcipher_alg twofish_algs[] = { @@ -240,35 +100,6 @@ static struct skcipher_alg twofish_algs[] = { .setkey = twofish_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(twofish)", - .base.cra_driver_name = "__ctr-twofish-avx", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct twofish_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = TF_MIN_KEY_SIZE, - .max_keysize = TF_MAX_KEY_SIZE, - .ivsize = TF_BLOCK_SIZE, - .chunksize = TF_BLOCK_SIZE, - .setkey = twofish_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(twofish)", - .base.cra_driver_name = "__xts-twofish-avx", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = TF_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct twofish_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * TF_MIN_KEY_SIZE, - .max_keysize = 2 * TF_MAX_KEY_SIZE, - .ivsize = TF_BLOCK_SIZE, - .setkey = xts_twofish_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, }, }; diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c index 768af6075479..03725696397c 100644 --- a/arch/x86/crypto/twofish_glue_3way.c +++ b/arch/x86/crypto/twofish_glue_3way.c @@ -5,17 +5,16 @@ * Copyright (c) 2011 Jussi Kivilinna */ -#include -#include #include -#include -#include #include #include #include #include #include +#include "twofish.h" +#include "ecb_cbc_helpers.h" + EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way); EXPORT_SYMBOL_GPL(twofish_dec_blk_3way); @@ -30,143 +29,48 @@ static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src) __twofish_enc_blk_3way(ctx, dst, src, false); } -static inline void twofish_enc_blk_xor_3way(const void *ctx, u8 *dst, - const u8 *src) +void twofish_dec_blk_cbc_3way(const void *ctx, u8 *dst, const u8 *src) { - __twofish_enc_blk_3way(ctx, dst, src, true); -} + u8 buf[2][TF_BLOCK_SIZE]; + const u8 *s = src; -void twofish_dec_blk_cbc_3way(const void *ctx, u8 *d, const u8 *s) -{ - u128 ivs[2]; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; + if (dst == src) + s = memcpy(buf, src, sizeof(buf)); + twofish_dec_blk_3way(ctx, dst, src); + crypto_xor(dst + TF_BLOCK_SIZE, s, sizeof(buf)); - ivs[0] = src[0]; - ivs[1] = src[1]; - - twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); - - u128_xor(&dst[1], &dst[1], &ivs[0]); - u128_xor(&dst[2], &dst[2], &ivs[1]); } EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way); -void twofish_enc_blk_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblk; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - if (dst != src) - *dst = *src; - - le128_to_be128(&ctrblk, iv); - le128_inc(iv); - - twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); - u128_xor(dst, dst, (u128 *)&ctrblk); -} -EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr); - -void twofish_enc_blk_ctr_3way(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblks[3]; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - if (dst != src) { - dst[0] = src[0]; - dst[1] = src[1]; - dst[2] = src[2]; - } - - le128_to_be128(&ctrblks[0], iv); - le128_inc(iv); - le128_to_be128(&ctrblks[1], iv); - le128_inc(iv); - le128_to_be128(&ctrblks[2], iv); - le128_inc(iv); - - twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks); -} -EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way); - -static const struct common_glue_ctx twofish_enc = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 3, - .fn_u = { .ecb = twofish_enc_blk_3way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = twofish_enc_blk } - } } -}; - -static const struct common_glue_ctx twofish_ctr = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 3, - .fn_u = { .ctr = twofish_enc_blk_ctr_3way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = twofish_enc_blk_ctr } - } } -}; - -static const struct common_glue_ctx twofish_dec = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 3, - .fn_u = { .ecb = twofish_dec_blk_3way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = twofish_dec_blk } - } } -}; - -static const struct common_glue_ctx twofish_dec_cbc = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 3, - .fn_u = { .cbc = twofish_dec_blk_cbc_3way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = twofish_dec_blk } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&twofish_enc, req); + ECB_WALK_START(req, TF_BLOCK_SIZE, -1); + ECB_BLOCK(3, twofish_enc_blk_3way); + ECB_BLOCK(1, twofish_enc_blk); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&twofish_dec, req); + ECB_WALK_START(req, TF_BLOCK_SIZE, -1); + ECB_BLOCK(3, twofish_dec_blk_3way); + ECB_BLOCK(1, twofish_dec_blk); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req); + CBC_WALK_START(req, TF_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(twofish_enc_blk); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&twofish_ctr, req); + CBC_WALK_START(req, TF_BLOCK_SIZE, -1); + CBC_DEC_BLOCK(3, twofish_dec_blk_cbc_3way); + CBC_DEC_BLOCK(1, twofish_dec_blk); + CBC_WALK_END(); } static struct skcipher_alg tf_skciphers[] = { @@ -195,20 +99,6 @@ static struct skcipher_alg tf_skciphers[] = { .setkey = twofish_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "ctr(twofish)", - .base.cra_driver_name = "ctr-twofish-3way", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct twofish_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = TF_MIN_KEY_SIZE, - .max_keysize = TF_MAX_KEY_SIZE, - .ivsize = TF_BLOCK_SIZE, - .chunksize = TF_BLOCK_SIZE, - .setkey = twofish_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, }, }; diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index 0904f5676e4d..a2433ae8a65e 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -249,30 +249,23 @@ static __always_inline bool get_and_clear_inhcall(void) { return false; } static __always_inline void restore_inhcall(bool inhcall) { } #endif -static void __xen_pv_evtchn_do_upcall(void) +static void __xen_pv_evtchn_do_upcall(struct pt_regs *regs) { - irq_enter_rcu(); + struct pt_regs *old_regs = set_irq_regs(regs); + inc_irq_stat(irq_hv_callback_count); xen_hvm_evtchn_do_upcall(); - irq_exit_rcu(); + set_irq_regs(old_regs); } __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs) { - struct pt_regs *old_regs; + irqentry_state_t state = irqentry_enter(regs); bool inhcall; - irqentry_state_t state; - state = irqentry_enter(regs); - old_regs = set_irq_regs(regs); - - instrumentation_begin(); - run_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs); - instrumentation_begin(); - - set_irq_regs(old_regs); + run_sysvec_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs); inhcall = get_and_clear_inhcall(); if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) { diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index ce0464d630a2..400908dff42e 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -754,47 +754,6 @@ SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) SYM_CODE_END(.Lbad_gs) .previous -/* - * rdi: New stack pointer points to the top word of the stack - * rsi: Function pointer - * rdx: Function argument (can be NULL if none) - */ -SYM_FUNC_START(asm_call_on_stack) -SYM_INNER_LABEL(asm_call_sysvec_on_stack, SYM_L_GLOBAL) -SYM_INNER_LABEL(asm_call_irq_on_stack, SYM_L_GLOBAL) - /* - * Save the frame pointer unconditionally. This allows the ORC - * unwinder to handle the stack switch. - */ - pushq %rbp - mov %rsp, %rbp - - /* - * The unwinder relies on the word at the top of the new stack - * page linking back to the previous RSP. - */ - mov %rsp, (%rdi) - mov %rdi, %rsp - /* Move the argument to the right place */ - mov %rdx, %rdi - -1: - .pushsection .discard.instr_begin - .long 1b - . - .popsection - - CALL_NOSPEC rsi - -2: - .pushsection .discard.instr_end - .long 2b - . - .popsection - - /* Restore the previous stack pointer from RBP. */ - leaveq - ret -SYM_FUNC_END(asm_call_on_stack) - #ifdef CONFIG_XEN_PV /* * A note on the "critical region" in our callback handler. diff --git a/arch/x86/entry/syscalls/Makefile b/arch/x86/entry/syscalls/Makefile index 6fb9b57ed5ba..d8c4f6c9eadc 100644 --- a/arch/x86/entry/syscalls/Makefile +++ b/arch/x86/entry/syscalls/Makefile @@ -6,8 +6,8 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(out)' ] || mkdir -p '$(out)') \ $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') -syscall32 := $(srctree)/$(src)/syscall_32.tbl -syscall64 := $(srctree)/$(src)/syscall_64.tbl +syscall32 := $(src)/syscall_32.tbl +syscall64 := $(src)/syscall_64.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,37 +21,37 @@ quiet_cmd_systbl = SYSTBL $@ cmd_systbl = $(CONFIG_SHELL) '$(systbl)' $< $@ quiet_cmd_hypercalls = HYPERCALLS $@ - cmd_hypercalls = $(CONFIG_SHELL) '$<' $@ $(filter-out $<,$^) + cmd_hypercalls = $(CONFIG_SHELL) '$<' $@ $(filter-out $<, $(real-prereqs)) syshdr_abi_unistd_32 := i386 -$(uapi)/unistd_32.h: $(syscall32) $(syshdr) +$(uapi)/unistd_32.h: $(syscall32) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abi_unistd_32_ia32 := i386 syshdr_pfx_unistd_32_ia32 := ia32_ -$(out)/unistd_32_ia32.h: $(syscall32) $(syshdr) +$(out)/unistd_32_ia32.h: $(syscall32) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abi_unistd_x32 := common,x32 syshdr_offset_unistd_x32 := __X32_SYSCALL_BIT -$(uapi)/unistd_x32.h: $(syscall64) $(syshdr) +$(uapi)/unistd_x32.h: $(syscall64) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abi_unistd_64 := common,64 -$(uapi)/unistd_64.h: $(syscall64) $(syshdr) +$(uapi)/unistd_64.h: $(syscall64) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abi_unistd_64_x32 := x32 syshdr_pfx_unistd_64_x32 := x32_ -$(out)/unistd_64_x32.h: $(syscall64) $(syshdr) +$(out)/unistd_64_x32.h: $(syscall64) $(syshdr) FORCE $(call if_changed,syshdr) -$(out)/syscalls_32.h: $(syscall32) $(systbl) +$(out)/syscalls_32.h: $(syscall32) $(systbl) FORCE $(call if_changed,systbl) -$(out)/syscalls_64.h: $(syscall64) $(systbl) +$(out)/syscalls_64.h: $(syscall64) $(systbl) FORCE $(call if_changed,systbl) -$(out)/xen-hypercalls.h: $(srctree)/scripts/xen-hypercalls.sh +$(out)/xen-hypercalls.h: $(srctree)/scripts/xen-hypercalls.sh FORCE $(call if_changed,hypercalls) $(out)/xen-hypercalls.h: $(srctree)/include/xen/interface/xen*.h @@ -62,9 +62,10 @@ syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h syshdr-$(CONFIG_X86_64) += syscalls_64.h syshdr-$(CONFIG_XEN) += xen-hypercalls.h -targets += $(uapisyshdr-y) $(syshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +syshdr-y := $(addprefix $(out)/, $(syshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(syshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(out)/,$(syshdr-y)) +all: $(uapisyshdr-y) $(syshdr-y) @: diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 874aeacde2dd..a1c9f496fca6 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -446,3 +446,4 @@ 439 i386 faccessat2 sys_faccessat2 440 i386 process_madvise sys_process_madvise 441 i386 epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 i386 mount_setattr sys_mount_setattr diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 78672124d28b..7bf01cbe582f 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -363,6 +363,7 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr # # Due to a historical design error, certain syscalls are numbered differently diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 02e3e42f380b..05c4abc2fdfd 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -91,7 +91,7 @@ ifneq ($(RETPOLINE_VDSO_CFLAGS),) endif endif -$(vobjs): KBUILD_CFLAGS := $(filter-out $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL) +$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL) # # vDSO code runs in userspace and -pg doesn't help with profiling anyway. @@ -150,6 +150,7 @@ KBUILD_CFLAGS_32 := $(filter-out -fno-pic,$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out -mfentry,$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=0 -fpic KBUILD_CFLAGS_32 += -fno-stack-protector KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls) diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h new file mode 100644 index 000000000000..e003a01b7c67 --- /dev/null +++ b/arch/x86/include/asm/acrn.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_ACRN_H +#define _ASM_X86_ACRN_H + +/* + * This CPUID returns feature bitmaps in EAX. + * Guest VM uses this to detect the appropriate feature bit. + */ +#define ACRN_CPUID_FEATURES 0x40000001 +/* Bit 0 indicates whether guest VM is privileged */ +#define ACRN_FEATURE_PRIVILEGED_VM BIT(0) + +void acrn_setup_intr_handler(void (*handler)(void)); +void acrn_remove_intr_handler(void); + +static inline u32 acrn_cpuid_base(void) +{ + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) + return hypervisor_cpuid_base("ACRNACRNACRN", 0); + + return 0; +} + +/* + * Hypercalls for ACRN + * + * - VMCALL instruction is used to implement ACRN hypercalls. + * - ACRN hypercall ABI: + * - Hypercall number is passed in R8 register. + * - Up to 2 arguments are passed in RDI, RSI. + * - Return value will be placed in RAX. + * + * Because GCC doesn't support R8 register as direct register constraints, use + * supported constraint as input with a explicit MOV to R8 in beginning of asm. + */ +static inline long acrn_hypercall0(unsigned long hcall_id) +{ + long result; + + asm volatile("movl %1, %%r8d\n\t" + "vmcall\n\t" + : "=a" (result) + : "g" (hcall_id) + : "r8", "memory"); + + return result; +} + +static inline long acrn_hypercall1(unsigned long hcall_id, + unsigned long param1) +{ + long result; + + asm volatile("movl %1, %%r8d\n\t" + "vmcall\n\t" + : "=a" (result) + : "g" (hcall_id), "D" (param1) + : "r8", "memory"); + + return result; +} + +static inline long acrn_hypercall2(unsigned long hcall_id, + unsigned long param1, + unsigned long param2) +{ + long result; + + asm volatile("movl %1, %%r8d\n\t" + "vmcall\n\t" + : "=a" (result) + : "g" (hcall_id), "D" (param1), "S" (param2) + : "r8", "memory"); + + return result; +} + +#endif /* _ASM_X86_ACRN_H */ diff --git a/arch/x86/include/asm/apb_timer.h b/arch/x86/include/asm/apb_timer.h deleted file mode 100644 index 87ce8e963215..000000000000 --- a/arch/x86/include/asm/apb_timer.h +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * apb_timer.h: Driver for Langwell APB timer based on Synopsis DesignWare - * - * (C) Copyright 2009 Intel Corporation - * Author: Jacob Pan (jacob.jun.pan@intel.com) - * - * Note: - */ - -#ifndef ASM_X86_APBT_H -#define ASM_X86_APBT_H -#include - -#ifdef CONFIG_APB_TIMER - -/* default memory mapped register base */ -#define LNW_SCU_ADDR 0xFF100000 -#define LNW_EXT_TIMER_OFFSET 0x1B800 -#define APBT_DEFAULT_BASE (LNW_SCU_ADDR+LNW_EXT_TIMER_OFFSET) -#define LNW_EXT_TIMER_PGOFFSET 0x800 - -/* APBT clock speed range from PCLK to fabric base, 25-100MHz */ -#define APBT_MAX_FREQ 50000000 -#define APBT_MIN_FREQ 1000000 -#define APBT_MMAP_SIZE 1024 - -extern void apbt_time_init(void); -extern void apbt_setup_secondary_clock(void); - -extern struct sfi_timer_table_entry *sfi_get_mtmr(int hint); -extern void sfi_free_mtmr(struct sfi_timer_table_entry *mtmr); -extern int sfi_mtimer_num; - -#else /* CONFIG_APB_TIMER */ - -static inline void apbt_time_init(void) { } - -#endif -#endif /* ASM_X86_APBT_H */ diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h deleted file mode 100644 index 777c0f63418c..000000000000 --- a/arch/x86/include/asm/crypto/glue_helper.h +++ /dev/null @@ -1,118 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Shared glue code for 128bit block ciphers - */ - -#ifndef _CRYPTO_GLUE_HELPER_H -#define _CRYPTO_GLUE_HELPER_H - -#include -#include -#include -#include - -typedef void (*common_glue_func_t)(const void *ctx, u8 *dst, const u8 *src); -typedef void (*common_glue_cbc_func_t)(const void *ctx, u8 *dst, const u8 *src); -typedef void (*common_glue_ctr_func_t)(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -typedef void (*common_glue_xts_func_t)(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -struct common_glue_func_entry { - unsigned int num_blocks; /* number of blocks that @fn will process */ - union { - common_glue_func_t ecb; - common_glue_cbc_func_t cbc; - common_glue_ctr_func_t ctr; - common_glue_xts_func_t xts; - } fn_u; -}; - -struct common_glue_ctx { - unsigned int num_funcs; - int fpu_blocks_limit; /* -1 means fpu not needed at all */ - - /* - * First funcs entry must have largest num_blocks and last funcs entry - * must have num_blocks == 1! - */ - struct common_glue_func_entry funcs[]; -}; - -static inline bool glue_fpu_begin(unsigned int bsize, int fpu_blocks_limit, - struct skcipher_walk *walk, - bool fpu_enabled, unsigned int nbytes) -{ - if (likely(fpu_blocks_limit < 0)) - return false; - - if (fpu_enabled) - return true; - - /* - * Vector-registers are only used when chunk to be processed is large - * enough, so do not enable FPU until it is necessary. - */ - if (nbytes < bsize * (unsigned int)fpu_blocks_limit) - return false; - - /* prevent sleeping if FPU is in use */ - skcipher_walk_atomise(walk); - - kernel_fpu_begin(); - return true; -} - -static inline void glue_fpu_end(bool fpu_enabled) -{ - if (fpu_enabled) - kernel_fpu_end(); -} - -static inline void le128_to_be128(be128 *dst, const le128 *src) -{ - dst->a = cpu_to_be64(le64_to_cpu(src->a)); - dst->b = cpu_to_be64(le64_to_cpu(src->b)); -} - -static inline void be128_to_le128(le128 *dst, const be128 *src) -{ - dst->a = cpu_to_le64(be64_to_cpu(src->a)); - dst->b = cpu_to_le64(be64_to_cpu(src->b)); -} - -static inline void le128_inc(le128 *i) -{ - u64 a = le64_to_cpu(i->a); - u64 b = le64_to_cpu(i->b); - - b++; - if (!b) - a++; - - i->a = cpu_to_le64(a); - i->b = cpu_to_le64(b); -} - -extern int glue_ecb_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req); - -extern int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn, - struct skcipher_request *req); - -extern int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req); - -extern int glue_ctr_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req); - -extern int glue_xts_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req, - common_glue_func_t tweak_fn, void *tweak_ctx, - void *crypt_ctx, bool decrypt); - -extern void glue_xts_crypt_128bit_one(const void *ctx, u8 *dst, - const u8 *src, le128 *iv, - common_glue_func_t fn); - -#endif /* _CRYPTO_GLUE_HELPER_H */ diff --git a/arch/x86/include/asm/crypto/serpent-avx.h b/arch/x86/include/asm/crypto/serpent-avx.h deleted file mode 100644 index 251c2c89d7cf..000000000000 --- a/arch/x86/include/asm/crypto/serpent-avx.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASM_X86_SERPENT_AVX_H -#define ASM_X86_SERPENT_AVX_H - -#include -#include -#include - -struct crypto_skcipher; - -#define SERPENT_PARALLEL_BLOCKS 8 - -struct serpent_xts_ctx { - struct serpent_ctx tweak_ctx; - struct serpent_ctx crypt_ctx; -}; - -asmlinkage void serpent_ecb_enc_8way_avx(const void *ctx, u8 *dst, - const u8 *src); -asmlinkage void serpent_ecb_dec_8way_avx(const void *ctx, u8 *dst, - const u8 *src); - -asmlinkage void serpent_cbc_dec_8way_avx(const void *ctx, u8 *dst, - const u8 *src); -asmlinkage void serpent_ctr_8way_avx(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -asmlinkage void serpent_xts_enc_8way_avx(const void *ctx, u8 *dst, - const u8 *src, le128 *iv); -asmlinkage void serpent_xts_dec_8way_avx(const void *ctx, u8 *dst, - const u8 *src, le128 *iv); - -extern void __serpent_crypt_ctr(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -extern void serpent_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv); -extern void serpent_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv); - -extern int xts_serpent_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen); - -#endif diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h index 9f1a0a987e5e..d0dcefb5cc59 100644 --- a/arch/x86/include/asm/fixmap.h +++ b/arch/x86/include/asm/fixmap.h @@ -108,9 +108,6 @@ enum fixed_addresses { #ifdef CONFIG_PARAVIRT_XXL FIX_PARAVIRT_BOOTMAP, #endif -#ifdef CONFIG_X86_INTEL_MID - FIX_LNW_VRTC, -#endif #ifdef CONFIG_ACPI_APEI_GHES /* Used for GHES mapping from assorted contexts */ diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 41e2e2e1b439..5eb3bdf36a41 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -187,23 +187,22 @@ __visible noinstr void func(struct pt_regs *regs, unsigned long error_code) * has to be done in the function body if necessary. */ #define DEFINE_IDTENTRY_IRQ(func) \ -static __always_inline void __##func(struct pt_regs *regs, u8 vector); \ +static void __##func(struct pt_regs *regs, u32 vector); \ \ __visible noinstr void func(struct pt_regs *regs, \ unsigned long error_code) \ { \ irqentry_state_t state = irqentry_enter(regs); \ + u32 vector = (u32)(u8)error_code; \ \ instrumentation_begin(); \ - irq_enter_rcu(); \ kvm_set_cpu_l1tf_flush_l1d(); \ - __##func (regs, (u8)error_code); \ - irq_exit_rcu(); \ + run_irq_on_irqstack_cond(__##func, regs, vector); \ instrumentation_end(); \ irqentry_exit(regs, state); \ } \ \ -static __always_inline void __##func(struct pt_regs *regs, u8 vector) +static noinline void __##func(struct pt_regs *regs, u32 vector) /** * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points @@ -237,10 +236,8 @@ __visible noinstr void func(struct pt_regs *regs) \ irqentry_state_t state = irqentry_enter(regs); \ \ instrumentation_begin(); \ - irq_enter_rcu(); \ kvm_set_cpu_l1tf_flush_l1d(); \ run_sysvec_on_irqstack_cond(__##func, regs); \ - irq_exit_rcu(); \ instrumentation_end(); \ irqentry_exit(regs, state); \ } \ diff --git a/arch/x86/include/asm/insn.h b/arch/x86/include/asm/insn.h index a8c3d284fa46..95a448fbb44c 100644 --- a/arch/x86/include/asm/insn.h +++ b/arch/x86/include/asm/insn.h @@ -7,9 +7,12 @@ * Copyright (C) IBM Corporation, 2009 */ +#include /* insn_attr_t is defined in inat.h */ #include +#if defined(__BYTE_ORDER) ? __BYTE_ORDER == __LITTLE_ENDIAN : defined(__LITTLE_ENDIAN) + struct insn_field { union { insn_value_t value; @@ -20,6 +23,48 @@ struct insn_field { unsigned char nbytes; }; +static inline void insn_field_set(struct insn_field *p, insn_value_t v, + unsigned char n) +{ + p->value = v; + p->nbytes = n; +} + +static inline void insn_set_byte(struct insn_field *p, unsigned char n, + insn_byte_t v) +{ + p->bytes[n] = v; +} + +#else + +struct insn_field { + insn_value_t value; + union { + insn_value_t little; + insn_byte_t bytes[4]; + }; + /* !0 if we've run insn_get_xxx() for this field */ + unsigned char got; + unsigned char nbytes; +}; + +static inline void insn_field_set(struct insn_field *p, insn_value_t v, + unsigned char n) +{ + p->value = v; + p->little = __cpu_to_le32(v); + p->nbytes = n; +} + +static inline void insn_set_byte(struct insn_field *p, unsigned char n, + insn_byte_t v) +{ + p->bytes[n] = v; + p->value = __le32_to_cpu(p->little); +} +#endif + struct insn { struct insn_field prefixes; /* * Prefixes diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index cf0e25f45422..c201083b34f6 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -1,15 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * intel-mid.h: Intel MID specific setup code + * Intel MID specific setup code * - * (C) Copyright 2009 Intel Corporation + * (C) Copyright 2009, 2021 Intel Corporation */ #ifndef _ASM_X86_INTEL_MID_H #define _ASM_X86_INTEL_MID_H -#include #include -#include extern int intel_mid_pci_init(void); extern int intel_mid_pci_set_power_state(struct pci_dev *pdev, pci_power_t state); @@ -22,93 +20,18 @@ extern void intel_mid_pwr_power_off(void); extern int intel_mid_pwr_get_lss_id(struct pci_dev *pdev); -extern int get_gpio_by_name(const char *name); -extern int __init sfi_parse_mrtc(struct sfi_table_header *table); -extern int __init sfi_parse_mtmr(struct sfi_table_header *table); -extern int sfi_mrtc_num; -extern struct sfi_rtc_table_entry sfi_mrtc_array[]; - -/* - * Here defines the array of devices platform data that IAFW would export - * through SFI "DEVS" table, we use name and type to match the device and - * its platform data. - */ -struct devs_id { - char name[SFI_NAME_LEN + 1]; - u8 type; - u8 delay; - u8 msic; - void *(*get_platform_data)(void *info); -}; - -#define sfi_device(i) \ - static const struct devs_id *const __intel_mid_sfi_##i##_dev __used \ - __section(".x86_intel_mid_dev.init") = &i - -/** -* struct mid_sd_board_info - template for SD device creation -* @name: identifies the driver -* @bus_num: board-specific identifier for a given SD controller -* @max_clk: the maximum frequency device supports -* @platform_data: the particular data stored there is driver-specific -*/ -struct mid_sd_board_info { - char name[SFI_NAME_LEN]; - int bus_num; - unsigned short addr; - u32 max_clk; - void *platform_data; -}; - -/* - * Medfield is the follow-up of Moorestown, it combines two chip solution into - * one. Other than that it also added always-on and constant tsc and lapic - * timers. Medfield is the platform name, and the chip name is called Penwell - * we treat Medfield/Penwell as a variant of Moorestown. Penwell can be - * identified via MSRs. - */ -enum intel_mid_cpu_type { - /* 1 was Moorestown */ - INTEL_MID_CPU_CHIP_PENWELL = 2, - INTEL_MID_CPU_CHIP_CLOVERVIEW, - INTEL_MID_CPU_CHIP_TANGIER, -}; - -extern enum intel_mid_cpu_type __intel_mid_cpu_chip; - #ifdef CONFIG_X86_INTEL_MID -static inline enum intel_mid_cpu_type intel_mid_identify_cpu(void) -{ - return __intel_mid_cpu_chip; -} - -static inline bool intel_mid_has_msic(void) -{ - return (intel_mid_identify_cpu() == INTEL_MID_CPU_CHIP_PENWELL); -} - extern void intel_scu_devices_create(void); extern void intel_scu_devices_destroy(void); #else /* !CONFIG_X86_INTEL_MID */ -#define intel_mid_identify_cpu() 0 -#define intel_mid_has_msic() 0 - static inline void intel_scu_devices_create(void) { } static inline void intel_scu_devices_destroy(void) { } #endif /* !CONFIG_X86_INTEL_MID */ -enum intel_mid_timer_options { - INTEL_MID_TIMER_DEFAULT, - INTEL_MID_TIMER_APBT_ONLY, - INTEL_MID_TIMER_LAPIC_APBT, -}; - -extern enum intel_mid_timer_options intel_mid_timer_options; - /* Bus Select SoC Fuse value */ #define BSEL_SOC_FUSE_MASK 0x7 /* FSB 133MHz */ @@ -118,16 +41,4 @@ extern enum intel_mid_timer_options intel_mid_timer_options; /* FSB 83MHz */ #define BSEL_SOC_FUSE_111 0x7 -#define SFI_MTMR_MAX_NUM 8 -#define SFI_MRTC_MAX 8 - -/* VRTC timer */ -#define MRST_VRTC_MAP_SZ 1024 -/* #define MRST_VRTC_PGOFFSET 0xc00 */ - -extern void intel_mid_rtc_init(void); - -/* The offset for the mapping of global gpio pin to irq */ -#define INTEL_MID_IRQ_OFFSET 0x100 - #endif /* _ASM_X86_INTEL_MID_H */ diff --git a/arch/x86/include/asm/intel_mid_vrtc.h b/arch/x86/include/asm/intel_mid_vrtc.h deleted file mode 100644 index 0b44b1abe4d9..000000000000 --- a/arch/x86/include/asm/intel_mid_vrtc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _INTEL_MID_VRTC_H -#define _INTEL_MID_VRTC_H - -extern unsigned char vrtc_cmos_read(unsigned char reg); -extern void vrtc_cmos_write(unsigned char val, unsigned char reg); -extern void vrtc_get_time(struct timespec64 *now); -extern int vrtc_set_mmss(const struct timespec64 *now); - -#endif diff --git a/arch/x86/include/asm/intel_scu_ipc.h b/arch/x86/include/asm/intel_scu_ipc.h index 11d457af68c5..8537f597d20a 100644 --- a/arch/x86/include/asm/intel_scu_ipc.h +++ b/arch/x86/include/asm/intel_scu_ipc.h @@ -65,6 +65,4 @@ static inline int intel_scu_ipc_dev_command(struct intel_scu_ipc_dev *scu, int c inlen, out, outlen); } -#include - #endif diff --git a/arch/x86/include/asm/intel_scu_ipc_legacy.h b/arch/x86/include/asm/intel_scu_ipc_legacy.h deleted file mode 100644 index 4cf13fecb673..000000000000 --- a/arch/x86/include/asm/intel_scu_ipc_legacy.h +++ /dev/null @@ -1,91 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_X86_INTEL_SCU_IPC_LEGACY_H_ -#define _ASM_X86_INTEL_SCU_IPC_LEGACY_H_ - -#include - -#define IPCMSG_INDIRECT_READ 0x02 -#define IPCMSG_INDIRECT_WRITE 0x05 - -#define IPCMSG_COLD_OFF 0x80 /* Only for Tangier */ - -#define IPCMSG_WARM_RESET 0xF0 -#define IPCMSG_COLD_RESET 0xF1 -#define IPCMSG_SOFT_RESET 0xF2 -#define IPCMSG_COLD_BOOT 0xF3 - -#define IPCMSG_VRTC 0xFA /* Set vRTC device */ -/* Command id associated with message IPCMSG_VRTC */ -#define IPC_CMD_VRTC_SETTIME 1 /* Set time */ -#define IPC_CMD_VRTC_SETALARM 2 /* Set alarm */ - -/* Don't call these in new code - they will be removed eventually */ - -/* Read single register */ -static inline int intel_scu_ipc_ioread8(u16 addr, u8 *data) -{ - return intel_scu_ipc_dev_ioread8(NULL, addr, data); -} - -/* Read a vector */ -static inline int intel_scu_ipc_readv(u16 *addr, u8 *data, int len) -{ - return intel_scu_ipc_dev_readv(NULL, addr, data, len); -} - -/* Write single register */ -static inline int intel_scu_ipc_iowrite8(u16 addr, u8 data) -{ - return intel_scu_ipc_dev_iowrite8(NULL, addr, data); -} - -/* Write a vector */ -static inline int intel_scu_ipc_writev(u16 *addr, u8 *data, int len) -{ - return intel_scu_ipc_dev_writev(NULL, addr, data, len); -} - -/* Update single register based on the mask */ -static inline int intel_scu_ipc_update_register(u16 addr, u8 data, u8 mask) -{ - return intel_scu_ipc_dev_update(NULL, addr, data, mask); -} - -/* Issue commands to the SCU with or without data */ -static inline int intel_scu_ipc_simple_command(int cmd, int sub) -{ - return intel_scu_ipc_dev_simple_command(NULL, cmd, sub); -} - -static inline int intel_scu_ipc_command(int cmd, int sub, u32 *in, int inlen, - u32 *out, int outlen) -{ - /* New API takes both inlen and outlen as bytes so convert here */ - size_t inbytes = inlen * sizeof(u32); - size_t outbytes = outlen * sizeof(u32); - - return intel_scu_ipc_dev_command_with_size(NULL, cmd, sub, in, inbytes, - inlen, out, outbytes); -} - -extern struct blocking_notifier_head intel_scu_notifier; - -static inline void intel_scu_notifier_add(struct notifier_block *nb) -{ - blocking_notifier_chain_register(&intel_scu_notifier, nb); -} - -static inline void intel_scu_notifier_remove(struct notifier_block *nb) -{ - blocking_notifier_chain_unregister(&intel_scu_notifier, nb); -} - -static inline int intel_scu_notifier_post(unsigned long v, void *p) -{ - return blocking_notifier_call_chain(&intel_scu_notifier, v, p); -} - -#define SCU_AVAILABLE 1 -#define SCU_DOWN 2 - -#endif diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h index 76d389691b5b..768aa234cbb4 100644 --- a/arch/x86/include/asm/irq.h +++ b/arch/x86/include/asm/irq.h @@ -25,8 +25,6 @@ static inline int irq_canonicalize(int irq) extern int irq_init_percpu_irqstack(unsigned int cpu); -#define __ARCH_HAS_DO_SOFTIRQ - struct irq_desc; extern void fixup_irqs(void); diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index 775816965c6a..9b2a0ff76c73 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -7,100 +7,217 @@ #include #ifdef CONFIG_X86_64 -static __always_inline bool irqstack_active(void) -{ - return __this_cpu_read(irq_count) != -1; + +/* + * Macro to inline switching to an interrupt stack and invoking function + * calls from there. The following rules apply: + * + * - Ordering: + * + * 1. Write the stack pointer into the top most place of the irq + * stack. This ensures that the various unwinders can link back to the + * original stack. + * + * 2. Switch the stack pointer to the top of the irq stack. + * + * 3. Invoke whatever needs to be done (@asm_call argument) + * + * 4. Pop the original stack pointer from the top of the irq stack + * which brings it back to the original stack where it left off. + * + * - Function invocation: + * + * To allow flexible usage of the macro, the actual function code including + * the store of the arguments in the call ABI registers is handed in via + * the @asm_call argument. + * + * - Local variables: + * + * @tos: + * The @tos variable holds a pointer to the top of the irq stack and + * _must_ be allocated in a non-callee saved register as this is a + * restriction coming from objtool. + * + * Note, that (tos) is both in input and output constraints to ensure + * that the compiler does not assume that R11 is left untouched in + * case this macro is used in some place where the per cpu interrupt + * stack pointer is used again afterwards + * + * - Function arguments: + * The function argument(s), if any, have to be defined in register + * variables at the place where this is invoked. Storing the + * argument(s) in the proper register(s) is part of the @asm_call + * + * - Constraints: + * + * The constraints have to be done very carefully because the compiler + * does not know about the assembly call. + * + * output: + * As documented already above the @tos variable is required to be in + * the output constraints to make the compiler aware that R11 cannot be + * reused after the asm() statement. + * + * For builds with CONFIG_UNWIND_FRAME_POINTER ASM_CALL_CONSTRAINT is + * required as well as this prevents certain creative GCC variants from + * misplacing the ASM code. + * + * input: + * - func: + * Immediate, which tells the compiler that the function is referenced. + * + * - tos: + * Register. The actual register is defined by the variable declaration. + * + * - function arguments: + * The constraints are handed in via the 'argconstr' argument list. They + * describe the register arguments which are used in @asm_call. + * + * clobbers: + * Function calls can clobber anything except the callee-saved + * registers. Tell the compiler. + */ +#define call_on_irqstack(func, asm_call, argconstr...) \ +{ \ + register void *tos asm("r11"); \ + \ + tos = ((void *)__this_cpu_read(hardirq_stack_ptr)); \ + \ + asm_inline volatile( \ + "movq %%rsp, (%[tos]) \n" \ + "movq %[tos], %%rsp \n" \ + \ + asm_call \ + \ + "popq %%rsp \n" \ + \ + : "+r" (tos), ASM_CALL_CONSTRAINT \ + : [__func] "i" (func), [tos] "r" (tos) argconstr \ + : "cc", "rax", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10", \ + "memory" \ + ); \ } -void asm_call_on_stack(void *sp, void (*func)(void), void *arg); -void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs), - struct pt_regs *regs); -void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc), - struct irq_desc *desc); +/* Macros to assert type correctness for run_*_on_irqstack macros */ +#define assert_function_type(func, proto) \ + static_assert(__builtin_types_compatible_p(typeof(&func), proto)) -static __always_inline void __run_on_irqstack(void (*func)(void)) -{ - void *tos = __this_cpu_read(hardirq_stack_ptr); +#define assert_arg_type(arg, proto) \ + static_assert(__builtin_types_compatible_p(typeof(arg), proto)) - __this_cpu_add(irq_count, 1); - asm_call_on_stack(tos - 8, func, NULL); - __this_cpu_sub(irq_count, 1); +/* + * Macro to invoke system vector and device interrupt C handlers. + */ +#define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...) \ +{ \ + /* \ + * User mode entry and interrupt on the irq stack do not \ + * switch stacks. If from user mode the task stack is empty. \ + */ \ + if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) { \ + irq_enter_rcu(); \ + func(c_args); \ + irq_exit_rcu(); \ + } else { \ + /* \ + * Mark the irq stack inuse _before_ and unmark _after_ \ + * switching stacks. Interrupts are disabled in both \ + * places. Invoke the stack switch macro with the call \ + * sequence which matches the above direct invocation. \ + */ \ + __this_cpu_write(hardirq_stack_inuse, true); \ + call_on_irqstack(func, asm_call, constr); \ + __this_cpu_write(hardirq_stack_inuse, false); \ + } \ } -static __always_inline void -__run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), - struct pt_regs *regs) -{ - void *tos = __this_cpu_read(hardirq_stack_ptr); +/* + * Function call sequence for __call_on_irqstack() for system vectors. + * + * Note that irq_enter_rcu() and irq_exit_rcu() do not use the input + * mechanism because these functions are global and cannot be optimized out + * when compiling a particular source file which uses one of these macros. + * + * The argument (regs) does not need to be pushed or stashed in a callee + * saved register to be safe vs. the irq_enter_rcu() call because the + * clobbers already prevent the compiler from storing it in a callee + * clobbered register. As the compiler has to preserve @regs for the final + * call to idtentry_exit() anyway, it's likely that it does not cause extra + * effort for this asm magic. + */ +#define ASM_CALL_SYSVEC \ + "call irq_enter_rcu \n" \ + "movq %[arg1], %%rdi \n" \ + "call %P[__func] \n" \ + "call irq_exit_rcu \n" - __this_cpu_add(irq_count, 1); - asm_call_sysvec_on_stack(tos - 8, func, regs); - __this_cpu_sub(irq_count, 1); +#define SYSVEC_CONSTRAINTS , [arg1] "r" (regs) + +#define run_sysvec_on_irqstack_cond(func, regs) \ +{ \ + assert_function_type(func, void (*)(struct pt_regs *)); \ + assert_arg_type(regs, struct pt_regs *); \ + \ + call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC, \ + SYSVEC_CONSTRAINTS, regs); \ } -static __always_inline void -__run_irq_on_irqstack(void (*func)(struct irq_desc *desc), - struct irq_desc *desc) -{ - void *tos = __this_cpu_read(hardirq_stack_ptr); +/* + * As in ASM_CALL_SYSVEC above the clobbers force the compiler to store + * @regs and @vector in callee saved registers. + */ +#define ASM_CALL_IRQ \ + "call irq_enter_rcu \n" \ + "movq %[arg1], %%rdi \n" \ + "movl %[arg2], %%esi \n" \ + "call %P[__func] \n" \ + "call irq_exit_rcu \n" - __this_cpu_add(irq_count, 1); - asm_call_irq_on_stack(tos - 8, func, desc); - __this_cpu_sub(irq_count, 1); +#define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" (vector) + +#define run_irq_on_irqstack_cond(func, regs, vector) \ +{ \ + assert_function_type(func, void (*)(struct pt_regs *, u32)); \ + assert_arg_type(regs, struct pt_regs *); \ + assert_arg_type(vector, u32); \ + \ + call_on_irqstack_cond(func, regs, ASM_CALL_IRQ, \ + IRQ_CONSTRAINTS, regs, vector); \ +} + +#define ASM_CALL_SOFTIRQ \ + "call %P[__func] \n" + +/* + * Macro to invoke __do_softirq on the irq stack. This is only called from + * task context when bottom halfs are about to be reenabled and soft + * interrupts are pending to be processed. The interrupt stack cannot be in + * use here. + */ +#define do_softirq_own_stack() \ +{ \ + __this_cpu_write(hardirq_stack_inuse, true); \ + call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \ + __this_cpu_write(hardirq_stack_inuse, false); \ } #else /* CONFIG_X86_64 */ -static inline bool irqstack_active(void) { return false; } -static inline void __run_on_irqstack(void (*func)(void)) { } -static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), - struct pt_regs *regs) { } -static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), - struct irq_desc *desc) { } +/* System vector handlers always run on the stack they interrupted. */ +#define run_sysvec_on_irqstack_cond(func, regs) \ +{ \ + irq_enter_rcu(); \ + func(regs); \ + irq_exit_rcu(); \ +} + +/* Switches to the irq stack within func() */ +#define run_irq_on_irqstack_cond(func, regs, vector) \ +{ \ + irq_enter_rcu(); \ + func(regs, vector); \ + irq_exit_rcu(); \ +} + #endif /* !CONFIG_X86_64 */ -static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs) -{ - if (IS_ENABLED(CONFIG_X86_32)) - return false; - if (!regs) - return !irqstack_active(); - return !user_mode(regs) && !irqstack_active(); -} - - -static __always_inline void run_on_irqstack_cond(void (*func)(void), - struct pt_regs *regs) -{ - lockdep_assert_irqs_disabled(); - - if (irq_needs_irq_stack(regs)) - __run_on_irqstack(func); - else - func(); -} - -static __always_inline void -run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs), - struct pt_regs *regs) -{ - lockdep_assert_irqs_disabled(); - - if (irq_needs_irq_stack(regs)) - __run_sysvec_on_irqstack(func, regs); - else - func(regs); -} - -static __always_inline void -run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc, - struct pt_regs *regs) -{ - lockdep_assert_irqs_disabled(); - - if (irq_needs_irq_stack(regs)) - __run_irq_on_irqstack(func, desc); - else - func(desc); -} - #endif diff --git a/arch/x86/include/asm/kfence.h b/arch/x86/include/asm/kfence.h new file mode 100644 index 000000000000..97bbb4a9083a --- /dev/null +++ b/arch/x86/include/asm/kfence.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * x86 KFENCE support. + * + * Copyright (C) 2020, Google LLC. + */ + +#ifndef _ASM_X86_KFENCE_H +#define _ASM_X86_KFENCE_H + +#include +#include + +#include +#include +#include +#include + +/* Force 4K pages for __kfence_pool. */ +static inline bool arch_kfence_init_pool(void) +{ + unsigned long addr; + + for (addr = (unsigned long)__kfence_pool; is_kfence_address((void *)addr); + addr += PAGE_SIZE) { + unsigned int level; + + if (!lookup_address(addr, &level)) + return false; + + if (level != PG_LEVEL_4K) + set_memory_4k(addr, 1); + } + + return true; +} + +/* Protect the given page and flush TLB. */ +static inline bool kfence_protect_page(unsigned long addr, bool protect) +{ + unsigned int level; + pte_t *pte = lookup_address(addr, &level); + + if (WARN_ON(!pte || level != PG_LEVEL_4K)) + return false; + + /* + * We need to avoid IPIs, as we may get KFENCE allocations or faults + * with interrupts disabled. Therefore, the below is best-effort, and + * does not flush TLBs on all CPUs. We can tolerate some inaccuracy; + * lazy fault handling takes care of faults after the page is PRESENT. + */ + + if (protect) + set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT)); + else + set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT)); + + /* Flush this CPU's TLB. */ + flush_tlb_one_kernel(addr); + return true; +} + +#endif /* _ASM_X86_KFENCE_H */ diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 355a2ab8fc09..323641097f63 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -93,11 +93,7 @@ KVM_X86_OP(check_intercept) KVM_X86_OP(handle_exit_irqoff) KVM_X86_OP_NULL(request_immediate_exit) KVM_X86_OP(sched_in) -KVM_X86_OP_NULL(slot_enable_log_dirty) -KVM_X86_OP_NULL(slot_disable_log_dirty) -KVM_X86_OP_NULL(flush_log_dirty) -KVM_X86_OP_NULL(enable_log_dirty_pt_masked) -KVM_X86_OP_NULL(cpu_dirty_log_size) +KVM_X86_OP_NULL(update_cpu_dirty_logging) KVM_X86_OP_NULL(pre_block) KVM_X86_OP_NULL(post_block) KVM_X86_OP_NULL(vcpu_blocking) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 84499aad01a4..0cf71ff2b2e5 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -89,6 +89,8 @@ KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_APF_READY KVM_ARCH_REQ(28) #define KVM_REQ_MSR_FILTER_CHANGED KVM_ARCH_REQ(29) +#define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \ + KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define CR0_RESERVED_BITS \ (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ @@ -1007,6 +1009,7 @@ struct kvm_arch { u32 bsp_vcpu_id; u64 disabled_quirks; + int cpu_dirty_logging_count; enum kvm_irqchip_mode irqchip_mode; u8 nr_reserved_ioapic_pins; @@ -1271,30 +1274,11 @@ struct kvm_x86_ops { void (*sched_in)(struct kvm_vcpu *kvm, int cpu); /* - * Arch-specific dirty logging hooks. These hooks are only supposed to - * be valid if the specific arch has hardware-accelerated dirty logging - * mechanism. Currently only for PML on VMX. - * - * - slot_enable_log_dirty: - * called when enabling log dirty mode for the slot. - * - slot_disable_log_dirty: - * called when disabling log dirty mode for the slot. - * also called when slot is created with log dirty disabled. - * - flush_log_dirty: - * called before reporting dirty_bitmap to userspace. - * - enable_log_dirty_pt_masked: - * called when reenabling log dirty for the GFNs in the mask after - * corresponding bits are cleared in slot->dirty_bitmap. + * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A zero + * value indicates CPU dirty logging is unsupported or disabled. */ - void (*slot_enable_log_dirty)(struct kvm *kvm, - struct kvm_memory_slot *slot); - void (*slot_disable_log_dirty)(struct kvm *kvm, - struct kvm_memory_slot *slot); - void (*flush_log_dirty)(struct kvm *kvm); - void (*enable_log_dirty_pt_masked)(struct kvm *kvm, - struct kvm_memory_slot *slot, - gfn_t offset, unsigned long mask); - int (*cpu_dirty_log_size)(void); + int cpu_dirty_log_size; + void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); /* pmu operations of sub-arch */ const struct kvm_pmu_ops *pmu_ops; @@ -1437,11 +1421,6 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, struct kvm_memory_slot *memslot); void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, struct kvm_memory_slot *memslot); -void kvm_mmu_slot_set_dirty(struct kvm *kvm, - struct kvm_memory_slot *memslot); -void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, - struct kvm_memory_slot *slot, - gfn_t gfn_offset, unsigned long mask); void kvm_mmu_zap_all(struct kvm *kvm); void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen); unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm); @@ -1613,7 +1592,6 @@ void kvm_inject_nmi(struct kvm_vcpu *vcpu); void kvm_update_dr7(struct kvm_vcpu *vcpu); int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn); -int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); int kvm_mmu_load(struct kvm_vcpu *vcpu); void kvm_mmu_unload(struct kvm_vcpu *vcpu); diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h index fdbffec4cfde..5a2baf28a1dc 100644 --- a/arch/x86/include/asm/orc_types.h +++ b/arch/x86/include/asm/orc_types.h @@ -40,6 +40,8 @@ #define ORC_REG_MAX 15 #ifndef __ASSEMBLY__ +#include + /* * This struct is more or less a vastly simplified version of the DWARF Call * Frame Information standard. It contains only the necessary parts of DWARF @@ -51,10 +53,18 @@ struct orc_entry { s16 sp_offset; s16 bp_offset; +#if defined(__LITTLE_ENDIAN_BITFIELD) unsigned sp_reg:4; unsigned bp_reg:4; unsigned type:2; unsigned end:1; +#elif defined(__BIG_ENDIAN_BITFIELD) + unsigned bp_reg:4; + unsigned sp_reg:4; + unsigned unused:5; + unsigned end:1; + unsigned type:2; +#endif } __packed; #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/include/asm/platform_sst_audio.h b/arch/x86/include/asm/platform_sst_audio.h index 16b9f220bdeb..40f92270515b 100644 --- a/arch/x86/include/asm/platform_sst_audio.h +++ b/arch/x86/include/asm/platform_sst_audio.h @@ -10,8 +10,6 @@ #ifndef _PLATFORM_SST_AUDIO_H_ #define _PLATFORM_SST_AUDIO_H_ -#include - #define MAX_NUM_STREAMS_MRFLD 25 #define MAX_NUM_STREAMS MAX_NUM_STREAMS_MRFLD diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index c20a52b5534b..dc6d149bf851 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -426,8 +426,6 @@ struct irq_stack { char stack[IRQ_STACK_SIZE]; } __aligned(IRQ_STACK_SIZE); -DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); - #ifdef CONFIG_X86_32 DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack); #else @@ -454,7 +452,8 @@ static inline unsigned long cpu_kernelmode_gs_base(int cpu) return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu); } -DECLARE_PER_CPU(unsigned int, irq_count); +DECLARE_PER_CPU(void *, hardirq_stack_ptr); +DECLARE_PER_CPU(bool, hardirq_stack_inuse); extern asmlinkage void ignore_sysret(void); /* Save actual FS/GS selectors and bases to current->thread */ @@ -473,9 +472,9 @@ struct stack_canary { }; DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary); #endif -/* Per CPU softirq stack pointer */ +DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr); -#endif /* X86_64 */ +#endif /* !X86_64 */ extern unsigned int fpu_kernel_xstate_size; extern unsigned int fpu_user_xstate_size; diff --git a/arch/x86/include/asm/softirq_stack.h b/arch/x86/include/asm/softirq_stack.h new file mode 100644 index 000000000000..889d53d6a0e1 --- /dev/null +++ b/arch/x86/include/asm/softirq_stack.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_SOFTIRQ_STACK_H +#define _ASM_X86_SOFTIRQ_STACK_H + +#ifdef CONFIG_X86_64 +# include +#else +# include +#endif + +#endif diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index 664d4610d700..8e574c0afef8 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -48,17 +48,8 @@ UNWIND_HINT_REGS base=\base offset=\offset partial=1 .endm -.macro UNWIND_HINT_FUNC sp_offset=8 - UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=\sp_offset type=UNWIND_HINT_TYPE_CALL -.endm - -/* - * RET_OFFSET: Used on instructions that terminate a function; mostly RETURN - * and sibling calls. On these, sp_offset denotes the expected offset from - * initial_func_cfi. - */ -.macro UNWIND_HINT_RET_OFFSET sp_offset=8 - UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_RET_OFFSET sp_offset=\sp_offset +.macro UNWIND_HINT_FUNC + UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=8 type=UNWIND_HINT_TYPE_FUNC .endm #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 5eeb808eb024..2ddf08351f0b 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -116,7 +116,6 @@ obj-$(CONFIG_VM86) += vm86_32.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_HPET_TIMER) += hpet.o -obj-$(CONFIG_APB_TIMER) += apb_timer.o obj-$(CONFIG_AMD_NB) += amd_nb.o obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o diff --git a/arch/x86/kernel/acpi/Makefile b/arch/x86/kernel/acpi/Makefile index f1bb57b0e41e..cf340d85946a 100644 --- a/arch/x86/kernel/acpi/Makefile +++ b/arch/x86/kernel/acpi/Makefile @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD_wakeup_$(BITS).o := y obj-$(CONFIG_ACPI) += boot.o obj-$(CONFIG_ACPI_SLEEP) += sleep.o wakeup_$(BITS).o diff --git a/arch/x86/kernel/acpi/wakeup_64.S b/arch/x86/kernel/acpi/wakeup_64.S index 5d3a0b8fd379..56b6865afb2a 100644 --- a/arch/x86/kernel/acpi/wakeup_64.S +++ b/arch/x86/kernel/acpi/wakeup_64.S @@ -1,12 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0-only */ .text #include +#include #include #include #include #include #include #include +#include # Copyright 2003 Pavel Machek -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#define APBT_CLOCKEVENT_RATING 110 -#define APBT_CLOCKSOURCE_RATING 250 - -#define APBT_CLOCKEVENT0_NUM (0) -#define APBT_CLOCKSOURCE_NUM (2) - -static phys_addr_t apbt_address; -static int apb_timer_block_enabled; -static void __iomem *apbt_virt_address; - -/* - * Common DW APB timer info - */ -static unsigned long apbt_freq; - -struct apbt_dev { - struct dw_apb_clock_event_device *timer; - unsigned int num; - int cpu; - unsigned int irq; - char name[10]; -}; - -static struct dw_apb_clocksource *clocksource_apbt; - -static inline void __iomem *adev_virt_addr(struct apbt_dev *adev) -{ - return apbt_virt_address + adev->num * APBTMRS_REG_SIZE; -} - -static DEFINE_PER_CPU(struct apbt_dev, cpu_apbt_dev); - -#ifdef CONFIG_SMP -static unsigned int apbt_num_timers_used; -#endif - -static inline void apbt_set_mapping(void) -{ - struct sfi_timer_table_entry *mtmr; - int phy_cs_timer_id = 0; - - if (apbt_virt_address) { - pr_debug("APBT base already mapped\n"); - return; - } - mtmr = sfi_get_mtmr(APBT_CLOCKEVENT0_NUM); - if (mtmr == NULL) { - printk(KERN_ERR "Failed to get MTMR %d from SFI\n", - APBT_CLOCKEVENT0_NUM); - return; - } - apbt_address = (phys_addr_t)mtmr->phys_addr; - if (!apbt_address) { - printk(KERN_WARNING "No timer base from SFI, use default\n"); - apbt_address = APBT_DEFAULT_BASE; - } - apbt_virt_address = ioremap(apbt_address, APBT_MMAP_SIZE); - if (!apbt_virt_address) { - pr_debug("Failed mapping APBT phy address at %lu\n",\ - (unsigned long)apbt_address); - goto panic_noapbt; - } - apbt_freq = mtmr->freq_hz; - sfi_free_mtmr(mtmr); - - /* Now figure out the physical timer id for clocksource device */ - mtmr = sfi_get_mtmr(APBT_CLOCKSOURCE_NUM); - if (mtmr == NULL) - goto panic_noapbt; - - /* Now figure out the physical timer id */ - pr_debug("Use timer %d for clocksource\n", - (int)(mtmr->phys_addr & 0xff) / APBTMRS_REG_SIZE); - phy_cs_timer_id = (unsigned int)(mtmr->phys_addr & 0xff) / - APBTMRS_REG_SIZE; - - clocksource_apbt = dw_apb_clocksource_init(APBT_CLOCKSOURCE_RATING, - "apbt0", apbt_virt_address + phy_cs_timer_id * - APBTMRS_REG_SIZE, apbt_freq); - return; - -panic_noapbt: - panic("Failed to setup APB system timer\n"); - -} - -static inline void apbt_clear_mapping(void) -{ - iounmap(apbt_virt_address); - apbt_virt_address = NULL; -} - -static int __init apbt_clockevent_register(void) -{ - struct sfi_timer_table_entry *mtmr; - struct apbt_dev *adev = this_cpu_ptr(&cpu_apbt_dev); - - mtmr = sfi_get_mtmr(APBT_CLOCKEVENT0_NUM); - if (mtmr == NULL) { - printk(KERN_ERR "Failed to get MTMR %d from SFI\n", - APBT_CLOCKEVENT0_NUM); - return -ENODEV; - } - - adev->num = smp_processor_id(); - adev->timer = dw_apb_clockevent_init(smp_processor_id(), "apbt0", - intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT ? - APBT_CLOCKEVENT_RATING - 100 : APBT_CLOCKEVENT_RATING, - adev_virt_addr(adev), 0, apbt_freq); - /* Firmware does EOI handling for us. */ - adev->timer->eoi = NULL; - - if (intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT) { - global_clock_event = &adev->timer->ced; - printk(KERN_DEBUG "%s clockevent registered as global\n", - global_clock_event->name); - } - - dw_apb_clockevent_register(adev->timer); - - sfi_free_mtmr(mtmr); - return 0; -} - -#ifdef CONFIG_SMP - -static void apbt_setup_irq(struct apbt_dev *adev) -{ - irq_modify_status(adev->irq, 0, IRQ_MOVE_PCNTXT); - irq_set_affinity(adev->irq, cpumask_of(adev->cpu)); -} - -/* Should be called with per cpu */ -void apbt_setup_secondary_clock(void) -{ - struct apbt_dev *adev; - int cpu; - - /* Don't register boot CPU clockevent */ - cpu = smp_processor_id(); - if (!cpu) - return; - - adev = this_cpu_ptr(&cpu_apbt_dev); - if (!adev->timer) { - adev->timer = dw_apb_clockevent_init(cpu, adev->name, - APBT_CLOCKEVENT_RATING, adev_virt_addr(adev), - adev->irq, apbt_freq); - adev->timer->eoi = NULL; - } else { - dw_apb_clockevent_resume(adev->timer); - } - - printk(KERN_INFO "Registering CPU %d clockevent device %s, cpu %08x\n", - cpu, adev->name, adev->cpu); - - apbt_setup_irq(adev); - dw_apb_clockevent_register(adev->timer); - - return; -} - -/* - * this notify handler process CPU hotplug events. in case of S0i3, nonboot - * cpus are disabled/enabled frequently, for performance reasons, we keep the - * per cpu timer irq registered so that we do need to do free_irq/request_irq. - * - * TODO: it might be more reliable to directly disable percpu clockevent device - * without the notifier chain. currently, cpu 0 may get interrupts from other - * cpu timers during the offline process due to the ordering of notification. - * the extra interrupt is harmless. - */ -static int apbt_cpu_dead(unsigned int cpu) -{ - struct apbt_dev *adev = &per_cpu(cpu_apbt_dev, cpu); - - dw_apb_clockevent_pause(adev->timer); - if (system_state == SYSTEM_RUNNING) { - pr_debug("skipping APBT CPU %u offline\n", cpu); - } else { - pr_debug("APBT clockevent for cpu %u offline\n", cpu); - dw_apb_clockevent_stop(adev->timer); - } - return 0; -} - -static __init int apbt_late_init(void) -{ - if (intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT || - !apb_timer_block_enabled) - return 0; - return cpuhp_setup_state(CPUHP_X86_APB_DEAD, "x86/apb:dead", NULL, - apbt_cpu_dead); -} -fs_initcall(apbt_late_init); -#else - -void apbt_setup_secondary_clock(void) {} - -#endif /* CONFIG_SMP */ - -static int apbt_clocksource_register(void) -{ - u64 start, now; - u64 t1; - - /* Start the counter, use timer 2 as source, timer 0/1 for event */ - dw_apb_clocksource_start(clocksource_apbt); - - /* Verify whether apbt counter works */ - t1 = dw_apb_clocksource_read(clocksource_apbt); - start = rdtsc(); - - /* - * We don't know the TSC frequency yet, but waiting for - * 200000 TSC cycles is safe: - * 4 GHz == 50us - * 1 GHz == 200us - */ - do { - rep_nop(); - now = rdtsc(); - } while ((now - start) < 200000UL); - - /* APBT is the only always on clocksource, it has to work! */ - if (t1 == dw_apb_clocksource_read(clocksource_apbt)) - panic("APBT counter not counting. APBT disabled\n"); - - dw_apb_clocksource_register(clocksource_apbt); - - return 0; -} - -/* - * Early setup the APBT timer, only use timer 0 for booting then switch to - * per CPU timer if possible. - * returns 1 if per cpu apbt is setup - * returns 0 if no per cpu apbt is chosen - * panic if set up failed, this is the only platform timer on Moorestown. - */ -void __init apbt_time_init(void) -{ -#ifdef CONFIG_SMP - int i; - struct sfi_timer_table_entry *p_mtmr; - struct apbt_dev *adev; -#endif - - if (apb_timer_block_enabled) - return; - apbt_set_mapping(); - if (!apbt_virt_address) - goto out_noapbt; - /* - * Read the frequency and check for a sane value, for ESL model - * we extend the possible clock range to allow time scaling. - */ - - if (apbt_freq < APBT_MIN_FREQ || apbt_freq > APBT_MAX_FREQ) { - pr_debug("APBT has invalid freq 0x%lx\n", apbt_freq); - goto out_noapbt; - } - if (apbt_clocksource_register()) { - pr_debug("APBT has failed to register clocksource\n"); - goto out_noapbt; - } - if (!apbt_clockevent_register()) - apb_timer_block_enabled = 1; - else { - pr_debug("APBT has failed to register clockevent\n"); - goto out_noapbt; - } -#ifdef CONFIG_SMP - /* kernel cmdline disable apb timer, so we will use lapic timers */ - if (intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT) { - printk(KERN_INFO "apbt: disabled per cpu timer\n"); - return; - } - pr_debug("%s: %d CPUs online\n", __func__, num_online_cpus()); - if (num_possible_cpus() <= sfi_mtimer_num) - apbt_num_timers_used = num_possible_cpus(); - else - apbt_num_timers_used = 1; - pr_debug("%s: %d APB timers used\n", __func__, apbt_num_timers_used); - - /* here we set up per CPU timer data structure */ - for (i = 0; i < apbt_num_timers_used; i++) { - adev = &per_cpu(cpu_apbt_dev, i); - adev->num = i; - adev->cpu = i; - p_mtmr = sfi_get_mtmr(i); - if (p_mtmr) - adev->irq = p_mtmr->irq; - else - printk(KERN_ERR "Failed to get timer for cpu %d\n", i); - snprintf(adev->name, sizeof(adev->name) - 1, "apbt%d", i); - } -#endif - - return; - -out_noapbt: - apbt_clear_mapping(); - apb_timer_block_enabled = 0; - panic("failed to enable APB timer\n"); -} diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 819db00c9388..bda4f2a36868 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2138,18 +2138,11 @@ void __init register_lapic_address(unsigned long address) * Local APIC interrupts */ -/** - * spurious_interrupt - Catch all for interrupts raised on unused vectors - * @regs: Pointer to pt_regs on stack - * @vector: The vector number - * - * This is invoked from ASM entry code to catch all interrupts which - * trigger on an entry which is routed to the common_spurious idtentry - * point. - * - * Also called from sysvec_spurious_apic_interrupt(). +/* + * Common handling code for spurious_interrupt and spurious_vector entry + * points below. No point in allowing the compiler to inline it twice. */ -DEFINE_IDTENTRY_IRQ(spurious_interrupt) +static noinline void handle_spurious_interrupt(u8 vector) { u32 v; @@ -2184,9 +2177,23 @@ out: trace_spurious_apic_exit(vector); } +/** + * spurious_interrupt - Catch all for interrupts raised on unused vectors + * @regs: Pointer to pt_regs on stack + * @vector: The vector number + * + * This is invoked from ASM entry code to catch all interrupts which + * trigger on an entry which is routed to the common_spurious idtentry + * point. + */ +DEFINE_IDTENTRY_IRQ(spurious_interrupt) +{ + handle_spurious_interrupt(vector); +} + DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt) { - __spurious_interrupt(regs, SPURIOUS_APIC_VECTOR); + handle_spurious_interrupt(SPURIOUS_APIC_VECTOR); } /* diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index e4ab4804b20d..c3b60c37c728 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -198,7 +198,7 @@ static int __init parse_noapic(char *str) } early_param("noapic", parse_noapic); -/* Will be called in mpparse/acpi/sfi codes for saving IRQ info */ +/* Will be called in mpparse/ACPI codes for saving IRQ info */ void mp_save_irq(struct mpc_intsrc *m) { int i; @@ -2863,7 +2863,7 @@ int mp_register_ioapic(int id, u32 address, u32 gsi_base, /* * If mp_register_ioapic() is called during early boot stage when - * walking ACPI/SFI/DT tables, it's too early to create irqdomain, + * walking ACPI/DT tables, it's too early to create irqdomain, * we are still using bootmem allocator. So delay it to setup_IO_APIC(). */ if (hotplug) { diff --git a/arch/x86/kernel/cpu/acrn.c b/arch/x86/kernel/cpu/acrn.c index 0b2c03943ac6..23f5f27b5a02 100644 --- a/arch/x86/kernel/cpu/acrn.c +++ b/arch/x86/kernel/cpu/acrn.c @@ -10,6 +10,8 @@ */ #include + +#include #include #include #include @@ -19,7 +21,7 @@ static u32 __init acrn_detect(void) { - return hypervisor_cpuid_base("ACRNACRNACRN", 0); + return acrn_cpuid_base(); } static void __init acrn_init_platform(void) @@ -55,6 +57,18 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_acrn_hv_callback) set_irq_regs(old_regs); } +void acrn_setup_intr_handler(void (*handler)(void)) +{ + acrn_intr_handler = handler; +} +EXPORT_SYMBOL_GPL(acrn_setup_intr_handler); + +void acrn_remove_intr_handler(void) +{ + acrn_intr_handler = NULL; +} +EXPORT_SYMBOL_GPL(acrn_remove_intr_handler); + const __initconst struct hypervisor_x86 x86_hyper_acrn = { .name = "ACRN", .detect = acrn_detect, diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 9215b91bc044..ab640abe26b6 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1742,8 +1742,8 @@ DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned = &init_task; EXPORT_PER_CPU_SYMBOL(current_task); -DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); -DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1; +DEFINE_PER_CPU(void *, hardirq_stack_ptr); +DEFINE_PER_CPU(bool, hardirq_stack_inuse); DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT; EXPORT_PER_CPU_SYMBOL(__preempt_count); diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index 1dd851397bd9..5601b95944fa 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -128,12 +128,21 @@ static __always_inline bool in_exception_stack(unsigned long *stack, struct stac static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info *info) { - unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr); - unsigned long *begin = end - (IRQ_STACK_SIZE / sizeof(long)); + unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr); + unsigned long *begin; /* - * This is a software stack, so 'end' can be a valid stack pointer. - * It just means the stack is empty. + * @end points directly to the top most stack entry to avoid a -8 + * adjustment in the stack switch hotpath. Adjust it back before + * calculating @begin. + */ + end++; + begin = end - (IRQ_STACK_SIZE / sizeof(long)); + + /* + * Due to the switching logic RSP can never be == @end because the + * final operation is 'popq %rsp' which means after that RSP points + * to the original stack and not to @end. */ if (stack < begin || stack >= end) return false; @@ -143,8 +152,9 @@ static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info info->end = end; /* - * The next stack pointer is the first thing pushed by the entry code - * after switching to the irq stack. + * The next stack pointer is stored at the top of the irq stack + * before switching to the irq stack. Actual stack entries are all + * below that. */ info->next_sp = (unsigned long *)*(end - 1); diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 0d54099c2a3a..7c273846c687 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -184,6 +184,7 @@ SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) * It is also used to copy the retq for trampolines. */ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) + UNWIND_HINT_FUNC retq SYM_FUNC_END(ftrace_epilogue) @@ -276,7 +277,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) restore_mcount_regs 8 /* Restore flags */ popfq - UNWIND_HINT_RET_OFFSET + UNWIND_HINT_FUNC jmp ftrace_epilogue SYM_FUNC_END(ftrace_regs_caller) @@ -333,8 +334,7 @@ SYM_FUNC_START(ftrace_graph_caller) retq SYM_FUNC_END(ftrace_graph_caller) -SYM_CODE_START(return_to_handler) - UNWIND_HINT_EMPTY +SYM_FUNC_START(return_to_handler) subq $24, %rsp /* Save the return values */ @@ -349,5 +349,5 @@ SYM_CODE_START(return_to_handler) movq (%rsp), %rax addq $24, %rsp JMP_NOSPEC rdi -SYM_CODE_END(return_to_handler) +SYM_FUNC_END(return_to_handler) #endif diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index d4ad344e80bf..58aa712973ac 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -228,7 +228,7 @@ static __always_inline void handle_irq(struct irq_desc *desc, struct pt_regs *regs) { if (IS_ENABLED(CONFIG_X86_64)) - run_irq_on_irqstack_cond(desc->handle_irq, desc, regs); + generic_handle_irq_desc(desc); else __handle_irq(desc, regs); } diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index 0b79efc87be5..044902d5a3c4 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -22,6 +22,7 @@ #include #include +#include #ifdef CONFIG_DEBUG_STACKOVERFLOW diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index 440eed558558..1c0fb96b9e39 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -48,7 +49,8 @@ static int map_irq_stack(unsigned int cpu) if (!va) return -ENOMEM; - per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE; + /* Store actual TOS to avoid adjustment in the hotpath */ + per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; return 0; } #else @@ -60,7 +62,8 @@ static int map_irq_stack(unsigned int cpu) { void *va = per_cpu_ptr(&irq_stack_backing_store, cpu); - per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE; + /* Store actual TOS to avoid adjustment in the hotpath */ + per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; return 0; } #endif @@ -71,8 +74,3 @@ int irq_init_percpu_irqstack(unsigned int cpu) return 0; return map_irq_stack(cpu); } - -void do_softirq_own_stack(void) -{ - run_on_irqstack_cond(__do_softirq, NULL); -} diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 145a7ac0c19a..9c214d7085a4 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -161,7 +161,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, #endif /* Kernel thread ? */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); kthread_frame_init(frame, sp, arg); return 0; diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index ad582f9ac5a6..d08307df69ad 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -539,7 +539,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) int cpu = smp_processor_id(); WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) && - this_cpu_read(irq_count) != -1); + this_cpu_read(hardirq_stack_inuse)); if (!test_thread_flag(TIF_NEED_FPU_LOAD)) switch_fpu_prepare(prev_fpu, cpu); diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 740f3bdb3f61..d883176ef2ce 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -1185,7 +1184,6 @@ void __init setup_arch(char **cmdline_p) * Read APIC and some other early information from ACPI tables. */ acpi_boot_init(); - sfi_init(); x86_dtb_init(); /* diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index 73f800100066..2a1d47f47eee 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -471,7 +471,7 @@ bool unwind_next_frame(struct unwind_state *state) break; case ORC_REG_SP_INDIRECT: - sp = state->sp + orc->sp_offset; + sp = state->sp; indirect = true; break; @@ -521,6 +521,9 @@ bool unwind_next_frame(struct unwind_state *state) if (indirect) { if (!deref_stack_reg(state, sp, &sp)) goto err; + + if (orc->sp_reg == ORC_REG_SP_INDIRECT) + sp += orc->sp_offset; } /* Find IP, SP and possibly regs: */ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index c8f2592ccc99..6bd2f8b830e4 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -408,7 +408,7 @@ void kvm_set_cpu_caps(void) kvm_cpu_cap_mask(CPUID_7_0_EBX, F(FSGSBASE) | F(BMI1) | F(HLE) | F(AVX2) | F(SMEP) | - F(BMI2) | F(ERMS) | 0 /*INVPCID*/ | F(RTM) | 0 /*MPX*/ | F(RDSEED) | + F(BMI2) | F(ERMS) | F(INVPCID) | F(RTM) | 0 /*MPX*/ | F(RDSEED) | F(ADX) | F(SMAP) | F(AVX512IFMA) | F(AVX512F) | F(AVX512PF) | F(AVX512ER) | F(AVX512CD) | F(CLFLUSHOPT) | F(CLWB) | F(AVX512DQ) | F(SHA_NI) | F(AVX512BW) | F(AVX512VL) | 0 /*INTEL_PT*/ diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index e507568cd55d..d75524bc8423 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1165,7 +1165,8 @@ static bool spte_wrprot_for_clear_dirty(u64 *sptep) * - W bit on ad-disabled SPTEs. * Returns true iff any D or W bits were cleared. */ -static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head) +static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head, + struct kvm_memory_slot *slot) { u64 *sptep; struct rmap_iterator iter; @@ -1180,35 +1181,6 @@ static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head) return flush; } -static bool spte_set_dirty(u64 *sptep) -{ - u64 spte = *sptep; - - rmap_printk("spte %p %llx\n", sptep, *sptep); - - /* - * Similar to the !kvm_x86_ops.slot_disable_log_dirty case, - * do not bother adding back write access to pages marked - * SPTE_AD_WRPROT_ONLY_MASK. - */ - spte |= shadow_dirty_mask; - - return mmu_spte_update(sptep, spte); -} - -static bool __rmap_set_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head) -{ - u64 *sptep; - struct rmap_iterator iter; - bool flush = false; - - for_each_rmap_spte(rmap_head, &iter, sptep) - if (spte_ad_enabled(*sptep)) - flush |= spte_set_dirty(sptep); - - return flush; -} - /** * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages * @kvm: kvm instance @@ -1248,9 +1220,9 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, * * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap. */ -void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, - struct kvm_memory_slot *slot, - gfn_t gfn_offset, unsigned long mask) +static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask) { struct kvm_rmap_head *rmap_head; @@ -1260,13 +1232,12 @@ void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, while (mask) { rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), PG_LEVEL_4K, slot); - __rmap_clear_dirty(kvm, rmap_head); + __rmap_clear_dirty(kvm, rmap_head, slot); /* clear the first set bit */ mask &= mask - 1; } } -EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked); /** * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected @@ -1282,20 +1253,15 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask) { - if (kvm_x86_ops.enable_log_dirty_pt_masked) - static_call(kvm_x86_enable_log_dirty_pt_masked)(kvm, slot, - gfn_offset, - mask); + if (kvm_x86_ops.cpu_dirty_log_size) + kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask); else kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); } int kvm_cpu_dirty_log_size(void) { - if (kvm_x86_ops.cpu_dirty_log_size) - return static_call(kvm_x86_cpu_dirty_log_size)(); - - return 0; + return kvm_x86_ops.cpu_dirty_log_size; } bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, @@ -1325,7 +1291,8 @@ static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn); } -static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head) +static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, + struct kvm_memory_slot *slot) { u64 *sptep; struct rmap_iterator iter; @@ -1345,7 +1312,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, struct kvm_memory_slot *slot, gfn_t gfn, int level, unsigned long data) { - return kvm_zap_rmapp(kvm, rmap_head); + return kvm_zap_rmapp(kvm, rmap_head, slot); } static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, @@ -2499,7 +2466,21 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) return r; } -EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page); + +static int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) +{ + gpa_t gpa; + int r; + + if (vcpu->arch.mmu->direct_map) + return 0; + + gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL); + + r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); + + return r; +} static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { @@ -2753,11 +2734,18 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) if (sp->role.level > PG_LEVEL_4K) return; + /* + * If addresses are being invalidated, skip prefetching to avoid + * accidentally prefetching those addresses. + */ + if (unlikely(vcpu->kvm->mmu_notifier_count)) + return; + __direct_pte_prefetch(vcpu, sp, sptep); } -static int host_pfn_mapping_level(struct kvm_vcpu *vcpu, gfn_t gfn, - kvm_pfn_t pfn, struct kvm_memory_slot *slot) +static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + struct kvm_memory_slot *slot) { unsigned long hva; pte_t *pte; @@ -2776,19 +2764,36 @@ static int host_pfn_mapping_level(struct kvm_vcpu *vcpu, gfn_t gfn, */ hva = __gfn_to_hva_memslot(slot, gfn); - pte = lookup_address_in_mm(vcpu->kvm->mm, hva, &level); + pte = lookup_address_in_mm(kvm->mm, hva, &level); if (unlikely(!pte)) return PG_LEVEL_4K; return level; } +int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_memory_slot *slot, + gfn_t gfn, kvm_pfn_t pfn, int max_level) +{ + struct kvm_lpage_info *linfo; + + max_level = min(max_level, max_huge_page_level); + for ( ; max_level > PG_LEVEL_4K; max_level--) { + linfo = lpage_info_slot(gfn, slot, max_level); + if (!linfo->disallow_lpage) + break; + } + + if (max_level == PG_LEVEL_4K) + return PG_LEVEL_4K; + + return host_pfn_mapping_level(kvm, gfn, pfn, slot); +} + int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, int max_level, kvm_pfn_t *pfnp, bool huge_page_disallowed, int *req_level) { struct kvm_memory_slot *slot; - struct kvm_lpage_info *linfo; kvm_pfn_t pfn = *pfnp; kvm_pfn_t mask; int level; @@ -2805,17 +2810,7 @@ int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, if (!slot) return PG_LEVEL_4K; - max_level = min(max_level, max_huge_page_level); - for ( ; max_level > PG_LEVEL_4K; max_level--) { - linfo = lpage_info_slot(gfn, slot, max_level); - if (!linfo->disallow_lpage) - break; - } - - if (max_level == PG_LEVEL_4K) - return PG_LEVEL_4K; - - level = host_pfn_mapping_level(vcpu, gfn, pfn, slot); + level = kvm_mmu_max_mapping_level(vcpu->kvm, slot, gfn, pfn, max_level); if (level == PG_LEVEL_4K) return level; @@ -3437,7 +3432,6 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); write_unlock(&vcpu->kvm->mmu_lock); } -EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots); static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gpa_t vaddr, u32 access, struct x86_exception *exception) @@ -3653,8 +3647,8 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, } static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, - gpa_t cr2_or_gpa, kvm_pfn_t *pfn, bool write, - bool *writable) + gpa_t cr2_or_gpa, kvm_pfn_t *pfn, hva_t *hva, + bool write, bool *writable) { struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); bool async; @@ -3667,7 +3661,8 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, } async = false; - *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable); + *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, + write, writable, hva); if (!async) return false; /* *pfn has correct page already */ @@ -3681,7 +3676,8 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, return true; } - *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, write, writable); + *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, + write, writable, hva); return false; } @@ -3694,6 +3690,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, gfn_t gfn = gpa >> PAGE_SHIFT; unsigned long mmu_seq; kvm_pfn_t pfn; + hva_t hva; int r; if (page_fault_handle_page_track(vcpu, error_code, gfn)) @@ -3712,7 +3709,8 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable)) + if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, &hva, + write, &map_writable)) return RET_PF_RETRY; if (handle_abnormal_pfn(vcpu, is_tdp ? 0 : gpa, gfn, pfn, ACC_ALL, &r)) @@ -3725,7 +3723,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, else write_lock(&vcpu->kvm->mmu_lock); - if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) + if (!is_noslot_pfn(pfn) && mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, hva)) goto out_unlock; r = make_mmu_pages_available(vcpu); if (r) @@ -5003,22 +5001,6 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, write_unlock(&vcpu->kvm->mmu_lock); } -int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) -{ - gpa_t gpa; - int r; - - if (vcpu->arch.mmu->direct_map) - return 0; - - gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL); - - r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); - - return r; -} -EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt); - int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, void *insn, int insn_len) { @@ -5117,7 +5099,6 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, mmu->invlpg(vcpu, gva, root_hpa); } } -EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_gva); void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) { @@ -5157,7 +5138,6 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) * for them. */ } -EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva); void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, int tdp_huge_page_level) @@ -5182,7 +5162,8 @@ void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, EXPORT_SYMBOL_GPL(kvm_configure_mmu); /* The return value indicates if tlb flush on all vcpus is needed. */ -typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head); +typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head, + struct kvm_memory_slot *slot); /* The caller should hold mmu-lock before calling this function. */ static __always_inline bool @@ -5196,7 +5177,7 @@ slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot, for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn, end_gfn, &iterator) { if (iterator.rmap) - flush |= fn(kvm, iterator.rmap); + flush |= fn(kvm, iterator.rmap, memslot); if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) { if (flush && lock_flush_tlb) { @@ -5229,22 +5210,6 @@ slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot, lock_flush_tlb); } -static __always_inline bool -slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot, - slot_level_handler fn, bool lock_flush_tlb) -{ - return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K, - KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); -} - -static __always_inline bool -slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot, - slot_level_handler fn, bool lock_flush_tlb) -{ - return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K + 1, - KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); -} - static __always_inline bool slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot, slot_level_handler fn, bool lock_flush_tlb) @@ -5485,7 +5450,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) } static bool slot_rmap_write_protect(struct kvm *kvm, - struct kvm_rmap_head *rmap_head) + struct kvm_rmap_head *rmap_head, + struct kvm_memory_slot *slot) { return __rmap_write_protect(kvm, rmap_head, false); } @@ -5519,7 +5485,8 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, } static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm, - struct kvm_rmap_head *rmap_head) + struct kvm_rmap_head *rmap_head, + struct kvm_memory_slot *slot) { u64 *sptep; struct rmap_iterator iter; @@ -5540,8 +5507,8 @@ restart: * mapping if the indirect sp has level = 1. */ if (sp->role.direct && !kvm_is_reserved_pfn(pfn) && - (kvm_is_zone_device_pfn(pfn) || - PageCompound(pfn_to_page(pfn)))) { + sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn, + pfn, PG_LEVEL_NUM)) { pte_list_remove(rmap_head, sptep); if (kvm_available_flush_tlb_with_range()) @@ -5561,12 +5528,13 @@ void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, const struct kvm_memory_slot *memslot) { /* FIXME: const-ify all uses of struct kvm_memory_slot. */ + struct kvm_memory_slot *slot = (struct kvm_memory_slot *)memslot; + write_lock(&kvm->mmu_lock); - slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot, - kvm_mmu_zap_collapsible_spte, true); + slot_handle_leaf(kvm, slot, kvm_mmu_zap_collapsible_spte, true); if (is_tdp_mmu_enabled(kvm)) - kvm_tdp_mmu_zap_collapsible_sptes(kvm, memslot); + kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot); write_unlock(&kvm->mmu_lock); } @@ -5605,40 +5573,6 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, if (flush) kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); } -EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty); - -void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, - struct kvm_memory_slot *memslot) -{ - bool flush; - - write_lock(&kvm->mmu_lock); - flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect, - false); - if (is_tdp_mmu_enabled(kvm)) - flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_2M); - write_unlock(&kvm->mmu_lock); - - if (flush) - kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); -} -EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access); - -void kvm_mmu_slot_set_dirty(struct kvm *kvm, - struct kvm_memory_slot *memslot) -{ - bool flush; - - write_lock(&kvm->mmu_lock); - flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false); - if (is_tdp_mmu_enabled(kvm)) - flush |= kvm_tdp_mmu_slot_set_dirty(kvm, memslot); - write_unlock(&kvm->mmu_lock); - - if (flush) - kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); -} -EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty); void kvm_mmu_zap_all(struct kvm *kvm) { diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index 9e38d3c5daad..72b0928f2b2d 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -84,7 +84,10 @@ static inline bool kvm_vcpu_ad_need_write_protect(struct kvm_vcpu *vcpu) * When using the EPT page-modification log, the GPAs in the log * would come from L2 rather than L1. Therefore, we need to rely * on write protection to record dirty pages. This also bypasses - * PML, since writes now result in a vmexit. + * PML, since writes now result in a vmexit. Note, this helper will + * tag SPTEs as needing write-protection even if PML is disabled or + * unsupported, but that's ok because the tag is consumed if and only + * if PML is enabled. Omit the PML check to save a few uops. */ return vcpu->arch.mmu == &vcpu->arch.guest_mmu; } @@ -138,6 +141,8 @@ enum { #define SET_SPTE_NEED_REMOTE_TLB_FLUSH BIT(1) #define SET_SPTE_SPURIOUS BIT(2) +int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_memory_slot *slot, + gfn_t gfn, kvm_pfn_t pfn, int max_level); int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, int max_level, kvm_pfn_t *pfnp, bool huge_page_disallowed, int *req_level); diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index d9f66cc459e8..55d7b473ac44 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -601,6 +601,13 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, if (sp->role.level > PG_LEVEL_4K) return; + /* + * If addresses are being invalidated, skip prefetching to avoid + * accidentally prefetching those addresses. + */ + if (unlikely(vcpu->kvm->mmu_notifier_count)) + return; + if (sp->role.direct) return __direct_pte_prefetch(vcpu, sp, sptep); @@ -790,6 +797,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, struct guest_walker walker; int r; kvm_pfn_t pfn; + hva_t hva; unsigned long mmu_seq; bool map_writable, is_self_change_mapping; int max_level; @@ -840,8 +848,8 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, write_fault, - &map_writable)) + if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, &hva, + write_fault, &map_writable)) return RET_PF_RETRY; if (handle_abnormal_pfn(vcpu, addr, walker.gfn, pfn, walker.pte_access, &r)) @@ -869,7 +877,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) + if (!is_noslot_pfn(pfn) && mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, hva)) goto out_unlock; kvm_mmu_audit(vcpu, AUDIT_PRE_PAGE_FAULT); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 71e100a5670f..c926c6b899a1 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1268,68 +1268,16 @@ void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, } } -/* - * Set the dirty status of all the SPTEs mapping GFNs in the memslot. This is - * only used for PML, and so will involve setting the dirty bit on each SPTE. - * Returns true if an SPTE has been changed and the TLBs need to be flushed. - */ -static bool set_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, - gfn_t start, gfn_t end) -{ - struct tdp_iter iter; - u64 new_spte; - bool spte_set = false; - - rcu_read_lock(); - - tdp_root_for_each_pte(iter, root, start, end) { - if (tdp_mmu_iter_cond_resched(kvm, &iter, false)) - continue; - - if (!is_shadow_present_pte(iter.old_spte) || - iter.old_spte & shadow_dirty_mask) - continue; - - new_spte = iter.old_spte | shadow_dirty_mask; - - tdp_mmu_set_spte(kvm, &iter, new_spte); - spte_set = true; - } - - rcu_read_unlock(); - return spte_set; -} - -/* - * Set the dirty status of all the SPTEs mapping GFNs in the memslot. This is - * only used for PML, and so will involve setting the dirty bit on each SPTE. - * Returns true if an SPTE has been changed and the TLBs need to be flushed. - */ -bool kvm_tdp_mmu_slot_set_dirty(struct kvm *kvm, struct kvm_memory_slot *slot) -{ - struct kvm_mmu_page *root; - int root_as_id; - bool spte_set = false; - - for_each_tdp_mmu_root_yield_safe(kvm, root) { - root_as_id = kvm_mmu_page_as_id(root); - if (root_as_id != slot->as_id) - continue; - - spte_set |= set_dirty_gfn_range(kvm, root, slot->base_gfn, - slot->base_gfn + slot->npages); - } - return spte_set; -} - /* * Clear leaf entries which could be replaced by large mappings, for * GFNs within the slot. */ static void zap_collapsible_spte_range(struct kvm *kvm, struct kvm_mmu_page *root, - gfn_t start, gfn_t end) + struct kvm_memory_slot *slot) { + gfn_t start = slot->base_gfn; + gfn_t end = start + slot->npages; struct tdp_iter iter; kvm_pfn_t pfn; bool spte_set = false; @@ -1348,7 +1296,8 @@ static void zap_collapsible_spte_range(struct kvm *kvm, pfn = spte_to_pfn(iter.old_spte); if (kvm_is_reserved_pfn(pfn) || - !PageTransCompoundMap(pfn_to_page(pfn))) + iter.level >= kvm_mmu_max_mapping_level(kvm, slot, iter.gfn, + pfn, PG_LEVEL_NUM)) continue; tdp_mmu_set_spte(kvm, &iter, 0); @@ -1366,7 +1315,7 @@ static void zap_collapsible_spte_range(struct kvm *kvm, * be replaced by large mappings, for GFNs within the slot. */ void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, - const struct kvm_memory_slot *slot) + struct kvm_memory_slot *slot) { struct kvm_mmu_page *root; int root_as_id; @@ -1376,8 +1325,7 @@ void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, if (root_as_id != slot->as_id) continue; - zap_collapsible_spte_range(kvm, root, slot->base_gfn, - slot->base_gfn + slot->npages); + zap_collapsible_spte_range(kvm, root, slot); } } diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h index b4b65e3699b3..3b761c111bff 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.h +++ b/arch/x86/kvm/mmu/tdp_mmu.h @@ -33,9 +33,8 @@ void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, unsigned long mask, bool wrprot); -bool kvm_tdp_mmu_slot_set_dirty(struct kvm *kvm, struct kvm_memory_slot *slot); void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, - const struct kvm_memory_slot *slot); + struct kvm_memory_slot *slot); bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn); diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index cc91738ab445..35891d9a1099 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -51,6 +51,23 @@ static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, nested_svm_vmexit(svm); } +static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_exception *fault) +{ + struct vcpu_svm *svm = to_svm(vcpu); + WARN_ON(!is_guest_mode(vcpu)); + + if (vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && + !svm->nested.nested_run_pending) { + svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; + svm->vmcb->control.exit_code_hi = 0; + svm->vmcb->control.exit_info_1 = fault->error_code; + svm->vmcb->control.exit_info_2 = fault->address; + nested_svm_vmexit(svm); + } else { + kvm_inject_page_fault(vcpu, fault); + } +} + static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) { struct vcpu_svm *svm = to_svm(vcpu); @@ -436,16 +453,33 @@ int enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb12_gpa, { int ret; + trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, + vmcb12->save.rip, + vmcb12->control.int_ctl, + vmcb12->control.event_inj, + vmcb12->control.nested_ctl); + + trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, + vmcb12->control.intercepts[INTERCEPT_CR] >> 16, + vmcb12->control.intercepts[INTERCEPT_EXCEPTION], + vmcb12->control.intercepts[INTERCEPT_WORD3], + vmcb12->control.intercepts[INTERCEPT_WORD4], + vmcb12->control.intercepts[INTERCEPT_WORD5]); + + svm->nested.vmcb12_gpa = vmcb12_gpa; load_nested_vmcb_control(svm, &vmcb12->control); - nested_prepare_vmcb_save(svm, vmcb12); nested_prepare_vmcb_control(svm); + nested_prepare_vmcb_save(svm, vmcb12); ret = nested_svm_load_cr3(&svm->vcpu, vmcb12->save.cr3, nested_npt_enabled(svm)); if (ret) return ret; + if (!npt_enabled) + svm->vcpu.arch.mmu->inject_page_fault = svm_inject_page_fault_nested; + svm_set_gif(svm, true); return 0; @@ -489,18 +523,6 @@ int nested_svm_vmrun(struct vcpu_svm *svm) goto out; } - trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, - vmcb12->save.rip, - vmcb12->control.int_ctl, - vmcb12->control.event_inj, - vmcb12->control.nested_ctl); - - trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, - vmcb12->control.intercepts[INTERCEPT_CR] >> 16, - vmcb12->control.intercepts[INTERCEPT_EXCEPTION], - vmcb12->control.intercepts[INTERCEPT_WORD3], - vmcb12->control.intercepts[INTERCEPT_WORD4], - vmcb12->control.intercepts[INTERCEPT_WORD5]); /* Clear internal status */ kvm_clear_exception_queue(&svm->vcpu); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index adb3619a3c16..c636021b066b 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -926,9 +926,6 @@ static __init void svm_set_cpu_caps(void) if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || boot_cpu_has(X86_FEATURE_AMD_SSBD)) kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); - - /* Enable INVPCID feature */ - kvm_cpu_cap_check_and_set(X86_FEATURE_INVPCID); } static __init int svm_hardware_setup(void) @@ -1103,12 +1100,12 @@ static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) static void svm_check_invpcid(struct vcpu_svm *svm) { /* - * Intercept INVPCID instruction only if shadow page table is - * enabled. Interception is not required with nested page table - * enabled. + * Intercept INVPCID if shadow paging is enabled to sync/free shadow + * roots, or if INVPCID is disabled in the guest to inject #UD. */ if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) { - if (!npt_enabled) + if (!npt_enabled || + !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID)) svm_set_intercept(svm, INTERCEPT_INVPCID); else svm_clr_intercept(svm, INTERCEPT_INVPCID); @@ -2214,15 +2211,20 @@ static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode) [SVM_INSTR_VMSAVE] = vmsave_interception, }; struct vcpu_svm *svm = to_svm(vcpu); + int ret; if (is_guest_mode(vcpu)) { svm->vmcb->control.exit_code = guest_mode_exit_codes[opcode]; svm->vmcb->control.exit_info_1 = 0; svm->vmcb->control.exit_info_2 = 0; - return nested_svm_vmexit(svm); - } else - return svm_instr_handlers[opcode](svm); + /* Returns '1' or -errno on failure, '0' on success. */ + ret = nested_svm_vmexit(svm); + if (ret) + return ret; + return 1; + } + return svm_instr_handlers[opcode](svm); } /* diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index b2f0b5e9cd63..bcca0b80e0d0 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2167,15 +2167,13 @@ static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); /* - * The PML address never changes, so it is constant in vmcs02. - * Conceptually we want to copy the PML index from vmcs01 here, - * and then back to vmcs01 on nested vmexit. But since we flush - * the log and reset GUEST_PML_INDEX on each vmexit, the PML - * index is also effectively constant in vmcs02. + * PML is emulated for L2, but never enabled in hardware as the MMU + * handles A/D emulation. Disabling PML for L2 also avoids having to + * deal with filtering out L2 GPAs from the buffer. */ if (enable_pml) { - vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); - vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); + vmcs_write64(PML_ADDRESS, 0); + vmcs_write16(GUEST_PML_INDEX, -1); } if (cpu_has_vmx_encls_vmexit()) @@ -2210,7 +2208,7 @@ static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) { - u32 exec_control, vmcs12_exec_ctrl; + u32 exec_control; u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) @@ -2284,11 +2282,11 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) SECONDARY_EXEC_APIC_REGISTER_VIRT | SECONDARY_EXEC_ENABLE_VMFUNC); if (nested_cpu_has(vmcs12, - CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) { - vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control & - ~SECONDARY_EXEC_ENABLE_PML; - exec_control |= vmcs12_exec_ctrl; - } + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) + exec_control |= vmcs12->secondary_vm_exec_control; + + /* PML is emulated and never enabled in hardware for L2. */ + exec_control &= ~SECONDARY_EXEC_ENABLE_PML; /* VMCS shadowing for L2 is emulated for now */ exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; @@ -4200,9 +4198,6 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &ignored)) nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); - if (!enable_ept) - vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault; - nested_vmx_transition_tlb_flush(vcpu, vmcs12, false); vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); @@ -4495,6 +4490,11 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, vmx_set_virtual_apic_mode(vcpu); } + if (vmx->nested.update_vmcs01_cpu_dirty_logging) { + vmx->nested.update_vmcs01_cpu_dirty_logging = false; + vmx_update_cpu_dirty_logging(vcpu); + } + /* Unpin physical memory we referred to in vmcs02 */ if (vmx->nested.apic_access_page) { kvm_release_page_clean(vmx->nested.apic_access_page); @@ -5793,7 +5793,10 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, case EXIT_REASON_PREEMPTION_TIMER: return true; case EXIT_REASON_PML_FULL: - /* We emulate PML support to L1. */ + /* + * PML is emulated for an L1 VMM and should never be enabled in + * vmcs02, always "handle" PML_FULL by exiting to userspace. + */ return true; case EXIT_REASON_VMFUNC: /* VM functions are emulated through L2->L0 vmexits. */ diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index d1df618cb7de..9efc1a6b8693 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -298,7 +298,7 @@ int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu) if (IS_ERR(event)) { pr_debug_ratelimited("%s: failed %ld\n", __func__, PTR_ERR(event)); - return -ENOENT; + return PTR_ERR(event); } lbr_desc->event = event; pmu->event_count++; @@ -320,7 +320,7 @@ static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu, if (!intel_pmu_is_valid_lbr_msr(vcpu, index)) return false; - if (!lbr_desc->event && !intel_pmu_create_guest_lbr_event(vcpu)) + if (!lbr_desc->event && intel_pmu_create_guest_lbr_event(vcpu) < 0) goto dummy; /* diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index e0a3a9be654b..50810d471462 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4277,7 +4277,12 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) */ exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; - if (!enable_pml) + /* + * PML is enabled/disabled when dirty logging of memsmlots changes, but + * it needs to be set here when dirty logging is already active, e.g. + * if this vCPU was created after dirty logging was enabled. + */ + if (!vcpu->kvm->arch.cpu_dirty_logging_count) exec_control &= ~SECONDARY_EXEC_ENABLE_PML; if (cpu_has_vmx_xsaves()) { @@ -4295,18 +4300,8 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) } vmx_adjust_sec_exec_feature(vmx, &exec_control, rdtscp, RDTSCP); - - /* - * Expose INVPCID if and only if PCID is also exposed to the guest. - * INVPCID takes a #UD when it's disabled in the VMCS, but a #GP or #PF - * if CR4.PCIDE=0. Enumerating CPUID.INVPCID=1 would lead to incorrect - * behavior from the guest perspective (it would expect #GP or #PF). - */ - if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID)) - guest_cpuid_clear(vcpu, X86_FEATURE_INVPCID); vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID); - vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND); vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdseed, RDSEED); @@ -5776,24 +5771,6 @@ static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); } -/* - * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap. - * Called before reporting dirty_bitmap to userspace. - */ -static void kvm_flush_pml_buffers(struct kvm *kvm) -{ - int i; - struct kvm_vcpu *vcpu; - /* - * We only need to kick vcpu out of guest mode here, as PML buffer - * is flushed at beginning of all VMEXITs, and it's obvious that only - * vcpus running in guest are possible to have unflushed GPAs in PML - * buffer. - */ - kvm_for_each_vcpu(i, vcpu, kvm) - kvm_vcpu_kick(vcpu); -} - static void vmx_dump_sel(char *name, uint32_t sel) { pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n", @@ -5976,9 +5953,10 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before * querying dirty_bitmap, we only need to kick all vcpus out of guest * mode as if vcpus is in root mode, the PML buffer must has been - * flushed already. + * flushed already. Note, PML is never enabled in hardware while + * running L2. */ - if (enable_pml) + if (enable_pml && !is_guest_mode(vcpu)) vmx_flush_pml_buffer(vcpu); /* @@ -5994,6 +5972,13 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) return handle_invalid_guest_state(vcpu); if (is_guest_mode(vcpu)) { + /* + * PML is never enabled when running L2, bail immediately if a + * PML full exit occurs as something is horribly wrong. + */ + if (exit_reason.basic == EXIT_REASON_PML_FULL) + goto unexpected_vmexit; + /* * The host physical addresses of some pages of guest memory * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC @@ -6851,13 +6836,15 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY)) kvm_machine_check(); + if (likely(!vmx->exit_reason.failed_vmentry)) + vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + trace_kvm_exit(vmx->exit_reason.full, vcpu, KVM_ISA_VMX); if (unlikely(vmx->exit_reason.failed_vmentry)) return EXIT_FASTPATH_NONE; vmx->loaded_vmcs->launched = 1; - vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); vmx_recover_nmi_blocking(vmx); vmx_complete_interrupts(vmx); @@ -7330,8 +7317,8 @@ static __init void vmx_set_cpu_caps(void) /* CPUID 0x7 */ if (kvm_mpx_supported()) kvm_cpu_cap_check_and_set(X86_FEATURE_MPX); - if (cpu_has_vmx_invpcid()) - kvm_cpu_cap_check_and_set(X86_FEATURE_INVPCID); + if (!cpu_has_vmx_invpcid()) + kvm_cpu_cap_clear(X86_FEATURE_INVPCID); if (vmx_pt_mode_is_host_guest()) kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); @@ -7509,30 +7496,24 @@ static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu) shrink_ple_window(vcpu); } -static void vmx_slot_enable_log_dirty(struct kvm *kvm, - struct kvm_memory_slot *slot) +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) { - if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) - kvm_mmu_slot_leaf_clear_dirty(kvm, slot); - kvm_mmu_slot_largepage_remove_write_access(kvm, slot); -} + struct vcpu_vmx *vmx = to_vmx(vcpu); -static void vmx_slot_disable_log_dirty(struct kvm *kvm, - struct kvm_memory_slot *slot) -{ - kvm_mmu_slot_set_dirty(kvm, slot); -} + if (is_guest_mode(vcpu)) { + vmx->nested.update_vmcs01_cpu_dirty_logging = true; + return; + } -static void vmx_flush_log_dirty(struct kvm *kvm) -{ - kvm_flush_pml_buffers(kvm); -} - -static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm, - struct kvm_memory_slot *memslot, - gfn_t offset, unsigned long mask) -{ - kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask); + /* + * Note, cpu_dirty_logging_count can be changed concurrent with this + * code, but in that case another update request will be made and so + * the guest will never run with a stale PML value. + */ + if (vcpu->kvm->arch.cpu_dirty_logging_count) + secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); + else + secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); } static int vmx_pre_block(struct kvm_vcpu *vcpu) @@ -7642,11 +7623,6 @@ static bool vmx_check_apicv_inhibit_reasons(ulong bit) return supported & BIT(bit); } -static int vmx_cpu_dirty_log_size(void) -{ - return enable_pml ? PML_ENTITY_NUM : 0; -} - static struct kvm_x86_ops vmx_x86_ops __initdata = { .hardware_unsetup = hardware_unsetup, @@ -7746,10 +7722,8 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .sched_in = vmx_sched_in, - .slot_enable_log_dirty = vmx_slot_enable_log_dirty, - .slot_disable_log_dirty = vmx_slot_disable_log_dirty, - .flush_log_dirty = vmx_flush_log_dirty, - .enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked, + .cpu_dirty_log_size = PML_ENTITY_NUM, + .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging, .pre_block = vmx_pre_block, .post_block = vmx_post_block, @@ -7777,7 +7751,6 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .msr_filter_changed = vmx_msr_filter_changed, .complete_emulated_msr = kvm_complete_insn_gp, - .cpu_dirty_log_size = vmx_cpu_dirty_log_size, .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector, }; @@ -7894,13 +7867,8 @@ static __init int hardware_setup(void) if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) enable_pml = 0; - if (!enable_pml) { - vmx_x86_ops.slot_enable_log_dirty = NULL; - vmx_x86_ops.slot_disable_log_dirty = NULL; - vmx_x86_ops.flush_log_dirty = NULL; - vmx_x86_ops.enable_log_dirty_pt_masked = NULL; - vmx_x86_ops.cpu_dirty_log_size = NULL; - } + if (!enable_pml) + vmx_x86_ops.cpu_dirty_log_size = 0; if (!cpu_has_vmx_preemption_timer()) enable_preemption_timer = false; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 12c53d05a902..89da5e1251f1 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -165,6 +165,7 @@ struct nested_vmx { bool change_vmcs01_virtual_apic_mode; bool reload_vmcs01_apic_access_page; + bool update_vmcs01_cpu_dirty_logging; /* * Enlightened VMCS has been enabled. It does not mean that L1 has to @@ -393,6 +394,7 @@ int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr); void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu); void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool value); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); static inline u8 vmx_get_rvi(void) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 884e5b3838c7..3712bb5245eb 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -5215,10 +5215,18 @@ static int kvm_vm_ioctl_reinject(struct kvm *kvm, void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { + /* - * Flush potentially hardware-cached dirty pages to dirty_bitmap. + * Flush all CPUs' dirty log buffers to the dirty_bitmap. Called + * before reporting dirty_bitmap to userspace. KVM flushes the buffers + * on all VM-Exits, thus we only need to kick running vCPUs to force a + * VM-Exit. */ - static_call_cond(kvm_x86_flush_log_dirty)(kvm); + struct kvm_vcpu *vcpu; + int i; + + kvm_for_each_vcpu(i, vcpu, kvm) + kvm_vcpu_kick(vcpu); } int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event, @@ -8980,6 +8988,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_check_async_pf_completion(vcpu); if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu)) static_call(kvm_x86_msr_filter_changed)(vcpu); + + if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) + static_call(kvm_x86_update_cpu_dirty_logging)(vcpu); } if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || @@ -10748,75 +10759,96 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, return 0; } + +static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable) +{ + struct kvm_arch *ka = &kvm->arch; + + if (!kvm_x86_ops.cpu_dirty_log_size) + return; + + if ((enable && ++ka->cpu_dirty_logging_count == 1) || + (!enable && --ka->cpu_dirty_logging_count == 0)) + kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING); + + WARN_ON_ONCE(ka->cpu_dirty_logging_count < 0); +} + static void kvm_mmu_slot_apply_flags(struct kvm *kvm, struct kvm_memory_slot *old, struct kvm_memory_slot *new, enum kvm_mr_change change) { + bool log_dirty_pages = new->flags & KVM_MEM_LOG_DIRTY_PAGES; + /* - * Nothing to do for RO slots or CREATE/MOVE/DELETE of a slot. - * See comments below. + * Update CPU dirty logging if dirty logging is being toggled. This + * applies to all operations. + */ + if ((old->flags ^ new->flags) & KVM_MEM_LOG_DIRTY_PAGES) + kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages); + + /* + * Nothing more to do for RO slots (which can't be dirtied and can't be + * made writable) or CREATE/MOVE/DELETE of a slot. + * + * For a memslot with dirty logging disabled: + * CREATE: No dirty mappings will already exist. + * MOVE/DELETE: The old mappings will already have been cleaned up by + * kvm_arch_flush_shadow_memslot() + * + * For a memslot with dirty logging enabled: + * CREATE: No shadow pages exist, thus nothing to write-protect + * and no dirty bits to clear. + * MOVE/DELETE: The old mappings will already have been cleaned up by + * kvm_arch_flush_shadow_memslot(). */ if ((change != KVM_MR_FLAGS_ONLY) || (new->flags & KVM_MEM_READONLY)) return; /* - * Dirty logging tracks sptes in 4k granularity, meaning that large - * sptes have to be split. If live migration is successful, the guest - * in the source machine will be destroyed and large sptes will be - * created in the destination. However, if the guest continues to run - * in the source machine (for example if live migration fails), small - * sptes will remain around and cause bad performance. - * - * Scan sptes if dirty logging has been stopped, dropping those - * which can be collapsed into a single large-page spte. Later - * page faults will create the large-page sptes. - * - * There is no need to do this in any of the following cases: - * CREATE: No dirty mappings will already exist. - * MOVE/DELETE: The old mappings will already have been cleaned up by - * kvm_arch_flush_shadow_memslot() + * READONLY and non-flags changes were filtered out above, and the only + * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty + * logging isn't being toggled on or off. */ - if ((old->flags & KVM_MEM_LOG_DIRTY_PAGES) && - !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + if (WARN_ON_ONCE(!((old->flags ^ new->flags) & KVM_MEM_LOG_DIRTY_PAGES))) + return; + + if (!log_dirty_pages) { + /* + * Dirty logging tracks sptes in 4k granularity, meaning that + * large sptes have to be split. If live migration succeeds, + * the guest in the source machine will be destroyed and large + * sptes will be created in the destination. However, if the + * guest continues to run in the source machine (for example if + * live migration fails), small sptes will remain around and + * cause bad performance. + * + * Scan sptes if dirty logging has been stopped, dropping those + * which can be collapsed into a single large-page spte. Later + * page faults will create the large-page sptes. + */ kvm_mmu_zap_collapsible_sptes(kvm, new); + } else { + /* By default, write-protect everything to log writes. */ + int level = PG_LEVEL_4K; - /* - * Enable or disable dirty logging for the slot. - * - * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of the old - * slot have been zapped so no dirty logging updates are needed for - * the old slot. - * For KVM_MR_CREATE and KVM_MR_MOVE, once the new slot is visible - * any mappings that might be created in it will consume the - * properties of the new slot and do not need to be updated here. - * - * When PML is enabled, the kvm_x86_ops dirty logging hooks are - * called to enable/disable dirty logging. - * - * When disabling dirty logging with PML enabled, the D-bit is set - * for sptes in the slot in order to prevent unnecessary GPA - * logging in the PML buffer (and potential PML buffer full VMEXIT). - * This guarantees leaving PML enabled for the guest's lifetime - * won't have any additional overhead from PML when the guest is - * running with dirty logging disabled. - * - * When enabling dirty logging, large sptes are write-protected - * so they can be split on first write. New large sptes cannot - * be created for this slot until the end of the logging. - * See the comments in fast_page_fault(). - * For small sptes, nothing is done if the dirty log is in the - * initial-all-set state. Otherwise, depending on whether pml - * is enabled the D-bit or the W-bit will be cleared. - */ - if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) { - if (kvm_x86_ops.slot_enable_log_dirty) { - static_call(kvm_x86_slot_enable_log_dirty)(kvm, new); - } else { - int level = - kvm_dirty_log_manual_protect_and_init_set(kvm) ? - PG_LEVEL_2M : PG_LEVEL_4K; + if (kvm_x86_ops.cpu_dirty_log_size) { + /* + * Clear all dirty bits, unless pages are treated as + * dirty from the get-go. + */ + if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) + kvm_mmu_slot_leaf_clear_dirty(kvm, new); + /* + * Write-protect large pages on write so that dirty + * logging happens at 4k granularity. No need to + * write-protect small SPTEs since write accesses are + * logged by the CPU via dirty bits. + */ + level = PG_LEVEL_2M; + } else if (kvm_dirty_log_manual_protect_and_init_set(kvm)) { /* * If we're with initial-all-set, we don't need * to write protect any small page because @@ -10825,10 +10857,9 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, * so that the page split can happen lazily on * the first write to the huge page. */ - kvm_mmu_slot_remove_write_access(kvm, new, level); + level = PG_LEVEL_2M; } - } else { - static_call_cond(kvm_x86_slot_disable_log_dirty)(kvm, new); + kvm_mmu_slot_remove_write_access(kvm, new, level); } } diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c index 404279563891..435630a6ec97 100644 --- a/arch/x86/lib/insn.c +++ b/arch/x86/lib/insn.c @@ -5,6 +5,7 @@ * Copyright (C) IBM Corporation, 2002, 2004, 2009 */ +#include #ifdef __KERNEL__ #include #else @@ -15,15 +16,28 @@ #include +#define leXX_to_cpu(t, r) \ +({ \ + __typeof__(t) v; \ + switch (sizeof(t)) { \ + case 4: v = le32_to_cpu(r); break; \ + case 2: v = le16_to_cpu(r); break; \ + case 1: v = r; break; \ + default: \ + BUILD_BUG(); break; \ + } \ + v; \ +}) + /* Verify next sizeof(t) bytes can be on the same instruction */ #define validate_next(t, insn, n) \ ((insn)->next_byte + sizeof(t) + n <= (insn)->end_kaddr) #define __get_next(t, insn) \ - ({ t r = *(t*)insn->next_byte; insn->next_byte += sizeof(t); r; }) + ({ t r = *(t*)insn->next_byte; insn->next_byte += sizeof(t); leXX_to_cpu(t, r); }) #define __peek_nbyte_next(t, insn, n) \ - ({ t r = *(t*)((insn)->next_byte + n); r; }) + ({ t r = *(t*)((insn)->next_byte + n); leXX_to_cpu(t, r); }) #define get_next(t, insn) \ ({ if (unlikely(!validate_next(t, insn, 0))) goto err_out; __get_next(t, insn); }) @@ -147,9 +161,9 @@ found: b = insn->prefixes.bytes[3]; for (i = 0; i < nb; i++) if (prefixes->bytes[i] == lb) - prefixes->bytes[i] = b; + insn_set_byte(prefixes, i, b); } - insn->prefixes.bytes[3] = lb; + insn_set_byte(&insn->prefixes, 3, lb); } /* Decode REX prefix */ @@ -157,8 +171,7 @@ found: b = peek_next(insn_byte_t, insn); attr = inat_get_opcode_attribute(b); if (inat_is_rex_prefix(attr)) { - insn->rex_prefix.value = b; - insn->rex_prefix.nbytes = 1; + insn_field_set(&insn->rex_prefix, b, 1); insn->next_byte++; if (X86_REX_W(b)) /* REX.W overrides opnd_size */ @@ -181,13 +194,13 @@ found: if (X86_MODRM_MOD(b2) != 3) goto vex_end; } - insn->vex_prefix.bytes[0] = b; - insn->vex_prefix.bytes[1] = b2; + insn_set_byte(&insn->vex_prefix, 0, b); + insn_set_byte(&insn->vex_prefix, 1, b2); if (inat_is_evex_prefix(attr)) { b2 = peek_nbyte_next(insn_byte_t, insn, 2); - insn->vex_prefix.bytes[2] = b2; + insn_set_byte(&insn->vex_prefix, 2, b2); b2 = peek_nbyte_next(insn_byte_t, insn, 3); - insn->vex_prefix.bytes[3] = b2; + insn_set_byte(&insn->vex_prefix, 3, b2); insn->vex_prefix.nbytes = 4; insn->next_byte += 4; if (insn->x86_64 && X86_VEX_W(b2)) @@ -195,7 +208,7 @@ found: insn->opnd_bytes = 8; } else if (inat_is_vex3_prefix(attr)) { b2 = peek_nbyte_next(insn_byte_t, insn, 2); - insn->vex_prefix.bytes[2] = b2; + insn_set_byte(&insn->vex_prefix, 2, b2); insn->vex_prefix.nbytes = 3; insn->next_byte += 3; if (insn->x86_64 && X86_VEX_W(b2)) @@ -207,7 +220,7 @@ found: * Makes it easier to decode vex.W, vex.vvvv, * vex.L and vex.pp. Masking with 0x7f sets vex.W == 0. */ - insn->vex_prefix.bytes[2] = b2 & 0x7f; + insn_set_byte(&insn->vex_prefix, 2, b2 & 0x7f); insn->vex_prefix.nbytes = 2; insn->next_byte += 2; } @@ -243,7 +256,7 @@ void insn_get_opcode(struct insn *insn) /* Get first opcode */ op = get_next(insn_byte_t, insn); - opcode->bytes[0] = op; + insn_set_byte(opcode, 0, op); opcode->nbytes = 1; /* Check if there is VEX prefix or not */ @@ -295,8 +308,7 @@ void insn_get_modrm(struct insn *insn) if (inat_has_modrm(insn->attr)) { mod = get_next(insn_byte_t, insn); - modrm->value = mod; - modrm->nbytes = 1; + insn_field_set(modrm, mod, 1); if (inat_is_group(insn->attr)) { pfx_id = insn_last_prefix_id(insn); insn->attr = inat_get_group_attribute(mod, pfx_id, @@ -334,7 +346,7 @@ int insn_rip_relative(struct insn *insn) * For rip-relative instructions, the mod field (top 2 bits) * is zero and the r/m field (bottom 3 bits) is 0x5. */ - return (modrm->nbytes && (modrm->value & 0xc7) == 0x5); + return (modrm->nbytes && (modrm->bytes[0] & 0xc7) == 0x5); } /** @@ -353,11 +365,11 @@ void insn_get_sib(struct insn *insn) if (!insn->modrm.got) insn_get_modrm(insn); if (insn->modrm.nbytes) { - modrm = (insn_byte_t)insn->modrm.value; + modrm = insn->modrm.bytes[0]; if (insn->addr_bytes != 2 && X86_MODRM_MOD(modrm) != 3 && X86_MODRM_RM(modrm) == 4) { - insn->sib.value = get_next(insn_byte_t, insn); - insn->sib.nbytes = 1; + insn_field_set(&insn->sib, + get_next(insn_byte_t, insn), 1); } } insn->sib.got = 1; @@ -407,19 +419,18 @@ void insn_get_displacement(struct insn *insn) if (mod == 3) goto out; if (mod == 1) { - insn->displacement.value = get_next(signed char, insn); - insn->displacement.nbytes = 1; + insn_field_set(&insn->displacement, + get_next(signed char, insn), 1); } else if (insn->addr_bytes == 2) { if ((mod == 0 && rm == 6) || mod == 2) { - insn->displacement.value = - get_next(short, insn); - insn->displacement.nbytes = 2; + insn_field_set(&insn->displacement, + get_next(short, insn), 2); } } else { if ((mod == 0 && rm == 5) || mod == 2 || (mod == 0 && base == 5)) { - insn->displacement.value = get_next(int, insn); - insn->displacement.nbytes = 4; + insn_field_set(&insn->displacement, + get_next(int, insn), 4); } } } @@ -435,18 +446,14 @@ static int __get_moffset(struct insn *insn) { switch (insn->addr_bytes) { case 2: - insn->moffset1.value = get_next(short, insn); - insn->moffset1.nbytes = 2; + insn_field_set(&insn->moffset1, get_next(short, insn), 2); break; case 4: - insn->moffset1.value = get_next(int, insn); - insn->moffset1.nbytes = 4; + insn_field_set(&insn->moffset1, get_next(int, insn), 4); break; case 8: - insn->moffset1.value = get_next(int, insn); - insn->moffset1.nbytes = 4; - insn->moffset2.value = get_next(int, insn); - insn->moffset2.nbytes = 4; + insn_field_set(&insn->moffset1, get_next(int, insn), 4); + insn_field_set(&insn->moffset2, get_next(int, insn), 4); break; default: /* opnd_bytes must be modified manually */ goto err_out; @@ -464,13 +471,11 @@ static int __get_immv32(struct insn *insn) { switch (insn->opnd_bytes) { case 2: - insn->immediate.value = get_next(short, insn); - insn->immediate.nbytes = 2; + insn_field_set(&insn->immediate, get_next(short, insn), 2); break; case 4: case 8: - insn->immediate.value = get_next(int, insn); - insn->immediate.nbytes = 4; + insn_field_set(&insn->immediate, get_next(int, insn), 4); break; default: /* opnd_bytes must be modified manually */ goto err_out; @@ -487,18 +492,15 @@ static int __get_immv(struct insn *insn) { switch (insn->opnd_bytes) { case 2: - insn->immediate1.value = get_next(short, insn); - insn->immediate1.nbytes = 2; + insn_field_set(&insn->immediate1, get_next(short, insn), 2); break; case 4: - insn->immediate1.value = get_next(int, insn); + insn_field_set(&insn->immediate1, get_next(int, insn), 4); insn->immediate1.nbytes = 4; break; case 8: - insn->immediate1.value = get_next(int, insn); - insn->immediate1.nbytes = 4; - insn->immediate2.value = get_next(int, insn); - insn->immediate2.nbytes = 4; + insn_field_set(&insn->immediate1, get_next(int, insn), 4); + insn_field_set(&insn->immediate2, get_next(int, insn), 4); break; default: /* opnd_bytes must be modified manually */ goto err_out; @@ -515,12 +517,10 @@ static int __get_immptr(struct insn *insn) { switch (insn->opnd_bytes) { case 2: - insn->immediate1.value = get_next(short, insn); - insn->immediate1.nbytes = 2; + insn_field_set(&insn->immediate1, get_next(short, insn), 2); break; case 4: - insn->immediate1.value = get_next(int, insn); - insn->immediate1.nbytes = 4; + insn_field_set(&insn->immediate1, get_next(int, insn), 4); break; case 8: /* ptr16:64 is not exist (no segment) */ @@ -528,8 +528,7 @@ static int __get_immptr(struct insn *insn) default: /* opnd_bytes must be modified manually */ goto err_out; } - insn->immediate2.value = get_next(unsigned short, insn); - insn->immediate2.nbytes = 2; + insn_field_set(&insn->immediate2, get_next(unsigned short, insn), 2); insn->immediate1.got = insn->immediate2.got = 1; return 1; @@ -565,22 +564,17 @@ void insn_get_immediate(struct insn *insn) switch (inat_immediate_size(insn->attr)) { case INAT_IMM_BYTE: - insn->immediate.value = get_next(signed char, insn); - insn->immediate.nbytes = 1; + insn_field_set(&insn->immediate, get_next(signed char, insn), 1); break; case INAT_IMM_WORD: - insn->immediate.value = get_next(short, insn); - insn->immediate.nbytes = 2; + insn_field_set(&insn->immediate, get_next(short, insn), 2); break; case INAT_IMM_DWORD: - insn->immediate.value = get_next(int, insn); - insn->immediate.nbytes = 4; + insn_field_set(&insn->immediate, get_next(int, insn), 4); break; case INAT_IMM_QWORD: - insn->immediate1.value = get_next(int, insn); - insn->immediate1.nbytes = 4; - insn->immediate2.value = get_next(int, insn); - insn->immediate2.nbytes = 4; + insn_field_set(&insn->immediate1, get_next(int, insn), 4); + insn_field_set(&insn->immediate2, get_next(int, insn), 4); break; case INAT_IMM_PTR: if (!__get_immptr(insn)) @@ -599,8 +593,7 @@ void insn_get_immediate(struct insn *insn) goto err_out; } if (inat_has_second_immediate(insn->attr)) { - insn->immediate2.value = get_next(signed char, insn); - insn->immediate2.nbytes = 1; + insn_field_set(&insn->immediate2, get_next(signed char, insn), 1); } done: insn->immediate.got = 1; diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index b4c43a9b1483..f6fb1d218dcc 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -28,7 +28,7 @@ SYM_FUNC_START_NOALIGN(__x86_retpoline_\reg) jmp .Lspec_trap_\@ .Ldo_rop_\@: mov %\reg, (%_ASM_SP) - UNWIND_HINT_RET_OFFSET + UNWIND_HINT_FUNC ret SYM_FUNC_END(__x86_retpoline_\reg) diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 525197381baa..a73347e2cdfc 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -9,6 +9,7 @@ #include /* oops_begin/end, ... */ #include /* search_exception_tables */ #include /* max_low_pfn */ +#include /* kfence_handle_page_fault */ #include /* NOKPROBE_SYMBOL, ... */ #include /* kmmio_handler, ... */ #include /* perf_sw_event */ @@ -680,6 +681,11 @@ page_fault_oops(struct pt_regs *regs, unsigned long error_code, if (IS_ENABLED(CONFIG_EFI)) efi_crash_gracefully_on_page_fault(address); + /* Only not-present faults should be handled by KFENCE. */ + if (!(error_code & X86_PF_PROT) && + kfence_handle_page_fault(address, error_code & X86_PF_WRITE, regs)) + return; + oops: /* * Oops. The kernel tried to access some bad page. We'll have to diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c index 8f665c352bf0..ca311aaa67b8 100644 --- a/arch/x86/mm/pat/memtype.c +++ b/arch/x86/mm/pat/memtype.c @@ -1164,12 +1164,14 @@ static void *memtype_seq_start(struct seq_file *seq, loff_t *pos) static void *memtype_seq_next(struct seq_file *seq, void *v, loff_t *pos) { + kfree(v); ++*pos; return memtype_get_idx(*pos); } static void memtype_seq_stop(struct seq_file *seq, void *v) { + kfree(v); } static int memtype_seq_show(struct seq_file *seq, void *v) @@ -1181,8 +1183,6 @@ static int memtype_seq_show(struct seq_file *seq, void *v) entry_print->end, cattr_name(entry_print->type)); - kfree(entry_print); - return 0; } diff --git a/arch/x86/pci/intel_mid_pci.c b/arch/x86/pci/intel_mid_pci.c index 95e2e6bd8d8c..8edd62206604 100644 --- a/arch/x86/pci/intel_mid_pci.c +++ b/arch/x86/pci/intel_mid_pci.c @@ -28,10 +28,12 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -140,6 +142,7 @@ static int pci_device_update_fixed(struct pci_bus *bus, unsigned int devfn, * type1_access_ok - check whether to use type 1 * @bus: bus number * @devfn: device & function in question + * @reg: configuration register offset * * If the bus is on a Lincroft chip and it exists, or is not on a Lincroft at * all, the we can go ahead with any reads & writes. If it's on a Lincroft, @@ -212,10 +215,17 @@ static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, where, size, value); } +static const struct x86_cpu_id intel_mid_cpu_ids[] = { + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, NULL), + {} +}; + static int intel_mid_pci_irq_enable(struct pci_dev *dev) { + const struct x86_cpu_id *id; struct irq_alloc_info info; bool polarity_low; + u16 model = 0; int ret; u8 gsi; @@ -228,8 +238,12 @@ static int intel_mid_pci_irq_enable(struct pci_dev *dev) return ret; } - switch (intel_mid_identify_cpu()) { - case INTEL_MID_CPU_CHIP_TANGIER: + id = x86_match_cpu(intel_mid_cpu_ids); + if (id) + model = id->model; + + switch (model) { + case INTEL_FAM6_ATOM_SILVERMONT_MID: polarity_low = false; /* Special treatment for IRQ0 */ diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c index 234998f196d4..de6bf0e7e8f8 100644 --- a/arch/x86/pci/mmconfig-shared.c +++ b/arch/x86/pci/mmconfig-shared.c @@ -11,9 +11,9 @@ * themselves. */ +#include #include #include -#include #include #include #include @@ -665,7 +665,7 @@ void __init pci_mmcfg_early_init(void) if (pci_mmcfg_check_hostbridge()) known_bridge = 1; else - acpi_sfi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); + acpi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); __pci_mmcfg_init(1); set_apei_filter(); @@ -683,7 +683,7 @@ void __init pci_mmcfg_late_init(void) /* MMCONFIG hasn't been enabled yet, try again */ if (pci_probe & PCI_PROBE_MASK & ~PCI_PROBE_MMCONF) { - acpi_sfi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); + acpi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); __pci_mmcfg_init(0); } } diff --git a/arch/x86/platform/Makefile b/arch/x86/platform/Makefile index b2f90a1a89f1..3ed03a2552d0 100644 --- a/arch/x86/platform/Makefile +++ b/arch/x86/platform/Makefile @@ -10,6 +10,5 @@ obj-y += intel-mid/ obj-y += intel-quark/ obj-y += olpc/ obj-y += scx200/ -obj-y += sfi/ obj-y += ts5500/ obj-y += uv/ diff --git a/arch/x86/platform/intel-mid/Makefile b/arch/x86/platform/intel-mid/Makefile index cc2549f0ccb1..ddfc08783fb8 100644 --- a/arch/x86/platform/intel-mid/Makefile +++ b/arch/x86/platform/intel-mid/Makefile @@ -1,7 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o intel_mid_vrtc.o pwr.o - -# SFI specific code -ifdef CONFIG_X86_INTEL_MID -obj-$(CONFIG_SFI) += sfi.o device_libs/ -endif +obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o pwr.o diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile deleted file mode 100644 index 480fed21cc7d..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# Family-Level Interface Shim (FLIS) -obj-$(subst m,y,$(CONFIG_PINCTRL_MERRIFIELD)) += platform_mrfld_pinctrl.o -# SDHCI Devices -obj-$(subst m,y,$(CONFIG_MMC_SDHCI_PCI)) += platform_mrfld_sd.o -# WiFi + BT -obj-$(subst m,y,$(CONFIG_BRCMFMAC_SDIO)) += platform_bcm43xx.o -obj-$(subst m,y,$(CONFIG_BT_HCIUART_BCM)) += platform_bt.o -# IPC Devices -obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic.o -obj-$(subst m,y,$(CONFIG_SND_MFLD_MACHINE)) += platform_msic_audio.o -obj-$(subst m,y,$(CONFIG_GPIO_MSIC)) += platform_msic_gpio.o -obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic_ocd.o -obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic_battery.o -obj-$(subst m,y,$(CONFIG_INTEL_MID_POWER_BUTTON)) += platform_msic_power_btn.o -obj-$(subst m,y,$(CONFIG_INTEL_MFLD_THERMAL)) += platform_msic_thermal.o -# SPI Devices -obj-$(subst m,y,$(CONFIG_SPI_SPIDEV)) += platform_mrfld_spidev.o -# I2C Devices -obj-$(subst m,y,$(CONFIG_SENSORS_EMC1403)) += platform_emc1403.o -obj-$(subst m,y,$(CONFIG_SENSORS_LIS3LV02D)) += platform_lis331.o -obj-$(subst m,y,$(CONFIG_MPU3050_I2C)) += platform_mpu3050.o -obj-$(subst m,y,$(CONFIG_INPUT_BMA150)) += platform_bma023.o -obj-$(subst m,y,$(CONFIG_DRM_MEDFIELD)) += platform_tc35876x.o -# I2C GPIO Expanders -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_max7315.o -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_pcal9555a.o -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_tca6416.o -# MISC Devices -obj-$(subst m,y,$(CONFIG_KEYBOARD_GPIO)) += platform_gpio_keys.o -obj-$(subst m,y,$(CONFIG_INTEL_MID_POWER_BUTTON)) += platform_mrfld_power_btn.o -obj-$(subst m,y,$(CONFIG_RTC_DRV_CMOS)) += platform_mrfld_rtc.o -obj-$(subst m,y,$(CONFIG_INTEL_MID_WATCHDOG)) += platform_mrfld_wdt.o diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c b/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c deleted file mode 100644 index 564c47c53f3a..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_bcm43xx.c: bcm43xx platform data initialization file - * - * (C) Copyright 2016 Intel Corporation - * Author: Andy Shevchenko - */ - -#include -#include -#include -#include -#include - -#include - -#define WLAN_SFI_GPIO_IRQ_NAME "WLAN-interrupt" -#define WLAN_SFI_GPIO_ENABLE_NAME "WLAN-enable" - -#define WLAN_DEV_NAME "0000:00:01.3" - -static struct regulator_consumer_supply bcm43xx_vmmc_supply = { - .dev_name = WLAN_DEV_NAME, - .supply = "vmmc", -}; - -static struct regulator_init_data bcm43xx_vmmc_data = { - .constraints = { - .valid_ops_mask = REGULATOR_CHANGE_STATUS, - }, - .num_consumer_supplies = 1, - .consumer_supplies = &bcm43xx_vmmc_supply, -}; - -static struct fixed_voltage_config bcm43xx_vmmc = { - .supply_name = "bcm43xx-vmmc-regulator", - /* - * Announce 2.0V here to be compatible with SDIO specification. The - * real voltage and signaling are still 1.8V. - */ - .microvolts = 2000000, /* 1.8V */ - .startup_delay = 250 * 1000, /* 250ms */ - .enabled_at_boot = 0, /* disabled at boot */ - .init_data = &bcm43xx_vmmc_data, -}; - -static struct platform_device bcm43xx_vmmc_regulator = { - .name = "reg-fixed-voltage", - .id = PLATFORM_DEVID_AUTO, - .dev = { - .platform_data = &bcm43xx_vmmc, - }, -}; - -static struct gpiod_lookup_table bcm43xx_vmmc_gpio_table = { - .dev_id = "reg-fixed-voltage.0", - .table = { - GPIO_LOOKUP("0000:00:0c.0", -1, NULL, GPIO_ACTIVE_LOW), - {} - }, -}; - -static int __init bcm43xx_regulator_register(void) -{ - struct gpiod_lookup_table *table = &bcm43xx_vmmc_gpio_table; - struct gpiod_lookup *lookup = table->table; - int ret; - - lookup[0].chip_hwnum = get_gpio_by_name(WLAN_SFI_GPIO_ENABLE_NAME); - gpiod_add_lookup_table(table); - - ret = platform_device_register(&bcm43xx_vmmc_regulator); - if (ret) { - pr_err("%s: vmmc regulator register failed\n", __func__); - return ret; - } - - return 0; -} - -static void __init *bcm43xx_platform_data(void *info) -{ - int ret; - - ret = bcm43xx_regulator_register(); - if (ret) - return NULL; - - pr_info("Using generic wifi platform data\n"); - - /* For now it's empty */ - return NULL; -} - -static const struct devs_id bcm43xx_clk_vmmc_dev_id __initconst = { - .name = "bcm43xx_clk_vmmc", - .type = SFI_DEV_TYPE_SD, - .get_platform_data = &bcm43xx_platform_data, -}; - -sfi_device(bcm43xx_clk_vmmc_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bma023.c b/arch/x86/platform/intel-mid/device_libs/platform_bma023.c deleted file mode 100644 index 32912a17f68e..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_bma023.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_bma023.c: bma023 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - */ - -#include - -static const struct devs_id bma023_dev_id __initconst = { - .name = "bma023", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, -}; - -sfi_device(bma023_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bt.c b/arch/x86/platform/intel-mid/device_libs/platform_bt.c deleted file mode 100644 index 2930b6e9473e..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_bt.c +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Bluetooth platform data initialization file - * - * (C) Copyright 2017 Intel Corporation - * Author: Andy Shevchenko - */ - -#include -#include -#include - -#include -#include -#include - -struct bt_sfi_data { - struct device *dev; - const char *name; - int (*setup)(struct bt_sfi_data *ddata); -}; - -static struct gpiod_lookup_table tng_bt_sfi_gpio_table = { - .dev_id = "hci_bcm", - .table = { - GPIO_LOOKUP("0000:00:0c.0", -1, "device-wakeup", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "shutdown", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "host-wakeup", GPIO_ACTIVE_HIGH), - { }, - }, -}; - -#define TNG_BT_SFI_GPIO_DEVICE_WAKEUP "bt_wakeup" -#define TNG_BT_SFI_GPIO_SHUTDOWN "BT-reset" -#define TNG_BT_SFI_GPIO_HOST_WAKEUP "bt_uart_enable" - -static int __init tng_bt_sfi_setup(struct bt_sfi_data *ddata) -{ - struct gpiod_lookup_table *table = &tng_bt_sfi_gpio_table; - struct gpiod_lookup *lookup = table->table; - struct pci_dev *pdev; - - /* Connected to /dev/ttyS0 */ - pdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(4, 1)); - if (!pdev) - return -ENODEV; - - ddata->dev = &pdev->dev; - ddata->name = table->dev_id; - - lookup[0].chip_hwnum = get_gpio_by_name(TNG_BT_SFI_GPIO_DEVICE_WAKEUP); - lookup[1].chip_hwnum = get_gpio_by_name(TNG_BT_SFI_GPIO_SHUTDOWN); - lookup[2].chip_hwnum = get_gpio_by_name(TNG_BT_SFI_GPIO_HOST_WAKEUP); - - gpiod_add_lookup_table(table); - return 0; -} - -static struct bt_sfi_data tng_bt_sfi_data __initdata = { - .setup = tng_bt_sfi_setup, -}; - -static const struct x86_cpu_id bt_sfi_cpu_ids[] = { - X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &tng_bt_sfi_data), - {} -}; - -static int __init bt_sfi_init(void) -{ - struct platform_device_info info; - struct platform_device *pdev; - const struct x86_cpu_id *id; - struct bt_sfi_data *ddata; - int ret; - - id = x86_match_cpu(bt_sfi_cpu_ids); - if (!id) - return -ENODEV; - - ddata = (struct bt_sfi_data *)id->driver_data; - if (!ddata) - return -ENODEV; - - ret = ddata->setup(ddata); - if (ret) - return ret; - - memset(&info, 0, sizeof(info)); - info.fwnode = ddata->dev->fwnode; - info.parent = ddata->dev; - info.name = ddata->name; - info.id = PLATFORM_DEVID_NONE; - - pdev = platform_device_register_full(&info); - if (IS_ERR(pdev)) - return PTR_ERR(pdev); - - dev_info(ddata->dev, "Registered Bluetooth device: %s\n", ddata->name); - return 0; -} -device_initcall(bt_sfi_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_emc1403.c b/arch/x86/platform/intel-mid/device_libs/platform_emc1403.c deleted file mode 100644 index a2508582a0b1..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_emc1403.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_emc1403.c: emc1403 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include - -static void __init *emc1403_platform_data(void *info) -{ - static short intr2nd_pdata; - struct i2c_board_info *i2c_info = info; - int intr = get_gpio_by_name("thermal_int"); - int intr2nd = get_gpio_by_name("thermal_alert"); - - if (intr < 0) - return NULL; - if (intr2nd < 0) - return NULL; - - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - intr2nd_pdata = intr2nd + INTEL_MID_IRQ_OFFSET; - - return &intr2nd_pdata; -} - -static const struct devs_id emc1403_dev_id __initconst = { - .name = "emc1403", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &emc1403_platform_data, -}; - -sfi_device(emc1403_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_gpio_keys.c b/arch/x86/platform/intel-mid/device_libs/platform_gpio_keys.c deleted file mode 100644 index d9435d2196a4..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_gpio_keys.c +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_gpio_keys.c: gpio_keys platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include - -#define DEVICE_NAME "gpio-keys" - -/* - * we will search these buttons in SFI GPIO table (by name) - * and register them dynamically. Please add all possible - * buttons here, we will shrink them if no GPIO found. - */ -static struct gpio_keys_button gpio_button[] = { - {KEY_POWER, -1, 1, "power_btn", EV_KEY, 0, 3000}, - {KEY_PROG1, -1, 1, "prog_btn1", EV_KEY, 0, 20}, - {KEY_PROG2, -1, 1, "prog_btn2", EV_KEY, 0, 20}, - {SW_LID, -1, 1, "lid_switch", EV_SW, 0, 20}, - {KEY_VOLUMEUP, -1, 1, "vol_up", EV_KEY, 0, 20}, - {KEY_VOLUMEDOWN, -1, 1, "vol_down", EV_KEY, 0, 20}, - {KEY_MUTE, -1, 1, "mute_enable", EV_KEY, 0, 20}, - {KEY_VOLUMEUP, -1, 1, "volume_up", EV_KEY, 0, 20}, - {KEY_VOLUMEDOWN, -1, 1, "volume_down", EV_KEY, 0, 20}, - {KEY_CAMERA, -1, 1, "camera_full", EV_KEY, 0, 20}, - {KEY_CAMERA_FOCUS, -1, 1, "camera_half", EV_KEY, 0, 20}, - {SW_KEYPAD_SLIDE, -1, 1, "MagSw1", EV_SW, 0, 20}, - {SW_KEYPAD_SLIDE, -1, 1, "MagSw2", EV_SW, 0, 20}, -}; - -static struct gpio_keys_platform_data gpio_keys = { - .buttons = gpio_button, - .rep = 1, - .nbuttons = -1, /* will fill it after search */ -}; - -static struct platform_device pb_device = { - .name = DEVICE_NAME, - .id = -1, - .dev = { - .platform_data = &gpio_keys, - }, -}; - -/* - * Shrink the non-existent buttons, register the gpio button - * device if there is some - */ -static int __init pb_keys_init(void) -{ - struct gpio_keys_button *gb = gpio_button; - int i, good = 0; - - for (i = 0; i < ARRAY_SIZE(gpio_button); i++) { - gb[i].gpio = get_gpio_by_name(gb[i].desc); - pr_debug("info[%2d]: name = %s, gpio = %d\n", i, gb[i].desc, - gb[i].gpio); - if (gb[i].gpio < 0) - continue; - - if (i != good) - gb[good] = gb[i]; - good++; - } - - if (good) { - gpio_keys.nbuttons = good; - return platform_device_register(&pb_device); - } - return 0; -} -late_initcall(pb_keys_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_lis331.c b/arch/x86/platform/intel-mid/device_libs/platform_lis331.c deleted file mode 100644 index a4485cd638c6..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_lis331.c +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_lis331.c: lis331 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include - -static void __init *lis331dl_platform_data(void *info) -{ - static short intr2nd_pdata; - struct i2c_board_info *i2c_info = info; - int intr = get_gpio_by_name("accel_int"); - int intr2nd = get_gpio_by_name("accel_2"); - - if (intr < 0) - return NULL; - if (intr2nd < 0) - return NULL; - - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - intr2nd_pdata = intr2nd + INTEL_MID_IRQ_OFFSET; - - return &intr2nd_pdata; -} - -static const struct devs_id lis331dl_dev_id __initconst = { - .name = "i2c_accel", - .type = SFI_DEV_TYPE_I2C, - .get_platform_data = &lis331dl_platform_data, -}; - -sfi_device(lis331dl_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_max7315.c b/arch/x86/platform/intel-mid/device_libs/platform_max7315.c deleted file mode 100644 index e9287c3184da..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_max7315.c +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_max7315.c: max7315 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include - -#define MAX7315_NUM 2 - -static void __init *max7315_platform_data(void *info) -{ - static struct pca953x_platform_data max7315_pdata[MAX7315_NUM]; - static int nr; - struct pca953x_platform_data *max7315 = &max7315_pdata[nr]; - struct i2c_board_info *i2c_info = info; - int gpio_base, intr; - char base_pin_name[SFI_NAME_LEN + 1]; - char intr_pin_name[SFI_NAME_LEN + 1]; - - if (nr == MAX7315_NUM) { - pr_err("too many max7315s, we only support %d\n", - MAX7315_NUM); - return NULL; - } - /* we have several max7315 on the board, we only need load several - * instances of the same pca953x driver to cover them - */ - strcpy(i2c_info->type, "max7315"); - if (nr++) { - snprintf(base_pin_name, sizeof(base_pin_name), - "max7315_%d_base", nr); - snprintf(intr_pin_name, sizeof(intr_pin_name), - "max7315_%d_int", nr); - } else { - strcpy(base_pin_name, "max7315_base"); - strcpy(intr_pin_name, "max7315_int"); - } - - gpio_base = get_gpio_by_name(base_pin_name); - intr = get_gpio_by_name(intr_pin_name); - - if (gpio_base < 0) - return NULL; - max7315->gpio_base = gpio_base; - if (intr != -1) { - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - max7315->irq_base = gpio_base + INTEL_MID_IRQ_OFFSET; - } else { - i2c_info->irq = -1; - max7315->irq_base = -1; - } - return max7315; -} - -static const struct devs_id max7315_dev_id __initconst = { - .name = "i2c_max7315", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &max7315_platform_data, -}; - -static const struct devs_id max7315_2_dev_id __initconst = { - .name = "i2c_max7315_2", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &max7315_platform_data, -}; - -sfi_device(max7315_dev_id); -sfi_device(max7315_2_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mpu3050.c b/arch/x86/platform/intel-mid/device_libs/platform_mpu3050.c deleted file mode 100644 index 28a182713934..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mpu3050.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_mpu3050.c: mpu3050 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include - -static void *mpu3050_platform_data(void *info) -{ - struct i2c_board_info *i2c_info = info; - int intr = get_gpio_by_name("mpu3050_int"); - - if (intr < 0) - return NULL; - - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - return NULL; -} - -static const struct devs_id mpu3050_dev_id __initconst = { - .name = "mpu3050", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &mpu3050_platform_data, -}; - -sfi_device(mpu3050_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c deleted file mode 100644 index 605e1f94ad89..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Merrifield FLIS platform device initialization file - * - * Copyright (C) 2016, Intel Corporation - * - * Author: Andy Shevchenko - */ - -#include -#include -#include - -#include - -#define FLIS_BASE_ADDR 0xff0c0000 -#define FLIS_LENGTH 0x8000 - -static struct resource mrfld_pinctrl_mmio_resource = { - .start = FLIS_BASE_ADDR, - .end = FLIS_BASE_ADDR + FLIS_LENGTH - 1, - .flags = IORESOURCE_MEM, -}; - -static struct platform_device mrfld_pinctrl_device = { - .name = "pinctrl-merrifield", - .id = PLATFORM_DEVID_NONE, - .resource = &mrfld_pinctrl_mmio_resource, - .num_resources = 1, -}; - -static int __init mrfld_pinctrl_init(void) -{ - if (intel_mid_identify_cpu() == INTEL_MID_CPU_CHIP_TANGIER) - return platform_device_register(&mrfld_pinctrl_device); - - return -ENODEV; -} -arch_initcall(mrfld_pinctrl_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_power_btn.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_power_btn.c deleted file mode 100644 index ec2afb41b34a..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_power_btn.c +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Merrifield power button support - * - * (C) Copyright 2017 Intel Corporation - * - * Author: Andy Shevchenko - */ - -#include -#include -#include -#include - -#include -#include - -static struct resource mrfld_power_btn_resources[] = { - { - .flags = IORESOURCE_IRQ, - }, -}; - -static struct platform_device mrfld_power_btn_dev = { - .name = "msic_power_btn", - .id = PLATFORM_DEVID_NONE, - .num_resources = ARRAY_SIZE(mrfld_power_btn_resources), - .resource = mrfld_power_btn_resources, -}; - -static int mrfld_power_btn_scu_status_change(struct notifier_block *nb, - unsigned long code, void *data) -{ - if (code == SCU_DOWN) { - platform_device_unregister(&mrfld_power_btn_dev); - return 0; - } - - return platform_device_register(&mrfld_power_btn_dev); -} - -static struct notifier_block mrfld_power_btn_scu_notifier = { - .notifier_call = mrfld_power_btn_scu_status_change, -}; - -static int __init register_mrfld_power_btn(void) -{ - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return -ENODEV; - - /* - * We need to be sure that the SCU IPC is ready before - * PMIC power button device can be registered: - */ - intel_scu_notifier_add(&mrfld_power_btn_scu_notifier); - - return 0; -} -arch_initcall(register_mrfld_power_btn); - -static void __init *mrfld_power_btn_platform_data(void *info) -{ - struct resource *res = mrfld_power_btn_resources; - struct sfi_device_table_entry *pentry = info; - - res->start = res->end = pentry->irq; - return NULL; -} - -static const struct devs_id mrfld_power_btn_dev_id __initconst = { - .name = "bcove_power_btn", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &mrfld_power_btn_platform_data, -}; - -sfi_device(mrfld_power_btn_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_rtc.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_rtc.c deleted file mode 100644 index 40e9808a9634..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_rtc.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Merrifield legacy RTC initialization file - * - * (C) Copyright 2017 Intel Corporation - * - * Author: Andy Shevchenko - */ - -#include - -#include -#include -#include -#include -#include - -static int __init mrfld_legacy_rtc_alloc_irq(void) -{ - struct irq_alloc_info info; - int ret; - - if (!x86_platform.legacy.rtc) - return -ENODEV; - - ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 1, 0); - ret = mp_map_gsi_to_irq(RTC_IRQ, IOAPIC_MAP_ALLOC, &info); - if (ret < 0) { - pr_info("Failed to allocate RTC interrupt. Disabling RTC\n"); - x86_platform.legacy.rtc = 0; - return ret; - } - - return 0; -} - -static int __init mrfld_legacy_rtc_init(void) -{ - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return -ENODEV; - - return mrfld_legacy_rtc_alloc_irq(); -} -arch_initcall(mrfld_legacy_rtc_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_sd.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_sd.c deleted file mode 100644 index fe3b7ff975f3..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_sd.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * SDHCI platform data initilisation file - * - * (C) Copyright 2016 Intel Corporation - * Author: Andy Shevchenko - */ - -#include -#include - -#include - -#include - -#define INTEL_MRFLD_SD 2 -#define INTEL_MRFLD_SD_CD_GPIO 77 - -static struct sdhci_pci_data mrfld_sdhci_pci_data = { - .rst_n_gpio = -EINVAL, - .cd_gpio = INTEL_MRFLD_SD_CD_GPIO, -}; - -static struct sdhci_pci_data * -mrfld_sdhci_pci_get_data(struct pci_dev *pdev, int slotno) -{ - unsigned int func = PCI_FUNC(pdev->devfn); - - if (func == INTEL_MRFLD_SD) - return &mrfld_sdhci_pci_data; - - return NULL; -} - -static int __init mrfld_sd_init(void) -{ - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return -ENODEV; - - sdhci_pci_get_data = mrfld_sdhci_pci_get_data; - return 0; -} -arch_initcall(mrfld_sd_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_spidev.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_spidev.c deleted file mode 100644 index b828f4fd40be..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_spidev.c +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * spidev platform data initialization file - * - * (C) Copyright 2014, 2016 Intel Corporation - * Authors: Andy Shevchenko - * Dan O'Donovan - */ - -#include -#include -#include -#include -#include - -#include - -#define MRFLD_SPI_DEFAULT_DMA_BURST 8 -#define MRFLD_SPI_DEFAULT_TIMEOUT 500 - -/* GPIO pin for spidev chipselect */ -#define MRFLD_SPIDEV_GPIO_CS 111 - -static struct pxa2xx_spi_chip spidev_spi_chip = { - .dma_burst_size = MRFLD_SPI_DEFAULT_DMA_BURST, - .timeout = MRFLD_SPI_DEFAULT_TIMEOUT, - .gpio_cs = MRFLD_SPIDEV_GPIO_CS, -}; - -static void __init *spidev_platform_data(void *info) -{ - struct spi_board_info *spi_info = info; - - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return ERR_PTR(-ENODEV); - - spi_info->mode = SPI_MODE_0; - spi_info->controller_data = &spidev_spi_chip; - - return NULL; -} - -static const struct devs_id spidev_dev_id __initconst = { - .name = "spidev", - .type = SFI_DEV_TYPE_SPI, - .delay = 0, - .get_platform_data = &spidev_platform_data, -}; - -sfi_device(spidev_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic.c b/arch/x86/platform/intel-mid/device_libs/platform_msic.c deleted file mode 100644 index b17783d0d4e7..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic.c: MSIC platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "platform_msic.h" - -struct intel_msic_platform_data msic_pdata; - -static struct resource msic_resources[] = { - { - .start = INTEL_MSIC_IRQ_PHYS_BASE, - .end = INTEL_MSIC_IRQ_PHYS_BASE + 64 - 1, - .flags = IORESOURCE_MEM, - }, -}; - -static struct platform_device msic_device = { - .name = "intel_msic", - .id = -1, - .dev = { - .platform_data = &msic_pdata, - }, - .num_resources = ARRAY_SIZE(msic_resources), - .resource = msic_resources, -}; - -static int msic_scu_status_change(struct notifier_block *nb, - unsigned long code, void *data) -{ - if (code == SCU_DOWN) { - platform_device_unregister(&msic_device); - return 0; - } - - return platform_device_register(&msic_device); -} - -static int __init msic_init(void) -{ - static struct notifier_block msic_scu_notifier = { - .notifier_call = msic_scu_status_change, - }; - - /* - * We need to be sure that the SCU IPC is ready before MSIC device - * can be registered. - */ - if (intel_mid_has_msic()) - intel_scu_notifier_add(&msic_scu_notifier); - - return 0; -} -arch_initcall(msic_init); - -/* - * msic_generic_platform_data - sets generic platform data for the block - * @info: pointer to the SFI device table entry for this block - * @block: MSIC block - * - * Function sets IRQ number from the SFI table entry for given device to - * the MSIC platform data. - */ -void *msic_generic_platform_data(void *info, enum intel_msic_block block) -{ - struct sfi_device_table_entry *entry = info; - - BUG_ON(block < 0 || block >= INTEL_MSIC_BLOCK_LAST); - msic_pdata.irq[block] = entry->irq; - - return NULL; -} diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic.h b/arch/x86/platform/intel-mid/device_libs/platform_msic.h deleted file mode 100644 index 91deb2e65b0e..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * platform_msic.h: MSIC platform data header file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ -#ifndef _PLATFORM_MSIC_H_ -#define _PLATFORM_MSIC_H_ - -extern struct intel_msic_platform_data msic_pdata; - -void *msic_generic_platform_data(void *info, enum intel_msic_block block); - -#endif diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_audio.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_audio.c deleted file mode 100644 index e765da78ad8c..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_audio.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_audio.c: MSIC audio platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void *msic_audio_platform_data(void *info) -{ - struct platform_device *pdev; - - pdev = platform_device_register_simple("sst-platform", -1, NULL, 0); - - if (IS_ERR(pdev)) { - pr_err("failed to create audio platform device\n"); - return NULL; - } - - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_AUDIO); -} - -static const struct devs_id msic_audio_dev_id __initconst = { - .name = "msic_audio", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_audio_platform_data, -}; - -sfi_device(msic_audio_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_battery.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_battery.c deleted file mode 100644 index f461f84903f8..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_battery.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_battery.c: MSIC battery platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_battery_platform_data(void *info) -{ - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_BATTERY); -} - -static const struct devs_id msic_battery_dev_id __initconst = { - .name = "msic_battery", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_battery_platform_data, -}; - -sfi_device(msic_battery_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_gpio.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_gpio.c deleted file mode 100644 index 71a7d6db3878..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_gpio.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_gpio.c: MSIC GPIO platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_gpio_platform_data(void *info) -{ - static struct intel_msic_gpio_pdata msic_gpio_pdata; - - int gpio = get_gpio_by_name("msic_gpio_base"); - - if (gpio < 0) - return NULL; - - msic_gpio_pdata.gpio_base = gpio; - msic_pdata.gpio = &msic_gpio_pdata; - - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_GPIO); -} - -static const struct devs_id msic_gpio_dev_id __initconst = { - .name = "msic_gpio", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_gpio_platform_data, -}; - -sfi_device(msic_gpio_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_ocd.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_ocd.c deleted file mode 100644 index 558c0d974430..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_ocd.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_ocd.c: MSIC OCD platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_ocd_platform_data(void *info) -{ - static struct intel_msic_ocd_pdata msic_ocd_pdata; - int gpio; - - gpio = get_gpio_by_name("ocd_gpio"); - - if (gpio < 0) - return NULL; - - msic_ocd_pdata.gpio = gpio; - msic_pdata.ocd = &msic_ocd_pdata; - - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_OCD); -} - -static const struct devs_id msic_ocd_dev_id __initconst = { - .name = "msic_ocd", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_ocd_platform_data, -}; - -sfi_device(msic_ocd_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_power_btn.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_power_btn.c deleted file mode 100644 index 3d3de2d59726..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_power_btn.c +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_power_btn.c: MSIC power btn platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_power_btn_platform_data(void *info) -{ - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_POWER_BTN); -} - -static const struct devs_id msic_power_btn_dev_id __initconst = { - .name = "msic_power_btn", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_power_btn_platform_data, -}; - -sfi_device(msic_power_btn_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_thermal.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_thermal.c deleted file mode 100644 index 4858da1d78c6..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_thermal.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_thermal.c: msic_thermal platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_thermal_platform_data(void *info) -{ - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_THERMAL); -} - -static const struct devs_id msic_thermal_dev_id __initconst = { - .name = "msic_thermal", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_thermal_platform_data, -}; - -sfi_device(msic_thermal_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c b/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c deleted file mode 100644 index 5609d8da3978..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * PCAL9555a platform data initialization file - * - * Copyright (C) 2016, Intel Corporation - * - * Authors: Andy Shevchenko - * Dan O'Donovan - */ - -#include -#include -#include -#include -#include - -#include - -#define PCAL9555A_NUM 4 - -static struct pca953x_platform_data pcal9555a_pdata[PCAL9555A_NUM]; -static int nr; - -static void __init *pcal9555a_platform_data(void *info) -{ - struct i2c_board_info *i2c_info = info; - char *type = i2c_info->type; - struct pca953x_platform_data *pcal9555a; - char base_pin_name[SFI_NAME_LEN + 1]; - char intr_pin_name[SFI_NAME_LEN + 1]; - int gpio_base, intr; - - snprintf(base_pin_name, sizeof(base_pin_name), "%s_base", type); - snprintf(intr_pin_name, sizeof(intr_pin_name), "%s_int", type); - - gpio_base = get_gpio_by_name(base_pin_name); - intr = get_gpio_by_name(intr_pin_name); - - /* Check if the SFI record valid */ - if (gpio_base == -1) - return NULL; - - if (nr >= PCAL9555A_NUM) { - pr_err("%s: Too many instances, only %d supported\n", __func__, - PCAL9555A_NUM); - return NULL; - } - - pcal9555a = &pcal9555a_pdata[nr++]; - pcal9555a->gpio_base = gpio_base; - - if (intr >= 0) { - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - pcal9555a->irq_base = gpio_base + INTEL_MID_IRQ_OFFSET; - } else { - i2c_info->irq = -1; - pcal9555a->irq_base = -1; - } - - strcpy(type, "pcal9555a"); - return pcal9555a; -} - -static const struct devs_id pcal9555a_1_dev_id __initconst = { - .name = "pcal9555a-1", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -static const struct devs_id pcal9555a_2_dev_id __initconst = { - .name = "pcal9555a-2", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -static const struct devs_id pcal9555a_3_dev_id __initconst = { - .name = "pcal9555a-3", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -static const struct devs_id pcal9555a_4_dev_id __initconst = { - .name = "pcal9555a-4", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -sfi_device(pcal9555a_1_dev_id); -sfi_device(pcal9555a_2_dev_id); -sfi_device(pcal9555a_3_dev_id); -sfi_device(pcal9555a_4_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_tc35876x.c b/arch/x86/platform/intel-mid/device_libs/platform_tc35876x.c deleted file mode 100644 index 139738bbdd36..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_tc35876x.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_tc35876x.c: tc35876x platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include - -static struct gpiod_lookup_table tc35876x_gpio_table = { - .dev_id = "i2c_disp_brig", - .table = { - GPIO_LOOKUP("0000:00:0c.0", -1, "bridge-reset", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "bl-en", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "vadd", GPIO_ACTIVE_HIGH), - { }, - }, -}; - -/*tc35876x DSI_LVDS bridge chip and panel platform data*/ -static void *tc35876x_platform_data(void *data) -{ - struct gpiod_lookup_table *table = &tc35876x_gpio_table; - struct gpiod_lookup *lookup = table->table; - - lookup[0].chip_hwnum = get_gpio_by_name("LCMB_RXEN"); - lookup[1].chip_hwnum = get_gpio_by_name("6S6P_BL_EN"); - lookup[2].chip_hwnum = get_gpio_by_name("EN_VREG_LCD_V3P3"); - gpiod_add_lookup_table(table); - - return NULL; -} - -static const struct devs_id tc35876x_dev_id __initconst = { - .name = "i2c_disp_brig", - .type = SFI_DEV_TYPE_I2C, - .get_platform_data = &tc35876x_platform_data, -}; - -sfi_device(tc35876x_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_tca6416.c b/arch/x86/platform/intel-mid/device_libs/platform_tca6416.c deleted file mode 100644 index e689d8f61059..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_tca6416.c +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_tca6416.c: tca6416 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include - -#define TCA6416_NAME "tca6416" -#define TCA6416_BASE "tca6416_base" -#define TCA6416_INTR "tca6416_int" - -static void *tca6416_platform_data(void *info) -{ - static struct pca953x_platform_data tca6416; - struct i2c_board_info *i2c_info = info; - int gpio_base, intr; - char base_pin_name[SFI_NAME_LEN + 1]; - char intr_pin_name[SFI_NAME_LEN + 1]; - - strcpy(i2c_info->type, TCA6416_NAME); - strcpy(base_pin_name, TCA6416_BASE); - strcpy(intr_pin_name, TCA6416_INTR); - - gpio_base = get_gpio_by_name(base_pin_name); - intr = get_gpio_by_name(intr_pin_name); - - if (gpio_base < 0) - return NULL; - tca6416.gpio_base = gpio_base; - if (intr >= 0) { - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - tca6416.irq_base = gpio_base + INTEL_MID_IRQ_OFFSET; - } else { - i2c_info->irq = -1; - tca6416.irq_base = -1; - } - return &tca6416; -} - -static const struct devs_id tca6416_dev_id __initconst = { - .name = "tca6416", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &tca6416_platform_data, -}; - -sfi_device(tca6416_dev_id); diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c index 780728161f7d..f4592dc7a1c1 100644 --- a/arch/x86/platform/intel-mid/intel-mid.c +++ b/arch/x86/platform/intel-mid/intel-mid.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * intel-mid.c: Intel MID platform setup code + * Intel MID platform setup code * - * (C) Copyright 2008, 2012 Intel Corporation + * (C) Copyright 2008, 2012, 2021 Intel Corporation * Author: Jacob Pan (jacob.jun.pan@intel.com) * Author: Sathyanarayanan Kuppuswamy */ @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -25,38 +24,13 @@ #include #include #include -#include #include #include #include -#include #include -/* - * the clockevent devices on Moorestown/Medfield can be APBT or LAPIC clock, - * cmdline option x86_intel_mid_timer can be used to override the configuration - * to prefer one or the other. - * at runtime, there are basically three timer configurations: - * 1. per cpu apbt clock only - * 2. per cpu always-on lapic clocks only, this is Penwell/Medfield only - * 3. per cpu lapic clock (C3STOP) and one apbt clock, with broadcast. - * - * by default (without cmdline option), platform code first detects cpu type - * to see if we are on lincroft or penwell, then set up both lapic or apbt - * clocks accordingly. - * i.e. by default, medfield uses configuration #2, moorestown uses #1. - * config #3 is supported but not recommended on medfield. - * - * rating and feature summary: - * lapic (with C3STOP) --------- 100 - * apbt (always-on) ------------ 110 - * lapic (always-on,ARAT) ------ 150 - */ - -enum intel_mid_timer_options intel_mid_timer_options; - -enum intel_mid_cpu_type __intel_mid_cpu_chip; -EXPORT_SYMBOL_GPL(__intel_mid_cpu_chip); +#define IPCMSG_COLD_OFF 0x80 /* Only for Tangier */ +#define IPCMSG_COLD_RESET 0xF1 static void intel_mid_power_off(void) { @@ -64,69 +38,32 @@ static void intel_mid_power_off(void) intel_mid_pwr_power_off(); /* Only for Tangier, the rest will ignore this command */ - intel_scu_ipc_simple_command(IPCMSG_COLD_OFF, 1); + intel_scu_ipc_dev_simple_command(NULL, IPCMSG_COLD_OFF, 1); }; static void intel_mid_reboot(void) { - intel_scu_ipc_simple_command(IPCMSG_COLD_RESET, 0); -} - -static void __init intel_mid_setup_bp_timer(void) -{ - apbt_time_init(); - setup_boot_APIC_clock(); + intel_scu_ipc_dev_simple_command(NULL, IPCMSG_COLD_RESET, 0); } static void __init intel_mid_time_init(void) { - sfi_table_parse(SFI_SIG_MTMR, NULL, NULL, sfi_parse_mtmr); - - switch (intel_mid_timer_options) { - case INTEL_MID_TIMER_APBT_ONLY: - break; - case INTEL_MID_TIMER_LAPIC_APBT: - /* Use apbt and local apic */ - x86_init.timers.setup_percpu_clockev = intel_mid_setup_bp_timer; - x86_cpuinit.setup_percpu_clockev = setup_secondary_APIC_clock; - return; - default: - if (!boot_cpu_has(X86_FEATURE_ARAT)) - break; - /* Lapic only, no apbt */ - x86_init.timers.setup_percpu_clockev = setup_boot_APIC_clock; - x86_cpuinit.setup_percpu_clockev = setup_secondary_APIC_clock; - return; - } - - x86_init.timers.setup_percpu_clockev = apbt_time_init; + /* Lapic only, no apbt */ + x86_init.timers.setup_percpu_clockev = setup_boot_APIC_clock; + x86_cpuinit.setup_percpu_clockev = setup_secondary_APIC_clock; } static void intel_mid_arch_setup(void) { - if (boot_cpu_data.x86 != 6) { - pr_err("Unknown Intel MID CPU (%d:%d), default to Penwell\n", - boot_cpu_data.x86, boot_cpu_data.x86_model); - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_PENWELL; - goto out; - } - switch (boot_cpu_data.x86_model) { - case 0x35: - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_CLOVERVIEW; - break; case 0x3C: case 0x4A: - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_TANGIER; x86_platform.legacy.rtc = 1; break; - case 0x27: default: - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_PENWELL; break; } -out: /* * Intel MID platforms are using explicitly defined regulators. * @@ -159,14 +96,11 @@ void __init x86_intel_mid_early_setup(void) x86_init.timers.timer_init = intel_mid_time_init; x86_init.timers.setup_percpu_clockev = x86_init_noop; - x86_init.timers.wallclock_init = intel_mid_rtc_init; x86_init.irqs.pre_vector_init = x86_init_noop; x86_init.oem.arch_setup = intel_mid_arch_setup; - x86_cpuinit.setup_percpu_clockev = apbt_setup_secondary_clock; - x86_platform.get_nmi_reason = intel_mid_get_nmi_reason; x86_init.pci.arch_init = intel_mid_pci_init; @@ -188,25 +122,3 @@ void __init x86_intel_mid_early_setup(void) x86_init.mpparse.get_smp_config = x86_init_uint_noop; set_bit(MP_BUS_ISA, mp_bus_not_pci); } - -/* - * if user does not want to use per CPU apb timer, just give it a lower rating - * than local apic timer and skip the late per cpu timer init. - */ -static inline int __init setup_x86_intel_mid_timer(char *arg) -{ - if (!arg) - return -EINVAL; - - if (strcmp("apbt_only", arg) == 0) - intel_mid_timer_options = INTEL_MID_TIMER_APBT_ONLY; - else if (strcmp("lapic_and_apbt", arg) == 0) - intel_mid_timer_options = INTEL_MID_TIMER_LAPIC_APBT; - else { - pr_warn("X86 INTEL_MID timer option %s not recognised use x86_intel_mid_timer=apbt_only or lapic_and_apbt\n", - arg); - return -EINVAL; - } - return 0; -} -__setup("x86_intel_mid_timer=", setup_x86_intel_mid_timer); diff --git a/arch/x86/platform/intel-mid/intel_mid_vrtc.c b/arch/x86/platform/intel-mid/intel_mid_vrtc.c deleted file mode 100644 index 2226da4f437a..000000000000 --- a/arch/x86/platform/intel-mid/intel_mid_vrtc.c +++ /dev/null @@ -1,173 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * intel_mid_vrtc.c: Driver for virtual RTC device on Intel MID platform - * - * (C) Copyright 2009 Intel Corporation - * - * Note: - * VRTC is emulated by system controller firmware, the real HW - * RTC is located in the PMIC device. SCU FW shadows PMIC RTC - * in a memory mapped IO space that is visible to the host IA - * processor. - * - * This driver is based on RTC CMOS driver. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -static unsigned char __iomem *vrtc_virt_base; - -unsigned char vrtc_cmos_read(unsigned char reg) -{ - unsigned char retval; - - /* vRTC's registers range from 0x0 to 0xD */ - if (reg > 0xd || !vrtc_virt_base) - return 0xff; - - lock_cmos_prefix(reg); - retval = __raw_readb(vrtc_virt_base + (reg << 2)); - lock_cmos_suffix(reg); - return retval; -} -EXPORT_SYMBOL_GPL(vrtc_cmos_read); - -void vrtc_cmos_write(unsigned char val, unsigned char reg) -{ - if (reg > 0xd || !vrtc_virt_base) - return; - - lock_cmos_prefix(reg); - __raw_writeb(val, vrtc_virt_base + (reg << 2)); - lock_cmos_suffix(reg); -} -EXPORT_SYMBOL_GPL(vrtc_cmos_write); - -void vrtc_get_time(struct timespec64 *now) -{ - u8 sec, min, hour, mday, mon; - unsigned long flags; - u32 year; - - spin_lock_irqsave(&rtc_lock, flags); - - while ((vrtc_cmos_read(RTC_FREQ_SELECT) & RTC_UIP)) - cpu_relax(); - - sec = vrtc_cmos_read(RTC_SECONDS); - min = vrtc_cmos_read(RTC_MINUTES); - hour = vrtc_cmos_read(RTC_HOURS); - mday = vrtc_cmos_read(RTC_DAY_OF_MONTH); - mon = vrtc_cmos_read(RTC_MONTH); - year = vrtc_cmos_read(RTC_YEAR); - - spin_unlock_irqrestore(&rtc_lock, flags); - - /* vRTC YEAR reg contains the offset to 1972 */ - year += 1972; - - pr_info("vRTC: sec: %d min: %d hour: %d day: %d " - "mon: %d year: %d\n", sec, min, hour, mday, mon, year); - - now->tv_sec = mktime64(year, mon, mday, hour, min, sec); - now->tv_nsec = 0; -} - -int vrtc_set_mmss(const struct timespec64 *now) -{ - unsigned long flags; - struct rtc_time tm; - int year; - int retval = 0; - - rtc_time64_to_tm(now->tv_sec, &tm); - if (!rtc_valid_tm(&tm) && tm.tm_year >= 72) { - /* - * tm.year is the number of years since 1900, and the - * vrtc need the years since 1972. - */ - year = tm.tm_year - 72; - spin_lock_irqsave(&rtc_lock, flags); - vrtc_cmos_write(year, RTC_YEAR); - vrtc_cmos_write(tm.tm_mon, RTC_MONTH); - vrtc_cmos_write(tm.tm_mday, RTC_DAY_OF_MONTH); - vrtc_cmos_write(tm.tm_hour, RTC_HOURS); - vrtc_cmos_write(tm.tm_min, RTC_MINUTES); - vrtc_cmos_write(tm.tm_sec, RTC_SECONDS); - spin_unlock_irqrestore(&rtc_lock, flags); - } else { - pr_err("%s: Invalid vRTC value: write of %llx to vRTC failed\n", - __func__, (s64)now->tv_sec); - retval = -EINVAL; - } - return retval; -} - -void __init intel_mid_rtc_init(void) -{ - unsigned long vrtc_paddr; - - sfi_table_parse(SFI_SIG_MRTC, NULL, NULL, sfi_parse_mrtc); - - vrtc_paddr = sfi_mrtc_array[0].phys_addr; - if (!sfi_mrtc_num || !vrtc_paddr) - return; - - vrtc_virt_base = (void __iomem *)set_fixmap_offset_nocache(FIX_LNW_VRTC, - vrtc_paddr); - x86_platform.get_wallclock = vrtc_get_time; - x86_platform.set_wallclock = vrtc_set_mmss; -} - -/* - * The Moorestown platform has a memory mapped virtual RTC device that emulates - * the programming interface of the RTC. - */ - -static struct resource vrtc_resources[] = { - [0] = { - .flags = IORESOURCE_MEM, - }, - [1] = { - .flags = IORESOURCE_IRQ, - } -}; - -static struct platform_device vrtc_device = { - .name = "rtc_mrst", - .id = -1, - .resource = vrtc_resources, - .num_resources = ARRAY_SIZE(vrtc_resources), -}; - -/* Register the RTC device if appropriate */ -static int __init intel_mid_device_create(void) -{ - /* No Moorestown, no device */ - if (!intel_mid_identify_cpu()) - return -ENODEV; - /* No timer, no device */ - if (!sfi_mrtc_num) - return -ENODEV; - - /* iomem resource */ - vrtc_resources[0].start = sfi_mrtc_array[0].phys_addr; - vrtc_resources[0].end = sfi_mrtc_array[0].phys_addr + - MRST_VRTC_MAP_SZ; - /* irq resource */ - vrtc_resources[1].start = sfi_mrtc_array[0].irq; - vrtc_resources[1].end = sfi_mrtc_array[0].irq; - - return platform_device_register(&vrtc_device); -} -device_initcall(intel_mid_device_create); diff --git a/arch/x86/platform/intel-mid/sfi.c b/arch/x86/platform/intel-mid/sfi.c deleted file mode 100644 index 30bd5714a3d4..000000000000 --- a/arch/x86/platform/intel-mid/sfi.c +++ /dev/null @@ -1,543 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * intel_mid_sfi.c: Intel MID SFI initialization code - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SFI_SIG_OEM0 "OEM0" -#define MAX_IPCDEVS 24 -#define MAX_SCU_SPI 24 -#define MAX_SCU_I2C 24 - -static struct platform_device *ipc_devs[MAX_IPCDEVS]; -static struct spi_board_info *spi_devs[MAX_SCU_SPI]; -static struct i2c_board_info *i2c_devs[MAX_SCU_I2C]; -static struct sfi_gpio_table_entry *gpio_table; -static struct sfi_timer_table_entry sfi_mtimer_array[SFI_MTMR_MAX_NUM]; -static int ipc_next_dev; -static int spi_next_dev; -static int i2c_next_dev; -static int i2c_bus[MAX_SCU_I2C]; -static int gpio_num_entry; -static u32 sfi_mtimer_usage[SFI_MTMR_MAX_NUM]; -int sfi_mrtc_num; -int sfi_mtimer_num; - -struct sfi_rtc_table_entry sfi_mrtc_array[SFI_MRTC_MAX]; -EXPORT_SYMBOL_GPL(sfi_mrtc_array); - -struct blocking_notifier_head intel_scu_notifier = - BLOCKING_NOTIFIER_INIT(intel_scu_notifier); -EXPORT_SYMBOL_GPL(intel_scu_notifier); - -#define intel_mid_sfi_get_pdata(dev, priv) \ - ((dev)->get_platform_data ? (dev)->get_platform_data(priv) : NULL) - -/* parse all the mtimer info to a static mtimer array */ -int __init sfi_parse_mtmr(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_timer_table_entry *pentry; - struct mpc_intsrc mp_irq; - int totallen; - - sb = (struct sfi_table_simple *)table; - if (!sfi_mtimer_num) { - sfi_mtimer_num = SFI_GET_NUM_ENTRIES(sb, - struct sfi_timer_table_entry); - pentry = (struct sfi_timer_table_entry *) sb->pentry; - totallen = sfi_mtimer_num * sizeof(*pentry); - memcpy(sfi_mtimer_array, pentry, totallen); - } - - pr_debug("SFI MTIMER info (num = %d):\n", sfi_mtimer_num); - pentry = sfi_mtimer_array; - for (totallen = 0; totallen < sfi_mtimer_num; totallen++, pentry++) { - pr_debug("timer[%d]: paddr = 0x%08x, freq = %dHz, irq = %d\n", - totallen, (u32)pentry->phys_addr, - pentry->freq_hz, pentry->irq); - mp_irq.type = MP_INTSRC; - mp_irq.irqtype = mp_INT; - mp_irq.irqflag = MP_IRQTRIG_EDGE | MP_IRQPOL_ACTIVE_HIGH; - mp_irq.srcbus = MP_BUS_ISA; - mp_irq.srcbusirq = pentry->irq; /* IRQ */ - mp_irq.dstapic = MP_APIC_ALL; - mp_irq.dstirq = pentry->irq; - mp_save_irq(&mp_irq); - mp_map_gsi_to_irq(pentry->irq, IOAPIC_MAP_ALLOC, NULL); - } - - return 0; -} - -struct sfi_timer_table_entry *sfi_get_mtmr(int hint) -{ - int i; - if (hint < sfi_mtimer_num) { - if (!sfi_mtimer_usage[hint]) { - pr_debug("hint taken for timer %d irq %d\n", - hint, sfi_mtimer_array[hint].irq); - sfi_mtimer_usage[hint] = 1; - return &sfi_mtimer_array[hint]; - } - } - /* take the first timer available */ - for (i = 0; i < sfi_mtimer_num;) { - if (!sfi_mtimer_usage[i]) { - sfi_mtimer_usage[i] = 1; - return &sfi_mtimer_array[i]; - } - i++; - } - return NULL; -} - -void sfi_free_mtmr(struct sfi_timer_table_entry *mtmr) -{ - int i; - for (i = 0; i < sfi_mtimer_num;) { - if (mtmr->irq == sfi_mtimer_array[i].irq) { - sfi_mtimer_usage[i] = 0; - return; - } - i++; - } -} - -/* parse all the mrtc info to a global mrtc array */ -int __init sfi_parse_mrtc(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_rtc_table_entry *pentry; - struct mpc_intsrc mp_irq; - - int totallen; - - sb = (struct sfi_table_simple *)table; - if (!sfi_mrtc_num) { - sfi_mrtc_num = SFI_GET_NUM_ENTRIES(sb, - struct sfi_rtc_table_entry); - pentry = (struct sfi_rtc_table_entry *)sb->pentry; - totallen = sfi_mrtc_num * sizeof(*pentry); - memcpy(sfi_mrtc_array, pentry, totallen); - } - - pr_debug("SFI RTC info (num = %d):\n", sfi_mrtc_num); - pentry = sfi_mrtc_array; - for (totallen = 0; totallen < sfi_mrtc_num; totallen++, pentry++) { - pr_debug("RTC[%d]: paddr = 0x%08x, irq = %d\n", - totallen, (u32)pentry->phys_addr, pentry->irq); - mp_irq.type = MP_INTSRC; - mp_irq.irqtype = mp_INT; - mp_irq.irqflag = MP_IRQTRIG_LEVEL | MP_IRQPOL_ACTIVE_LOW; - mp_irq.srcbus = MP_BUS_ISA; - mp_irq.srcbusirq = pentry->irq; /* IRQ */ - mp_irq.dstapic = MP_APIC_ALL; - mp_irq.dstirq = pentry->irq; - mp_save_irq(&mp_irq); - mp_map_gsi_to_irq(pentry->irq, IOAPIC_MAP_ALLOC, NULL); - } - return 0; -} - - -/* - * Parsing GPIO table first, since the DEVS table will need this table - * to map the pin name to the actual pin. - */ -static int __init sfi_parse_gpio(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_gpio_table_entry *pentry; - int num, i; - - if (gpio_table) - return 0; - sb = (struct sfi_table_simple *)table; - num = SFI_GET_NUM_ENTRIES(sb, struct sfi_gpio_table_entry); - pentry = (struct sfi_gpio_table_entry *)sb->pentry; - - gpio_table = kmemdup(pentry, num * sizeof(*pentry), GFP_KERNEL); - if (!gpio_table) - return -1; - gpio_num_entry = num; - - pr_debug("GPIO pin info:\n"); - for (i = 0; i < num; i++, pentry++) - pr_debug("info[%2d]: controller = %16.16s, pin_name = %16.16s," - " pin = %d\n", i, - pentry->controller_name, - pentry->pin_name, - pentry->pin_no); - return 0; -} - -int get_gpio_by_name(const char *name) -{ - struct sfi_gpio_table_entry *pentry = gpio_table; - int i; - - if (!pentry) - return -1; - for (i = 0; i < gpio_num_entry; i++, pentry++) { - if (!strncmp(name, pentry->pin_name, SFI_NAME_LEN)) - return pentry->pin_no; - } - return -EINVAL; -} - -static void __init intel_scu_ipc_device_register(struct platform_device *pdev) -{ - if (ipc_next_dev == MAX_IPCDEVS) - pr_err("too many SCU IPC devices"); - else - ipc_devs[ipc_next_dev++] = pdev; -} - -static void __init intel_scu_spi_device_register(struct spi_board_info *sdev) -{ - struct spi_board_info *new_dev; - - if (spi_next_dev == MAX_SCU_SPI) { - pr_err("too many SCU SPI devices"); - return; - } - - new_dev = kzalloc(sizeof(*sdev), GFP_KERNEL); - if (!new_dev) { - pr_err("failed to alloc mem for delayed spi dev %s\n", - sdev->modalias); - return; - } - *new_dev = *sdev; - - spi_devs[spi_next_dev++] = new_dev; -} - -static void __init intel_scu_i2c_device_register(int bus, - struct i2c_board_info *idev) -{ - struct i2c_board_info *new_dev; - - if (i2c_next_dev == MAX_SCU_I2C) { - pr_err("too many SCU I2C devices"); - return; - } - - new_dev = kzalloc(sizeof(*idev), GFP_KERNEL); - if (!new_dev) { - pr_err("failed to alloc mem for delayed i2c dev %s\n", - idev->type); - return; - } - *new_dev = *idev; - - i2c_bus[i2c_next_dev] = bus; - i2c_devs[i2c_next_dev++] = new_dev; -} - -/* Called by IPC driver */ -void intel_scu_devices_create(void) -{ - int i; - - for (i = 0; i < ipc_next_dev; i++) - platform_device_add(ipc_devs[i]); - - for (i = 0; i < spi_next_dev; i++) - spi_register_board_info(spi_devs[i], 1); - - for (i = 0; i < i2c_next_dev; i++) { - struct i2c_adapter *adapter; - struct i2c_client *client; - - adapter = i2c_get_adapter(i2c_bus[i]); - if (adapter) { - client = i2c_new_client_device(adapter, i2c_devs[i]); - if (IS_ERR(client)) - pr_err("can't create i2c device %s\n", - i2c_devs[i]->type); - } else - i2c_register_board_info(i2c_bus[i], i2c_devs[i], 1); - } - intel_scu_notifier_post(SCU_AVAILABLE, NULL); -} -EXPORT_SYMBOL_GPL(intel_scu_devices_create); - -/* Called by IPC driver */ -void intel_scu_devices_destroy(void) -{ - int i; - - intel_scu_notifier_post(SCU_DOWN, NULL); - - for (i = 0; i < ipc_next_dev; i++) - platform_device_del(ipc_devs[i]); -} -EXPORT_SYMBOL_GPL(intel_scu_devices_destroy); - -static void __init install_irq_resource(struct platform_device *pdev, int irq) -{ - /* Single threaded */ - static struct resource res __initdata = { - .name = "IRQ", - .flags = IORESOURCE_IRQ, - }; - res.start = irq; - platform_device_add_resources(pdev, &res, 1); -} - -static void __init sfi_handle_ipc_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct platform_device *pdev; - void *pdata = NULL; - - pr_debug("IPC bus, name = %16.16s, irq = 0x%2x\n", - pentry->name, pentry->irq); - - /* - * We need to call platform init of IPC devices to fill misc_pdata - * structure. It will be used in msic_init for initialization. - */ - pdata = intel_mid_sfi_get_pdata(dev, pentry); - if (IS_ERR(pdata)) - return; - - /* - * On Medfield the platform device creation is handled by the MSIC - * MFD driver so we don't need to do it here. - */ - if (dev->msic && intel_mid_has_msic()) - return; - - pdev = platform_device_alloc(pentry->name, 0); - if (pdev == NULL) { - pr_err("out of memory for SFI platform device '%s'.\n", - pentry->name); - return; - } - install_irq_resource(pdev, pentry->irq); - - pdev->dev.platform_data = pdata; - if (dev->delay) - intel_scu_ipc_device_register(pdev); - else - platform_device_add(pdev); -} - -static void __init sfi_handle_spi_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct spi_board_info spi_info; - void *pdata = NULL; - - memset(&spi_info, 0, sizeof(spi_info)); - strncpy(spi_info.modalias, pentry->name, SFI_NAME_LEN); - spi_info.irq = ((pentry->irq == (u8)0xff) ? 0 : pentry->irq); - spi_info.bus_num = pentry->host_num; - spi_info.chip_select = pentry->addr; - spi_info.max_speed_hz = pentry->max_freq; - pr_debug("SPI bus=%d, name=%16.16s, irq=0x%2x, max_freq=%d, cs=%d\n", - spi_info.bus_num, - spi_info.modalias, - spi_info.irq, - spi_info.max_speed_hz, - spi_info.chip_select); - - pdata = intel_mid_sfi_get_pdata(dev, &spi_info); - if (IS_ERR(pdata)) - return; - - spi_info.platform_data = pdata; - if (dev->delay) - intel_scu_spi_device_register(&spi_info); - else - spi_register_board_info(&spi_info, 1); -} - -static void __init sfi_handle_i2c_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct i2c_board_info i2c_info; - void *pdata = NULL; - - memset(&i2c_info, 0, sizeof(i2c_info)); - strncpy(i2c_info.type, pentry->name, SFI_NAME_LEN); - i2c_info.irq = ((pentry->irq == (u8)0xff) ? 0 : pentry->irq); - i2c_info.addr = pentry->addr; - pr_debug("I2C bus = %d, name = %16.16s, irq = 0x%2x, addr = 0x%x\n", - pentry->host_num, - i2c_info.type, - i2c_info.irq, - i2c_info.addr); - pdata = intel_mid_sfi_get_pdata(dev, &i2c_info); - i2c_info.platform_data = pdata; - if (IS_ERR(pdata)) - return; - - if (dev->delay) - intel_scu_i2c_device_register(pentry->host_num, &i2c_info); - else - i2c_register_board_info(pentry->host_num, &i2c_info, 1); -} - -static void __init sfi_handle_sd_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct mid_sd_board_info sd_info; - void *pdata; - - memset(&sd_info, 0, sizeof(sd_info)); - strncpy(sd_info.name, pentry->name, SFI_NAME_LEN); - sd_info.bus_num = pentry->host_num; - sd_info.max_clk = pentry->max_freq; - sd_info.addr = pentry->addr; - pr_debug("SD bus = %d, name = %16.16s, max_clk = %d, addr = 0x%x\n", - sd_info.bus_num, - sd_info.name, - sd_info.max_clk, - sd_info.addr); - pdata = intel_mid_sfi_get_pdata(dev, &sd_info); - if (IS_ERR(pdata)) - return; - - /* Nothing we can do with this for now */ - sd_info.platform_data = pdata; - - pr_debug("Successfully registered %16.16s", sd_info.name); -} - -extern struct devs_id *const __x86_intel_mid_dev_start[], - *const __x86_intel_mid_dev_end[]; - -static struct devs_id __init *get_device_id(u8 type, char *name) -{ - struct devs_id *const *dev_table; - - for (dev_table = __x86_intel_mid_dev_start; - dev_table < __x86_intel_mid_dev_end; dev_table++) { - struct devs_id *dev = *dev_table; - if (dev->type == type && - !strncmp(dev->name, name, SFI_NAME_LEN)) { - return dev; - } - } - - return NULL; -} - -static int __init sfi_parse_devs(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_device_table_entry *pentry; - struct devs_id *dev = NULL; - int num, i, ret; - int polarity; - struct irq_alloc_info info; - - sb = (struct sfi_table_simple *)table; - num = SFI_GET_NUM_ENTRIES(sb, struct sfi_device_table_entry); - pentry = (struct sfi_device_table_entry *)sb->pentry; - - for (i = 0; i < num; i++, pentry++) { - int irq = pentry->irq; - - if (irq != (u8)0xff) { /* native RTE case */ - /* these SPI2 devices are not exposed to system as PCI - * devices, but they have separate RTE entry in IOAPIC - * so we have to enable them one by one here - */ - if (intel_mid_identify_cpu() == - INTEL_MID_CPU_CHIP_TANGIER) { - if (!strncmp(pentry->name, "r69001-ts-i2c", 13)) - /* active low */ - polarity = 1; - else if (!strncmp(pentry->name, - "synaptics_3202", 14)) - /* active low */ - polarity = 1; - else if (irq == 41) - /* fast_int_1 */ - polarity = 1; - else - /* active high */ - polarity = 0; - } else { - /* PNW and CLV go with active low */ - polarity = 1; - } - - ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 1, polarity); - ret = mp_map_gsi_to_irq(irq, IOAPIC_MAP_ALLOC, &info); - WARN_ON(ret < 0); - } - - dev = get_device_id(pentry->type, pentry->name); - - if (!dev) - continue; - - switch (pentry->type) { - case SFI_DEV_TYPE_IPC: - sfi_handle_ipc_dev(pentry, dev); - break; - case SFI_DEV_TYPE_SPI: - sfi_handle_spi_dev(pentry, dev); - break; - case SFI_DEV_TYPE_I2C: - sfi_handle_i2c_dev(pentry, dev); - break; - case SFI_DEV_TYPE_SD: - sfi_handle_sd_dev(pentry, dev); - break; - case SFI_DEV_TYPE_UART: - case SFI_DEV_TYPE_HSI: - default: - break; - } - } - return 0; -} - -static int __init intel_mid_platform_init(void) -{ - sfi_table_parse(SFI_SIG_GPIO, NULL, NULL, sfi_parse_gpio); - sfi_table_parse(SFI_SIG_DEVS, NULL, NULL, sfi_parse_devs); - return 0; -} -arch_initcall(intel_mid_platform_init); diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index 43b4d864817e..d2ccadc247e6 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -16,6 +16,7 @@ #include #include #include +#include #include __HEAD @@ -105,6 +106,7 @@ SYM_CODE_START_LOCAL(pvh_start_xen) /* startup_64 expects boot_params in %rsi. */ mov $_pa(pvh_bootparams), %rsi mov $_pa(startup_64), %rax + ANNOTATE_RETPOLINE_SAFE jmp *%rax #else /* CONFIG_X86_64 */ diff --git a/arch/x86/platform/sfi/sfi.c b/arch/x86/platform/sfi/sfi.c deleted file mode 100644 index 6259563760f9..000000000000 --- a/arch/x86/platform/sfi/sfi.c +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sfi.c - x86 architecture SFI support. - * - * Copyright (c) 2009, Intel Corporation. - */ - -#define KMSG_COMPONENT "SFI" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#ifdef CONFIG_X86_LOCAL_APIC -static unsigned long sfi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE; - -/* All CPUs enumerated by SFI must be present and enabled */ -static void __init mp_sfi_register_lapic(u8 id) -{ - if (MAX_LOCAL_APIC - id <= 0) { - pr_warn("Processor #%d invalid (max %d)\n", id, MAX_LOCAL_APIC); - return; - } - - pr_info("registering lapic[%d]\n", id); - - generic_processor_info(id, GET_APIC_VERSION(apic_read(APIC_LVR))); -} - -static int __init sfi_parse_cpus(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_cpu_table_entry *pentry; - int i; - int cpu_num; - - sb = (struct sfi_table_simple *)table; - cpu_num = SFI_GET_NUM_ENTRIES(sb, struct sfi_cpu_table_entry); - pentry = (struct sfi_cpu_table_entry *)sb->pentry; - - for (i = 0; i < cpu_num; i++) { - mp_sfi_register_lapic(pentry->apic_id); - pentry++; - } - - smp_found_config = 1; - return 0; -} -#endif /* CONFIG_X86_LOCAL_APIC */ - -#ifdef CONFIG_X86_IO_APIC - -static int __init sfi_parse_ioapic(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_apic_table_entry *pentry; - int i, num; - struct ioapic_domain_cfg cfg = { - .type = IOAPIC_DOMAIN_STRICT, - .ops = &mp_ioapic_irqdomain_ops, - }; - - sb = (struct sfi_table_simple *)table; - num = SFI_GET_NUM_ENTRIES(sb, struct sfi_apic_table_entry); - pentry = (struct sfi_apic_table_entry *)sb->pentry; - - for (i = 0; i < num; i++) { - mp_register_ioapic(i, pentry->phys_addr, gsi_top, &cfg); - pentry++; - } - - WARN(pic_mode, KERN_WARNING - "SFI: pic_mod shouldn't be 1 when IOAPIC table is present\n"); - pic_mode = 0; - return 0; -} -#endif /* CONFIG_X86_IO_APIC */ - -/* - * sfi_platform_init(): register lapics & io-apics - */ -int __init sfi_platform_init(void) -{ -#ifdef CONFIG_X86_LOCAL_APIC - register_lapic_address(sfi_lapic_addr); - sfi_table_parse(SFI_SIG_CPUS, NULL, NULL, sfi_parse_cpus); -#endif -#ifdef CONFIG_X86_IO_APIC - sfi_table_parse(SFI_SIG_APIC, NULL, NULL, sfi_parse_ioapic); -#endif - return 0; -} diff --git a/arch/x86/power/Makefile b/arch/x86/power/Makefile index 6907b523e856..379777572bc9 100644 --- a/arch/x86/power/Makefile +++ b/arch/x86/power/Makefile @@ -1,9 +1,12 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD_hibernate_asm_$(BITS).o := y # __restore_processor_state() restores %gs after S3 resume and so should not # itself be stack-protected CFLAGS_cpu.o := -fno-stack-protector +# Clang may incorrectly inline functions with stack protector enabled into +# __restore_processor_state(): https://bugs.llvm.org/show_bug.cgi?id=47479 +CFLAGS_REMOVE_cpu.o := $(CC_FLAGS_LTO) + obj-$(CONFIG_PM_SLEEP) += cpu.o obj-$(CONFIG_HIBERNATION) += hibernate_$(BITS).o hibernate_asm_$(BITS).o hibernate.o diff --git a/arch/x86/power/hibernate_asm_64.S b/arch/x86/power/hibernate_asm_64.S index 7918b8415f13..d9bed596d849 100644 --- a/arch/x86/power/hibernate_asm_64.S +++ b/arch/x86/power/hibernate_asm_64.S @@ -21,84 +21,7 @@ #include #include #include - -SYM_FUNC_START(swsusp_arch_suspend) - movq $saved_context, %rax - movq %rsp, pt_regs_sp(%rax) - movq %rbp, pt_regs_bp(%rax) - movq %rsi, pt_regs_si(%rax) - movq %rdi, pt_regs_di(%rax) - movq %rbx, pt_regs_bx(%rax) - movq %rcx, pt_regs_cx(%rax) - movq %rdx, pt_regs_dx(%rax) - movq %r8, pt_regs_r8(%rax) - movq %r9, pt_regs_r9(%rax) - movq %r10, pt_regs_r10(%rax) - movq %r11, pt_regs_r11(%rax) - movq %r12, pt_regs_r12(%rax) - movq %r13, pt_regs_r13(%rax) - movq %r14, pt_regs_r14(%rax) - movq %r15, pt_regs_r15(%rax) - pushfq - popq pt_regs_flags(%rax) - - /* save cr3 */ - movq %cr3, %rax - movq %rax, restore_cr3(%rip) - - FRAME_BEGIN - call swsusp_save - FRAME_END - ret -SYM_FUNC_END(swsusp_arch_suspend) - -SYM_CODE_START(restore_image) - /* prepare to jump to the image kernel */ - movq restore_jump_address(%rip), %r8 - movq restore_cr3(%rip), %r9 - - /* prepare to switch to temporary page tables */ - movq temp_pgt(%rip), %rax - movq mmu_cr4_features(%rip), %rbx - - /* prepare to copy image data to their original locations */ - movq restore_pblist(%rip), %rdx - - /* jump to relocated restore code */ - movq relocated_restore_code(%rip), %rcx - jmpq *%rcx -SYM_CODE_END(restore_image) - - /* code below has been relocated to a safe page */ -SYM_CODE_START(core_restore_code) - /* switch to temporary page tables */ - movq %rax, %cr3 - /* flush TLB */ - movq %rbx, %rcx - andq $~(X86_CR4_PGE), %rcx - movq %rcx, %cr4; # turn off PGE - movq %cr3, %rcx; # flush TLB - movq %rcx, %cr3; - movq %rbx, %cr4; # turn PGE back on -.Lloop: - testq %rdx, %rdx - jz .Ldone - - /* get addresses from the pbe and copy the page */ - movq pbe_address(%rdx), %rsi - movq pbe_orig_address(%rdx), %rdi - movq $(PAGE_SIZE >> 3), %rcx - rep - movsq - - /* progress to the next pbe */ - movq pbe_next(%rdx), %rdx - jmp .Lloop - -.Ldone: - /* jump to the restore_registers address from the image header */ - jmpq *%r8 -SYM_CODE_END(core_restore_code) +#include /* code below belongs to the image kernel */ .align PAGE_SIZE @@ -145,3 +68,83 @@ SYM_FUNC_START(restore_registers) ret SYM_FUNC_END(restore_registers) + +SYM_FUNC_START(swsusp_arch_suspend) + movq $saved_context, %rax + movq %rsp, pt_regs_sp(%rax) + movq %rbp, pt_regs_bp(%rax) + movq %rsi, pt_regs_si(%rax) + movq %rdi, pt_regs_di(%rax) + movq %rbx, pt_regs_bx(%rax) + movq %rcx, pt_regs_cx(%rax) + movq %rdx, pt_regs_dx(%rax) + movq %r8, pt_regs_r8(%rax) + movq %r9, pt_regs_r9(%rax) + movq %r10, pt_regs_r10(%rax) + movq %r11, pt_regs_r11(%rax) + movq %r12, pt_regs_r12(%rax) + movq %r13, pt_regs_r13(%rax) + movq %r14, pt_regs_r14(%rax) + movq %r15, pt_regs_r15(%rax) + pushfq + popq pt_regs_flags(%rax) + + /* save cr3 */ + movq %cr3, %rax + movq %rax, restore_cr3(%rip) + + FRAME_BEGIN + call swsusp_save + FRAME_END + ret +SYM_FUNC_END(swsusp_arch_suspend) + +SYM_FUNC_START(restore_image) + /* prepare to jump to the image kernel */ + movq restore_jump_address(%rip), %r8 + movq restore_cr3(%rip), %r9 + + /* prepare to switch to temporary page tables */ + movq temp_pgt(%rip), %rax + movq mmu_cr4_features(%rip), %rbx + + /* prepare to copy image data to their original locations */ + movq restore_pblist(%rip), %rdx + + /* jump to relocated restore code */ + movq relocated_restore_code(%rip), %rcx + ANNOTATE_RETPOLINE_SAFE + jmpq *%rcx +SYM_FUNC_END(restore_image) + + /* code below has been relocated to a safe page */ +SYM_FUNC_START(core_restore_code) + /* switch to temporary page tables */ + movq %rax, %cr3 + /* flush TLB */ + movq %rbx, %rcx + andq $~(X86_CR4_PGE), %rcx + movq %rcx, %cr4; # turn off PGE + movq %cr3, %rcx; # flush TLB + movq %rcx, %cr3; + movq %rbx, %cr4; # turn PGE back on +.Lloop: + testq %rdx, %rdx + jz .Ldone + + /* get addresses from the pbe and copy the page */ + movq pbe_address(%rdx), %rsi + movq pbe_orig_address(%rdx), %rdi + movq $(PAGE_SIZE >> 3), %rcx + rep + movsq + + /* progress to the next pbe */ + movq pbe_next(%rdx), %rdx + jmp .Lloop + +.Ldone: + /* jump to the restore_registers address from the image header */ + ANNOTATE_RETPOLINE_SAFE + jmpq *%r8 +SYM_FUNC_END(core_restore_code) diff --git a/arch/x86/tools/Makefile b/arch/x86/tools/Makefile index 55b1ab378974..bddfc9a46645 100644 --- a/arch/x86/tools/Makefile +++ b/arch/x86/tools/Makefile @@ -29,14 +29,14 @@ posttest: $(obj)/insn_decoder_test vmlinux $(obj)/insn_sanity hostprogs += insn_decoder_test insn_sanity # -I needed for generated C source and C source which in the kernel tree. -HOSTCFLAGS_insn_decoder_test.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/uapi/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/uapi/ +HOSTCFLAGS_insn_decoder_test.o := -Wall -I$(srctree)/tools/arch/x86/lib/ -I$(srctree)/tools/arch/x86/include/ -I$(objtree)/arch/x86/lib/ -HOSTCFLAGS_insn_sanity.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/ +HOSTCFLAGS_insn_sanity.o := -Wall -I$(srctree)/tools/arch/x86/lib/ -I$(srctree)/tools/arch/x86/include/ -I$(objtree)/arch/x86/lib/ # Dependencies are also needed. -$(obj)/insn_decoder_test.o: $(srctree)/arch/x86/lib/insn.c $(srctree)/arch/x86/lib/inat.c $(srctree)/arch/x86/include/asm/inat_types.h $(srctree)/arch/x86/include/asm/inat.h $(srctree)/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c +$(obj)/insn_decoder_test.o: $(srctree)/tools/arch/x86/lib/insn.c $(srctree)/tools/arch/x86/lib/inat.c $(srctree)/tools/arch/x86/include/asm/inat_types.h $(srctree)/tools/arch/x86/include/asm/inat.h $(srctree)/tools/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c -$(obj)/insn_sanity.o: $(srctree)/arch/x86/lib/insn.c $(srctree)/arch/x86/lib/inat.c $(srctree)/arch/x86/include/asm/inat_types.h $(srctree)/arch/x86/include/asm/inat.h $(srctree)/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c +$(obj)/insn_sanity.o: $(srctree)/tools/arch/x86/lib/insn.c $(srctree)/tools/arch/x86/lib/inat.c $(srctree)/tools/arch/x86/include/asm/inat_types.h $(srctree)/tools/arch/x86/include/asm/inat.h $(srctree)/tools/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c HOST_EXTRACFLAGS += -I$(srctree)/tools/include hostprogs += relocs diff --git a/arch/x86/tools/insn_sanity.c b/arch/x86/tools/insn_sanity.c index 185ceba9d289..c6a0000ae635 100644 --- a/arch/x86/tools/insn_sanity.c +++ b/arch/x86/tools/insn_sanity.c @@ -14,10 +14,6 @@ #include #include #include - -#define unlikely(cond) (cond) -#define ARRAY_SIZE(a) (sizeof(a)/sizeof(a[0])) - #include #include #include diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c index 1c3a1962cade..04c5a44b9682 100644 --- a/arch/x86/tools/relocs.c +++ b/arch/x86/tools/relocs.c @@ -61,8 +61,8 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = { "(__iommu_table|__apicdrivers|__smp_locks)(|_end)|" "__(start|end)_pci_.*|" "__(start|end)_builtin_fw|" - "__(start|stop)___ksymtab(|_gpl|_unused|_unused_gpl|_gpl_future)|" - "__(start|stop)___kcrctab(|_gpl|_unused|_unused_gpl|_gpl_future)|" + "__(start|stop)___ksymtab(|_gpl)|" + "__(start|stop)___kcrctab(|_gpl)|" "__(start|stop)___param|" "__(start|stop)___modver|" "__(start|stop)___bug_table|" diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile index fc5c5ba4aacb..40b5779fce21 100644 --- a/arch/x86/xen/Makefile +++ b/arch/x86/xen/Makefile @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD_xen-asm.o := y ifdef CONFIG_FUNCTION_TRACER # Do not profile debug and lowlevel utilities diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S index 02f31341e435..1e626444712b 100644 --- a/arch/x86/xen/xen-asm.S +++ b/arch/x86/xen/xen-asm.S @@ -14,6 +14,7 @@ #include #include #include +#include #include @@ -118,6 +119,7 @@ SYM_FUNC_END(xen_read_cr2_direct); .macro xen_pv_trap name SYM_CODE_START(xen_\name) + UNWIND_HINT_EMPTY pop %rcx pop %r11 jmp \name @@ -157,6 +159,7 @@ xen_pv_trap asm_exc_xen_hypervisor_callback SYM_CODE_START(xen_early_idt_handler_array) i = 0 .rept NUM_EXCEPTION_VECTORS + UNWIND_HINT_EMPTY pop %rcx pop %r11 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE @@ -183,6 +186,7 @@ hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 * rsp->rax } */ SYM_CODE_START(xen_iret) + UNWIND_HINT_EMPTY pushq $0 jmp hypercall_iret SYM_CODE_END(xen_iret) @@ -203,7 +207,8 @@ SYM_CODE_END(xen_iret) */ /* Normal 64-bit system call target */ -SYM_FUNC_START(xen_syscall_target) +SYM_CODE_START(xen_syscall_target) + UNWIND_HINT_EMPTY popq %rcx popq %r11 @@ -216,12 +221,13 @@ SYM_FUNC_START(xen_syscall_target) movq $__USER_CS, 1*8(%rsp) jmp entry_SYSCALL_64_after_hwframe -SYM_FUNC_END(xen_syscall_target) +SYM_CODE_END(xen_syscall_target) #ifdef CONFIG_IA32_EMULATION /* 32-bit compat syscall target */ -SYM_FUNC_START(xen_syscall32_target) +SYM_CODE_START(xen_syscall32_target) + UNWIND_HINT_EMPTY popq %rcx popq %r11 @@ -234,10 +240,11 @@ SYM_FUNC_START(xen_syscall32_target) movq $__USER32_CS, 1*8(%rsp) jmp entry_SYSCALL_compat_after_hwframe -SYM_FUNC_END(xen_syscall32_target) +SYM_CODE_END(xen_syscall32_target) /* 32-bit compat sysenter target */ -SYM_FUNC_START(xen_sysenter_target) +SYM_CODE_START(xen_sysenter_target) + UNWIND_HINT_EMPTY /* * NB: Xen is polite and clears TF from EFLAGS for us. This means * that we don't need to guard against single step exceptions here. @@ -254,17 +261,18 @@ SYM_FUNC_START(xen_sysenter_target) movq $__USER32_CS, 1*8(%rsp) jmp entry_SYSENTER_compat_after_hwframe -SYM_FUNC_END(xen_sysenter_target) +SYM_CODE_END(xen_sysenter_target) #else /* !CONFIG_IA32_EMULATION */ -SYM_FUNC_START_ALIAS(xen_syscall32_target) -SYM_FUNC_START(xen_sysenter_target) +SYM_CODE_START(xen_syscall32_target) +SYM_CODE_START(xen_sysenter_target) + UNWIND_HINT_EMPTY lea 16(%rsp), %rsp /* strip %rcx, %r11 */ mov $-ENOSYS, %rax pushq $0 jmp hypercall_iret -SYM_FUNC_END(xen_sysenter_target) -SYM_FUNC_END_ALIAS(xen_syscall32_target) +SYM_CODE_END(xen_sysenter_target) +SYM_CODE_END(xen_syscall32_target) #endif /* CONFIG_IA32_EMULATION */ diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S index 2d7c8f34f56c..cb6538ae2fe0 100644 --- a/arch/x86/xen/xen-head.S +++ b/arch/x86/xen/xen-head.S @@ -68,8 +68,9 @@ SYM_CODE_END(asm_cpu_bringup_and_idle) .balign PAGE_SIZE SYM_CODE_START(hypercall_page) .rept (PAGE_SIZE / 32) - UNWIND_HINT_EMPTY - .skip 32 + UNWIND_HINT_FUNC + .skip 31, 0x90 + ret .endr #define HYPERCALL(n) \ diff --git a/arch/xtensa/kernel/process.c b/arch/xtensa/kernel/process.c index 397a7de56377..9534ef515d74 100644 --- a/arch/xtensa/kernel/process.c +++ b/arch/xtensa/kernel/process.c @@ -217,7 +217,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp_thread_fn, p->thread.sp = (unsigned long)childregs; - if (!(p->flags & PF_KTHREAD)) { + if (!(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { struct pt_regs *regs = current_pt_regs(); unsigned long usp = usp_thread_fn ? usp_thread_fn : regs->areg[1]; diff --git a/arch/xtensa/kernel/syscalls/Makefile b/arch/xtensa/kernel/syscalls/Makefile index 659faefdcb1d..285aaba832d9 100644 --- a/arch/xtensa/kernel/syscalls/Makefile +++ b/arch/xtensa/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,18 +21,19 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_abi_$(basetarget))' \ '$(systbl_offset_$(basetarget))' -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/xtensa/kernel/syscalls/syscall.tbl b/arch/xtensa/kernel/syscalls/syscall.tbl index 46116a28eeed..365a9b849224 100644 --- a/arch/xtensa/kernel/syscalls/syscall.tbl +++ b/arch/xtensa/kernel/syscalls/syscall.tbl @@ -412,3 +412,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index b398dde53af9..ec482e6641ff 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -125,6 +125,8 @@ #include #include +#include + #include "blk.h" #include "blk-mq.h" #include "blk-mq-tag.h" @@ -5621,7 +5623,7 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, spin_unlock_irq(&bfqd->lock); - blk_mq_sched_request_inserted(rq); + trace_block_rq_insert(rq); spin_lock_irq(&bfqd->lock); bfqq = bfq_init_rq(rq); diff --git a/block/blk-core.c b/block/blk-core.c index 5e752840b41a..fc60ff208497 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -59,6 +59,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap); EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete); EXPORT_TRACEPOINT_SYMBOL_GPL(block_split); EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug); +EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_insert); DEFINE_IDA(blk_queue_ida); diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index e8327c50d7c9..c176b7af56a7 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -80,6 +80,7 @@ static struct blk_crypto_keyslot { static struct blk_keyslot_manager blk_crypto_ksm; static struct workqueue_struct *blk_crypto_wq; static mempool_t *blk_crypto_bounce_page_pool; +static struct bio_set crypto_bio_split; /* * This is the key we set when evicting a keyslot. This *should* be the all 0's @@ -224,7 +225,8 @@ static bool blk_crypto_split_bio_if_needed(struct bio **bio_ptr) if (num_sectors < bio_sectors(bio)) { struct bio *split_bio; - split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL); + split_bio = bio_split(bio, num_sectors, GFP_NOIO, + &crypto_bio_split); if (!split_bio) { bio->bi_status = BLK_STS_RESOURCE; return false; @@ -538,9 +540,13 @@ static int blk_crypto_fallback_init(void) prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); - err = blk_ksm_init(&blk_crypto_ksm, blk_crypto_num_keyslots); + err = bioset_init(&crypto_bio_split, 64, 0, 0); if (err) goto out; + + err = blk_ksm_init(&blk_crypto_ksm, blk_crypto_num_keyslots); + if (err) + goto fail_free_bioset; err = -ENOMEM; blk_crypto_ksm.ksm_ll_ops = blk_crypto_ksm_ll_ops; @@ -591,6 +597,8 @@ fail_free_wq: destroy_workqueue(blk_crypto_wq); fail_free_ksm: blk_ksm_destroy(&blk_crypto_ksm); +fail_free_bioset: + bioset_exit(&crypto_bio_split); out: return err; } diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 09fcb18fa778..c5bdaafffa29 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -409,3 +409,4 @@ int blk_crypto_evict_key(struct request_queue *q, */ return blk_crypto_fallback_evict_key(key); } +EXPORT_SYMBOL_GPL(blk_crypto_evict_key); diff --git a/block/blk-map.c b/block/blk-map.c index 21630dccac62..369e204d14d0 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -150,9 +150,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, bmd->is_our_pages = !map_data; bmd->is_null_mapped = (map_data && map_data->null_mapped); - nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); - if (nr_pages > BIO_MAX_PAGES) - nr_pages = BIO_MAX_PAGES; + nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); ret = -ENOMEM; bio = bio_kmalloc(gfp_mask, nr_pages); diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c index deff4e826e23..ddb65e9e6fd9 100644 --- a/block/blk-mq-sched.c +++ b/block/blk-mq-sched.c @@ -384,12 +384,6 @@ bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq) } EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge); -void blk_mq_sched_request_inserted(struct request *rq) -{ - trace_block_rq_insert(rq); -} -EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted); - static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx, bool has_sched, struct request *rq) diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h index 0476360f05f1..5b18ab915c65 100644 --- a/block/blk-mq-sched.h +++ b/block/blk-mq-sched.h @@ -7,7 +7,6 @@ void blk_mq_sched_assign_ioc(struct request *rq); -void blk_mq_sched_request_inserted(struct request *rq); bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs, struct request **merged_request); bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio, diff --git a/block/blk-mq.c b/block/blk-mq.c index f21d922ecfaf..d4d7c1caa439 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -41,7 +41,7 @@ #include "blk-mq-sched.h" #include "blk-rq-qos.h" -static DEFINE_PER_CPU(struct list_head, blk_cpu_done); +static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); static void blk_mq_poll_stats_start(struct request_queue *q); static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); @@ -567,80 +567,29 @@ void blk_mq_end_request(struct request *rq, blk_status_t error) } EXPORT_SYMBOL(blk_mq_end_request); -/* - * Softirq action handler - move entries to local list and loop over them - * while passing them to the queue registered handler. - */ -static __latent_entropy void blk_done_softirq(struct softirq_action *h) +static void blk_complete_reqs(struct llist_head *list) { - struct list_head *cpu_list, local_list; + struct llist_node *entry = llist_reverse_order(llist_del_all(list)); + struct request *rq, *next; - local_irq_disable(); - cpu_list = this_cpu_ptr(&blk_cpu_done); - list_replace_init(cpu_list, &local_list); - local_irq_enable(); - - while (!list_empty(&local_list)) { - struct request *rq; - - rq = list_entry(local_list.next, struct request, ipi_list); - list_del_init(&rq->ipi_list); + llist_for_each_entry_safe(rq, next, entry, ipi_list) rq->q->mq_ops->complete(rq); - } } -static void blk_mq_trigger_softirq(struct request *rq) +static __latent_entropy void blk_done_softirq(struct softirq_action *h) { - struct list_head *list; - unsigned long flags; - - local_irq_save(flags); - list = this_cpu_ptr(&blk_cpu_done); - list_add_tail(&rq->ipi_list, list); - - /* - * If the list only contains our just added request, signal a raise of - * the softirq. If there are already entries there, someone already - * raised the irq but it hasn't run yet. - */ - if (list->next == &rq->ipi_list) - raise_softirq_irqoff(BLOCK_SOFTIRQ); - local_irq_restore(flags); + blk_complete_reqs(this_cpu_ptr(&blk_cpu_done)); } static int blk_softirq_cpu_dead(unsigned int cpu) { - /* - * If a CPU goes away, splice its entries to the current CPU - * and trigger a run of the softirq - */ - local_irq_disable(); - list_splice_init(&per_cpu(blk_cpu_done, cpu), - this_cpu_ptr(&blk_cpu_done)); - raise_softirq_irqoff(BLOCK_SOFTIRQ); - local_irq_enable(); - + blk_complete_reqs(&per_cpu(blk_cpu_done, cpu)); return 0; } - static void __blk_mq_complete_request_remote(void *data) { - struct request *rq = data; - - /* - * For most of single queue controllers, there is only one irq vector - * for handling I/O completion, and the only irq's affinity is set - * to all possible CPUs. On most of ARCHs, this affinity means the irq - * is handled on one specific CPU. - * - * So complete I/O requests in softirq context in case of single queue - * devices to avoid degrading I/O performance due to irqsoff latency. - */ - if (rq->q->nr_hw_queues == 1) - blk_mq_trigger_softirq(rq); - else - rq->q->mq_ops->complete(rq); + __raise_softirq_irqoff(BLOCK_SOFTIRQ); } static inline bool blk_mq_complete_need_ipi(struct request *rq) @@ -669,6 +618,30 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq) return cpu_online(rq->mq_ctx->cpu); } +static void blk_mq_complete_send_ipi(struct request *rq) +{ + struct llist_head *list; + unsigned int cpu; + + cpu = rq->mq_ctx->cpu; + list = &per_cpu(blk_cpu_done, cpu); + if (llist_add(&rq->ipi_list, list)) { + INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq); + smp_call_function_single_async(cpu, &rq->csd); + } +} + +static void blk_mq_raise_softirq(struct request *rq) +{ + struct llist_head *list; + + preempt_disable(); + list = this_cpu_ptr(&blk_cpu_done); + if (llist_add(&rq->ipi_list, list)) + raise_softirq(BLOCK_SOFTIRQ); + preempt_enable(); +} + bool blk_mq_complete_request_remote(struct request *rq) { WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); @@ -681,15 +654,15 @@ bool blk_mq_complete_request_remote(struct request *rq) return false; if (blk_mq_complete_need_ipi(rq)) { - INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq); - smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd); - } else { - if (rq->q->nr_hw_queues > 1) - return false; - blk_mq_trigger_softirq(rq); + blk_mq_complete_send_ipi(rq); + return true; } - return true; + if (rq->q->nr_hw_queues == 1) { + blk_mq_raise_softirq(rq); + return true; + } + return false; } EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote); @@ -3957,7 +3930,7 @@ static int __init blk_mq_init(void) int i; for_each_possible_cpu(i) - INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i)); + init_llist_head(&per_cpu(blk_cpu_done, i)); open_softirq(BLOCK_SOFTIRQ, blk_done_softirq); cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD, diff --git a/block/blk-pm.h b/block/blk-pm.h index a2283cc9f716..8a5a0d4b357f 100644 --- a/block/blk-pm.h +++ b/block/blk-pm.h @@ -21,31 +21,6 @@ static inline void blk_pm_mark_last_busy(struct request *rq) if (rq->q->dev && !(rq->rq_flags & RQF_PM)) pm_runtime_mark_last_busy(rq->q->dev); } - -static inline void blk_pm_requeue_request(struct request *rq) -{ - lockdep_assert_held(&rq->q->queue_lock); - - if (rq->q->dev && !(rq->rq_flags & RQF_PM)) - rq->q->nr_pending--; -} - -static inline void blk_pm_add_request(struct request_queue *q, - struct request *rq) -{ - lockdep_assert_held(&q->queue_lock); - - if (q->dev && !(rq->rq_flags & RQF_PM)) - q->nr_pending++; -} - -static inline void blk_pm_put_request(struct request *rq) -{ - lockdep_assert_held(&rq->q->queue_lock); - - if (rq->q->dev && !(rq->rq_flags & RQF_PM)) - --rq->q->nr_pending; -} #else static inline int blk_pm_resume_queue(const bool pm, struct request_queue *q) { @@ -55,19 +30,6 @@ static inline int blk_pm_resume_queue(const bool pm, struct request_queue *q) static inline void blk_pm_mark_last_busy(struct request *rq) { } - -static inline void blk_pm_requeue_request(struct request *rq) -{ -} - -static inline void blk_pm_add_request(struct request_queue *q, - struct request *rq) -{ -} - -static inline void blk_pm_put_request(struct request *rq) -{ -} #endif #endif /* _BLOCK_BLK_PM_H_ */ diff --git a/block/blk-settings.c b/block/blk-settings.c index 7dd8be314ac6..b4aa2f37fab6 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -504,6 +504,14 @@ void blk_queue_io_opt(struct request_queue *q, unsigned int opt) } EXPORT_SYMBOL(blk_queue_io_opt); +static unsigned int blk_round_down_sectors(unsigned int sectors, unsigned int lbs) +{ + sectors = round_down(sectors, lbs >> SECTOR_SHIFT); + if (sectors < PAGE_SIZE >> SECTOR_SHIFT) + sectors = PAGE_SIZE >> SECTOR_SHIFT; + return sectors; +} + /** * blk_stack_limits - adjust queue_limits for stacked devices * @t: the stacking driver limits (top device) @@ -630,6 +638,10 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, ret = -1; } + t->max_sectors = blk_round_down_sectors(t->max_sectors, t->logical_block_size); + t->max_hw_sectors = blk_round_down_sectors(t->max_hw_sectors, t->logical_block_size); + t->max_dev_sectors = blk_round_down_sectors(t->max_dev_sectors, t->logical_block_size); + /* Discard alignment and granularity */ if (b->discard_granularity) { alignment = queue_limit_discard_alignment(b, start); diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index ae39c7f3d83d..0f4f0c8a7825 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -434,10 +434,13 @@ static ssize_t queue_poll_store(struct request_queue *q, const char *page, if (ret < 0) return ret; - if (poll_on) + if (poll_on) { blk_queue_flag_set(QUEUE_FLAG_POLL, q); - else + } else { + blk_mq_freeze_queue(q); blk_queue_flag_clear(QUEUE_FLAG_POLL, q); + blk_mq_unfreeze_queue(q); + } return ret; } diff --git a/block/bounce.c b/block/bounce.c index fc55314aa426..87983a35079c 100644 --- a/block/bounce.c +++ b/block/bounce.c @@ -214,8 +214,7 @@ static void bounce_end_io_read_isa(struct bio *bio) __bounce_end_io_read(bio, &isa_page_pool); } -static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, - struct bio_set *bs) +static struct bio *bounce_clone_bio(struct bio *bio_src) { struct bvec_iter iter; struct bio_vec bv; @@ -242,10 +241,12 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, * asking for trouble and would force extra work on * __bio_clone_fast() anyways. */ - - bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs); - if (!bio) - return NULL; + if (bio_is_passthrough(bio_src)) + bio = bio_kmalloc(GFP_NOIO | __GFP_NOFAIL, + bio_segments(bio_src)); + else + bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), + &bounce_bio_set); bio->bi_bdev = bio_src->bi_bdev; if (bio_flagged(bio_src, BIO_REMAPPED)) bio_set_flag(bio, BIO_REMAPPED); @@ -269,11 +270,11 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, break; } - if (bio_crypt_clone(bio, bio_src, gfp_mask) < 0) + if (bio_crypt_clone(bio, bio_src, GFP_NOIO) < 0) goto err_put; if (bio_integrity(bio_src) && - bio_integrity_clone(bio, bio_src, gfp_mask) < 0) + bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) goto err_put; bio_clone_blkg_association(bio, bio_src); @@ -296,7 +297,6 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig, unsigned i = 0; bool bounce = false; int sectors = 0; - bool passthrough = bio_is_passthrough(*bio_orig); bio_for_each_segment(from, *bio_orig, iter) { if (i++ < BIO_MAX_PAGES) @@ -307,14 +307,14 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig, if (!bounce) return; - if (!passthrough && sectors < bio_sectors(*bio_orig)) { + if (!bio_is_passthrough(*bio_orig) && + sectors < bio_sectors(*bio_orig)) { bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split); bio_chain(bio, *bio_orig); submit_bio_noacct(*bio_orig); *bio_orig = bio; } - bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL : - &bounce_bio_set); + bio = bounce_clone_bio(*bio_orig); /* * Bvec table can't be updated by bio_for_each_segment_all(), diff --git a/block/genhd.c b/block/genhd.c index 36ff45bbaaaf..fcc530164b5a 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -74,7 +74,7 @@ bool set_capacity_and_notify(struct gendisk *disk, sector_t size) return false; pr_info("%s: detected capacity change from %lld to %lld\n", - disk->disk_name, size, capacity); + disk->disk_name, capacity, size); /* * Historically we did not send a uevent for changes to/from an empty @@ -476,7 +476,7 @@ void disk_uevent(struct gendisk *disk, enum kobject_action action) struct disk_part_iter piter; struct block_device *part; - disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0); + disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY_PART0); while ((part = disk_part_iter_next(&piter))) kobject_uevent(bdev_kobj(part), action); disk_part_iter_exit(&piter); diff --git a/block/ioctl.c b/block/ioctl.c index d61d652078f4..ff241e663c01 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -81,20 +81,27 @@ static int compat_blkpg_ioctl(struct block_device *bdev, } #endif -static int blkdev_reread_part(struct block_device *bdev) +static int blkdev_reread_part(struct block_device *bdev, fmode_t mode) { - int ret; + struct block_device *tmp; if (!disk_part_scan_enabled(bdev->bd_disk) || bdev_is_partition(bdev)) return -EINVAL; if (!capable(CAP_SYS_ADMIN)) return -EACCES; - mutex_lock(&bdev->bd_mutex); - ret = bdev_disk_changed(bdev, false); - mutex_unlock(&bdev->bd_mutex); + /* + * Reopen the device to revalidate the driver state and force a + * partition rescan. + */ + mode &= ~FMODE_EXCL; + set_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state); - return ret; + tmp = blkdev_get_by_dev(bdev->bd_dev, mode, NULL); + if (IS_ERR(tmp)) + return PTR_ERR(tmp); + blkdev_put(tmp, mode); + return 0; } static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode, @@ -498,7 +505,7 @@ static int blkdev_common_ioctl(struct block_device *bdev, fmode_t mode, bdev->bd_bdi->ra_pages = (arg * 512) / PAGE_SIZE; return 0; case BLKRRPART: - return blkdev_reread_part(bdev); + return blkdev_reread_part(bdev, mode); case BLKTRACESTART: case BLKTRACESTOP: case BLKTRACETEARDOWN: diff --git a/block/keyslot-manager.c b/block/keyslot-manager.c index 86f8195d8039..2c4a55bea6ca 100644 --- a/block/keyslot-manager.c +++ b/block/keyslot-manager.c @@ -29,6 +29,7 @@ #define pr_fmt(fmt) "blk-crypto: " fmt #include +#include #include #include #include @@ -62,6 +63,11 @@ static inline void blk_ksm_hw_exit(struct blk_keyslot_manager *ksm) pm_runtime_put_sync(ksm->dev); } +static inline bool blk_ksm_is_passthrough(struct blk_keyslot_manager *ksm) +{ + return ksm->num_slots == 0; +} + /** * blk_ksm_init() - Initialize a keyslot manager * @ksm: The keyslot_manager to initialize. @@ -127,6 +133,34 @@ err_destroy_ksm: } EXPORT_SYMBOL_GPL(blk_ksm_init); +static void blk_ksm_destroy_callback(void *ksm) +{ + blk_ksm_destroy(ksm); +} + +/** + * devm_blk_ksm_init() - Resource-managed blk_ksm_init() + * @dev: The device which owns the blk_keyslot_manager. + * @ksm: The blk_keyslot_manager to initialize. + * @num_slots: The number of key slots to manage. + * + * Like blk_ksm_init(), but causes blk_ksm_destroy() to be called automatically + * on driver detach. + * + * Return: 0 on success, or else a negative error code. + */ +int devm_blk_ksm_init(struct device *dev, struct blk_keyslot_manager *ksm, + unsigned int num_slots) +{ + int err = blk_ksm_init(ksm, num_slots); + + if (err) + return err; + + return devm_add_action_or_reset(dev, blk_ksm_destroy_callback, ksm); +} +EXPORT_SYMBOL_GPL(devm_blk_ksm_init); + static inline struct hlist_head * blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager *ksm, const struct blk_crypto_key *key) @@ -205,6 +239,10 @@ blk_status_t blk_ksm_get_slot_for_key(struct blk_keyslot_manager *ksm, int err; *slot_ptr = NULL; + + if (blk_ksm_is_passthrough(ksm)) + return BLK_STS_OK; + down_read(&ksm->lock); slot = blk_ksm_find_and_grab_keyslot(ksm, key); up_read(&ksm->lock); @@ -325,6 +363,16 @@ int blk_ksm_evict_key(struct blk_keyslot_manager *ksm, struct blk_ksm_keyslot *slot; int err = 0; + if (blk_ksm_is_passthrough(ksm)) { + if (ksm->ksm_ll_ops.keyslot_evict) { + blk_ksm_hw_enter(ksm); + err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, -1); + blk_ksm_hw_exit(ksm); + return err; + } + return 0; + } + blk_ksm_hw_enter(ksm); slot = blk_ksm_find_keyslot(ksm, key); if (!slot) @@ -360,6 +408,9 @@ void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager *ksm) { unsigned int slot; + if (blk_ksm_is_passthrough(ksm)) + return; + /* This is for device initialization, so don't resume the device */ down_write(&ksm->lock); for (slot = 0; slot < ksm->num_slots; slot++) { @@ -401,3 +452,127 @@ void blk_ksm_unregister(struct request_queue *q) { q->ksm = NULL; } + +/** + * blk_ksm_intersect_modes() - restrict supported modes by child device + * @parent: The keyslot manager for parent device + * @child: The keyslot manager for child device, or NULL + * + * Clear any crypto mode support bits in @parent that aren't set in @child. + * If @child is NULL, then all parent bits are cleared. + * + * Only use this when setting up the keyslot manager for a layered device, + * before it's been exposed yet. + */ +void blk_ksm_intersect_modes(struct blk_keyslot_manager *parent, + const struct blk_keyslot_manager *child) +{ + if (child) { + unsigned int i; + + parent->max_dun_bytes_supported = + min(parent->max_dun_bytes_supported, + child->max_dun_bytes_supported); + for (i = 0; i < ARRAY_SIZE(child->crypto_modes_supported); + i++) { + parent->crypto_modes_supported[i] &= + child->crypto_modes_supported[i]; + } + } else { + parent->max_dun_bytes_supported = 0; + memset(parent->crypto_modes_supported, 0, + sizeof(parent->crypto_modes_supported)); + } +} +EXPORT_SYMBOL_GPL(blk_ksm_intersect_modes); + +/** + * blk_ksm_is_superset() - Check if a KSM supports a superset of crypto modes + * and DUN bytes that another KSM supports. Here, + * "superset" refers to the mathematical meaning of the + * word - i.e. if two KSMs have the *same* capabilities, + * they *are* considered supersets of each other. + * @ksm_superset: The KSM that we want to verify is a superset + * @ksm_subset: The KSM that we want to verify is a subset + * + * Return: True if @ksm_superset supports a superset of the crypto modes and DUN + * bytes that @ksm_subset supports. + */ +bool blk_ksm_is_superset(struct blk_keyslot_manager *ksm_superset, + struct blk_keyslot_manager *ksm_subset) +{ + int i; + + if (!ksm_subset) + return true; + + if (!ksm_superset) + return false; + + for (i = 0; i < ARRAY_SIZE(ksm_superset->crypto_modes_supported); i++) { + if (ksm_subset->crypto_modes_supported[i] & + (~ksm_superset->crypto_modes_supported[i])) { + return false; + } + } + + if (ksm_subset->max_dun_bytes_supported > + ksm_superset->max_dun_bytes_supported) { + return false; + } + + return true; +} +EXPORT_SYMBOL_GPL(blk_ksm_is_superset); + +/** + * blk_ksm_update_capabilities() - Update the restrictions of a KSM to those of + * another KSM + * @target_ksm: The KSM whose restrictions to update. + * @reference_ksm: The KSM to whose restrictions this function will update + * @target_ksm's restrictions to. + * + * Blk-crypto requires that crypto capabilities that were + * advertised when a bio was created continue to be supported by the + * device until that bio is ended. This is turn means that a device cannot + * shrink its advertised crypto capabilities without any explicit + * synchronization with upper layers. So if there's no such explicit + * synchronization, @reference_ksm must support all the crypto capabilities that + * @target_ksm does + * (i.e. we need blk_ksm_is_superset(@reference_ksm, @target_ksm) == true). + * + * Note also that as long as the crypto capabilities are being expanded, the + * order of updates becoming visible is not important because it's alright + * for blk-crypto to see stale values - they only cause blk-crypto to + * believe that a crypto capability isn't supported when it actually is (which + * might result in blk-crypto-fallback being used if available, or the bio being + * failed). + */ +void blk_ksm_update_capabilities(struct blk_keyslot_manager *target_ksm, + struct blk_keyslot_manager *reference_ksm) +{ + memcpy(target_ksm->crypto_modes_supported, + reference_ksm->crypto_modes_supported, + sizeof(target_ksm->crypto_modes_supported)); + + target_ksm->max_dun_bytes_supported = + reference_ksm->max_dun_bytes_supported; +} +EXPORT_SYMBOL_GPL(blk_ksm_update_capabilities); + +/** + * blk_ksm_init_passthrough() - Init a passthrough keyslot manager + * @ksm: The keyslot manager to init + * + * Initialize a passthrough keyslot manager. + * Called by e.g. storage drivers to set up a keyslot manager in their + * request_queue, when the storage driver wants to manage its keys by itself. + * This is useful for inline encryption hardware that doesn't have the concept + * of keyslots, and for layered devices. + */ +void blk_ksm_init_passthrough(struct blk_keyslot_manager *ksm) +{ + memset(ksm, 0, sizeof(*ksm)); + init_rwsem(&ksm->lock); +} +EXPORT_SYMBOL_GPL(blk_ksm_init_passthrough); diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index c25c41d0d061..33d34d69cade 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -13,6 +13,8 @@ #include #include +#include + #include "blk.h" #include "blk-mq.h" #include "blk-mq-debugfs.h" @@ -353,19 +355,9 @@ static void kyber_timer_fn(struct timer_list *t) } } -static unsigned int kyber_sched_tags_shift(struct request_queue *q) -{ - /* - * All of the hardware queues have the same depth, so we can just grab - * the shift of the first one. - */ - return q->queue_hw_ctx[0]->sched_tags->bitmap_tags->sb.shift; -} - static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q) { struct kyber_queue_data *kqd; - unsigned int shift; int ret = -ENOMEM; int i; @@ -400,9 +392,6 @@ static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q) kqd->latency_targets[i] = kyber_latency_targets[i]; } - shift = kyber_sched_tags_shift(q); - kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U; - return kqd; err_buckets: @@ -458,9 +447,19 @@ static void kyber_ctx_queue_init(struct kyber_ctx_queue *kcq) INIT_LIST_HEAD(&kcq->rq_list[i]); } -static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) +static void kyber_depth_updated(struct blk_mq_hw_ctx *hctx) { struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data; + struct blk_mq_tags *tags = hctx->sched_tags; + unsigned int shift = tags->bitmap_tags->sb.shift; + + kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U; + + sbitmap_queue_min_shallow_depth(tags->bitmap_tags, kqd->async_depth); +} + +static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) +{ struct kyber_hctx_data *khd; int i; @@ -502,8 +501,7 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) khd->batching = 0; hctx->sched_data = khd; - sbitmap_queue_min_shallow_depth(hctx->sched_tags->bitmap_tags, - kqd->async_depth); + kyber_depth_updated(hctx); return 0; @@ -602,7 +600,7 @@ static void kyber_insert_requests(struct blk_mq_hw_ctx *hctx, list_move_tail(&rq->queuelist, head); sbitmap_set_bit(&khd->kcq_map[sched_domain], rq->mq_ctx->index_hw[hctx->type]); - blk_mq_sched_request_inserted(rq); + trace_block_rq_insert(rq); spin_unlock(&kcq->lock); } } @@ -1022,6 +1020,7 @@ static struct elevator_type kyber_sched = { .completed_request = kyber_completed_request, .dispatch_request = kyber_dispatch_request, .has_work = kyber_has_work, + .depth_updated = kyber_depth_updated, }, #ifdef CONFIG_BLK_DEBUG_FS .queue_debugfs_attrs = kyber_queue_debugfs_attrs, diff --git a/block/mq-deadline.c b/block/mq-deadline.c index b57470e154c8..f3631a287466 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -18,6 +18,8 @@ #include #include +#include + #include "blk.h" #include "blk-mq.h" #include "blk-mq-debugfs.h" @@ -496,7 +498,7 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, if (blk_mq_sched_try_insert_merge(q, rq)) return; - blk_mq_sched_request_inserted(rq); + trace_block_rq_insert(rq); if (at_head || blk_rq_is_passthrough(rq)) { if (at_head) diff --git a/certs/blacklist.c b/certs/blacklist.c index 6514f9ebc943..bffe4c6f4a9e 100644 --- a/certs/blacklist.c +++ b/certs/blacklist.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "blacklist.h" @@ -37,7 +38,7 @@ static int blacklist_vet_description(const char *desc) found_colon: desc++; for (; *desc; desc++) { - if (!isxdigit(*desc)) + if (!isxdigit(*desc) || isupper(*desc)) return -EINVAL; n++; } @@ -78,7 +79,7 @@ static struct key_type key_type_blacklist = { /** * mark_hash_blacklisted - Add a hash to the system blacklist - * @hash - The hash as a hex string with a type prefix (eg. "tbs:23aa429783") + * @hash: The hash as a hex string with a type prefix (eg. "tbs:23aa429783") */ int mark_hash_blacklisted(const char *hash) { @@ -156,13 +157,12 @@ static int __init blacklist_init(void) blacklist_keyring = keyring_alloc(".blacklist", - KUIDT_INIT(0), KGIDT_INIT(0), - current_cred(), + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), (KEY_POS_ALL & ~KEY_POS_SETATTR) | KEY_USR_VIEW | KEY_USR_READ | KEY_USR_SEARCH, KEY_ALLOC_NOT_IN_QUOTA | - KEY_FLAG_KEEP, + KEY_ALLOC_SET_KEEP, NULL, NULL); if (IS_ERR(blacklist_keyring)) panic("Can't allocate system blacklist keyring\n"); diff --git a/certs/system_keyring.c b/certs/system_keyring.c index 798291177186..4b693da488f1 100644 --- a/certs/system_keyring.c +++ b/certs/system_keyring.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -98,7 +99,7 @@ static __init int system_trusted_keyring_init(void) builtin_trusted_keys = keyring_alloc(".builtin_trusted_keys", - KUIDT_INIT(0), KGIDT_INIT(0), current_cred(), + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), ((KEY_POS_ALL & ~KEY_POS_SETATTR) | KEY_USR_VIEW | KEY_USR_READ | KEY_USR_SEARCH), KEY_ALLOC_NOT_IN_QUOTA, @@ -109,7 +110,7 @@ static __init int system_trusted_keyring_init(void) #ifdef CONFIG_SECONDARY_TRUSTED_KEYRING secondary_trusted_keys = keyring_alloc(".secondary_trusted_keys", - KUIDT_INIT(0), KGIDT_INIT(0), current_cred(), + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), ((KEY_POS_ALL & ~KEY_POS_SETATTR) | KEY_USR_VIEW | KEY_USR_READ | KEY_USR_SEARCH | KEY_USR_WRITE), diff --git a/crypto/Kconfig b/crypto/Kconfig index a367fcfeb5d4..15c9c28d9f53 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -210,11 +210,6 @@ config CRYPTO_SIMD tristate select CRYPTO_CRYPTD -config CRYPTO_GLUE_HELPER_X86 - tristate - depends on X86 - select CRYPTO_SKCIPHER - config CRYPTO_ENGINE tristate @@ -822,19 +817,6 @@ config CRYPTO_MICHAEL_MIC should not be used for other purposes because of the weakness of the algorithm. -config CRYPTO_RMD128 - tristate "RIPEMD-128 digest algorithm" - select CRYPTO_HASH - help - RIPEMD-128 (ISO/IEC 10118-3:2004). - - RIPEMD-128 is a 128-bit cryptographic hash function. It should only - be used as a secure replacement for RIPEMD. For other use cases, - RIPEMD-160 should be used. - - Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See - config CRYPTO_RMD160 tristate "RIPEMD-160 digest algorithm" select CRYPTO_HASH @@ -852,30 +834,6 @@ config CRYPTO_RMD160 Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. See -config CRYPTO_RMD256 - tristate "RIPEMD-256 digest algorithm" - select CRYPTO_HASH - help - RIPEMD-256 is an optional extension of RIPEMD-128 with a - 256 bit hash. It is intended for applications that require - longer hash-results, without needing a larger security level - (than RIPEMD-128). - - Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See - -config CRYPTO_RMD320 - tristate "RIPEMD-320 digest algorithm" - select CRYPTO_HASH - help - RIPEMD-320 is an optional extension of RIPEMD-160 with a - 320 bit hash. It is intended for applications that require - longer hash-results, without needing a larger security level - (than RIPEMD-160). - - Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See - config CRYPTO_SHA1 tristate "SHA1 digest algorithm" select CRYPTO_HASH @@ -1051,19 +1009,6 @@ config CRYPTO_STREEBOG https://tc26.ru/upload/iblock/fed/feddbb4d26b685903faa2ba11aea43f6.pdf https://tools.ietf.org/html/rfc6986 -config CRYPTO_TGR192 - tristate "Tiger digest algorithms" - select CRYPTO_HASH - help - Tiger hash algorithm 192, 160 and 128-bit hashes - - Tiger is a hash function optimized for 64-bit processors while - still having decent performance on 32-bit processors. - Tiger was developed by Ross Anderson and Eli Biham. - - See also: - . - config CRYPTO_WP512 tristate "Whirlpool digest algorithms" select CRYPTO_HASH @@ -1133,7 +1078,6 @@ config CRYPTO_AES_NI_INTEL select CRYPTO_LIB_AES select CRYPTO_ALGAPI select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 if 64BIT select CRYPTO_SIMD help Use Intel AES-NI instructions for AES algorithm. @@ -1256,6 +1200,7 @@ config CRYPTO_BLOWFISH_X86_64 depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_BLOWFISH_COMMON + imply CRYPTO_CTR help Blowfish cipher algorithm (x86_64), by Bruce Schneier. @@ -1286,7 +1231,7 @@ config CRYPTO_CAMELLIA_X86_64 depends on X86 && 64BIT depends on CRYPTO select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 + imply CRYPTO_CTR help Camellia cipher algorithm module (x86_64). @@ -1304,9 +1249,8 @@ config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 depends on CRYPTO select CRYPTO_SKCIPHER select CRYPTO_CAMELLIA_X86_64 - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SIMD - select CRYPTO_XTS + imply CRYPTO_XTS help Camellia cipher algorithm module (x86_64/AES-NI/AVX). @@ -1372,6 +1316,7 @@ config CRYPTO_CAST5_AVX_X86_64 select CRYPTO_CAST5 select CRYPTO_CAST_COMMON select CRYPTO_SIMD + imply CRYPTO_CTR help The CAST5 encryption algorithm (synonymous with CAST-128) is described in RFC2144. @@ -1393,9 +1338,9 @@ config CRYPTO_CAST6_AVX_X86_64 select CRYPTO_SKCIPHER select CRYPTO_CAST6 select CRYPTO_CAST_COMMON - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SIMD - select CRYPTO_XTS + imply CRYPTO_XTS + imply CRYPTO_CTR help The CAST6 encryption algorithm (synonymous with CAST-256) is described in RFC2612. @@ -1425,6 +1370,7 @@ config CRYPTO_DES3_EDE_X86_64 depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_LIB_DES + imply CRYPTO_CTR help Triple DES EDE (FIPS 46-3) algorithm. @@ -1454,18 +1400,6 @@ config CRYPTO_KHAZAD See also: -config CRYPTO_SALSA20 - tristate "Salsa20 stream cipher algorithm" - select CRYPTO_SKCIPHER - help - Salsa20 stream cipher algorithm. - - Salsa20 is a stream cipher submitted to eSTREAM, the ECRYPT - Stream Cipher Project. See - - The Salsa20 stream cipher algorithm is designed by Daniel J. - Bernstein . See - config CRYPTO_CHACHA20 tristate "ChaCha stream cipher algorithms" select CRYPTO_LIB_CHACHA_GENERIC @@ -1526,8 +1460,7 @@ config CRYPTO_SERPENT Serpent cipher algorithm, by Anderson, Biham & Knudsen. Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. Also includes the 'Tnepres' algorithm, a reversed - variant of Serpent for compatibility with old kerneli.org code. + of 8 bits. See also: @@ -1536,9 +1469,9 @@ config CRYPTO_SERPENT_SSE2_X86_64 tristate "Serpent cipher algorithm (x86_64/SSE2)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SERPENT select CRYPTO_SIMD + imply CRYPTO_CTR help Serpent cipher algorithm, by Anderson, Biham & Knudsen. @@ -1555,9 +1488,9 @@ config CRYPTO_SERPENT_SSE2_586 tristate "Serpent cipher algorithm (i586/SSE2)" depends on X86 && !64BIT select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SERPENT select CRYPTO_SIMD + imply CRYPTO_CTR help Serpent cipher algorithm, by Anderson, Biham & Knudsen. @@ -1574,10 +1507,10 @@ config CRYPTO_SERPENT_AVX_X86_64 tristate "Serpent cipher algorithm (x86_64/AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SERPENT select CRYPTO_SIMD - select CRYPTO_XTS + imply CRYPTO_XTS + imply CRYPTO_CTR help Serpent cipher algorithm, by Anderson, Biham & Knudsen. @@ -1675,6 +1608,7 @@ config CRYPTO_TWOFISH_586 depends on (X86 || UML_X86) && !64BIT select CRYPTO_ALGAPI select CRYPTO_TWOFISH_COMMON + imply CRYPTO_CTR help Twofish cipher algorithm. @@ -1691,6 +1625,7 @@ config CRYPTO_TWOFISH_X86_64 depends on (X86 || UML_X86) && 64BIT select CRYPTO_ALGAPI select CRYPTO_TWOFISH_COMMON + imply CRYPTO_CTR help Twofish cipher algorithm (x86_64). @@ -1708,7 +1643,6 @@ config CRYPTO_TWOFISH_X86_64_3WAY select CRYPTO_SKCIPHER select CRYPTO_TWOFISH_COMMON select CRYPTO_TWOFISH_X86_64 - select CRYPTO_GLUE_HELPER_X86 help Twofish cipher algorithm (x86_64, 3-way parallel). @@ -1727,11 +1661,11 @@ config CRYPTO_TWOFISH_AVX_X86_64 tristate "Twofish cipher algorithm (x86_64/AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SIMD select CRYPTO_TWOFISH_COMMON select CRYPTO_TWOFISH_X86_64 select CRYPTO_TWOFISH_X86_64_3WAY + imply CRYPTO_XTS help Twofish cipher algorithm (x86_64/AVX). diff --git a/crypto/Makefile b/crypto/Makefile index b279483fba50..cf23affb1678 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -67,9 +67,7 @@ obj-$(CONFIG_CRYPTO_XCBC) += xcbc.o obj-$(CONFIG_CRYPTO_NULL2) += crypto_null.o obj-$(CONFIG_CRYPTO_MD4) += md4.o obj-$(CONFIG_CRYPTO_MD5) += md5.o -obj-$(CONFIG_CRYPTO_RMD128) += rmd128.o obj-$(CONFIG_CRYPTO_RMD160) += rmd160.o -obj-$(CONFIG_CRYPTO_RMD256) += rmd256.o obj-$(CONFIG_CRYPTO_RMD320) += rmd320.o obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o obj-$(CONFIG_CRYPTO_SHA256) += sha256_generic.o @@ -79,7 +77,6 @@ obj-$(CONFIG_CRYPTO_SM3) += sm3_generic.o obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o CFLAGS_wp512.o := $(call cc-option,-fno-schedule-insns) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 -obj-$(CONFIG_CRYPTO_TGR192) += tgr192.o obj-$(CONFIG_CRYPTO_BLAKE2B) += blake2b_generic.o obj-$(CONFIG_CRYPTO_BLAKE2S) += blake2s_generic.o obj-$(CONFIG_CRYPTO_GF128MUL) += gf128mul.o @@ -141,7 +138,6 @@ obj-$(CONFIG_CRYPTO_TEA) += tea.o obj-$(CONFIG_CRYPTO_KHAZAD) += khazad.o obj-$(CONFIG_CRYPTO_ANUBIS) += anubis.o obj-$(CONFIG_CRYPTO_SEED) += seed.o -obj-$(CONFIG_CRYPTO_SALSA20) += salsa20_generic.o obj-$(CONFIG_CRYPTO_CHACHA20) += chacha_generic.o obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o diff --git a/crypto/adiantum.c b/crypto/adiantum.c index ce4d5725342c..84450130cb6b 100644 --- a/crypto/adiantum.c +++ b/crypto/adiantum.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -616,3 +617,4 @@ MODULE_DESCRIPTION("Adiantum length-preserving encryption mode"); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Eric Biggers "); MODULE_ALIAS_CRYPTO("adiantum"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c index c475c1129ff2..3f512efaba3a 100644 --- a/crypto/ansi_cprng.c +++ b/crypto/ansi_cprng.c @@ -7,6 +7,7 @@ * (C) Neil Horman */ +#include #include #include #include @@ -470,3 +471,4 @@ subsys_initcall(prng_mod_init); module_exit(prng_mod_fini); MODULE_ALIAS_CRYPTO("stdrng"); MODULE_ALIAS_CRYPTO("ansi_cprng"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/asymmetric_keys/asymmetric_type.c b/crypto/asymmetric_keys/asymmetric_type.c index 33e77d846caa..ad8af3d70ac0 100644 --- a/crypto/asymmetric_keys/asymmetric_type.c +++ b/crypto/asymmetric_keys/asymmetric_type.c @@ -152,7 +152,8 @@ EXPORT_SYMBOL_GPL(asymmetric_key_generate_id); /** * asymmetric_key_id_same - Return true if two asymmetric keys IDs are the same. - * @kid_1, @kid_2: The key IDs to compare + * @kid1: The key ID to compare + * @kid2: The key ID to compare */ bool asymmetric_key_id_same(const struct asymmetric_key_id *kid1, const struct asymmetric_key_id *kid2) @@ -168,7 +169,8 @@ EXPORT_SYMBOL_GPL(asymmetric_key_id_same); /** * asymmetric_key_id_partial - Return true if two asymmetric keys IDs * partially match - * @kid_1, @kid_2: The key IDs to compare + * @kid1: The key ID to compare + * @kid2: The key ID to compare */ bool asymmetric_key_id_partial(const struct asymmetric_key_id *kid1, const struct asymmetric_key_id *kid2) diff --git a/crypto/asymmetric_keys/pkcs7_parser.h b/crypto/asymmetric_keys/pkcs7_parser.h index 6565fdc2d4ca..e17f7ce4fb43 100644 --- a/crypto/asymmetric_keys/pkcs7_parser.h +++ b/crypto/asymmetric_keys/pkcs7_parser.h @@ -41,10 +41,9 @@ struct pkcs7_signed_info { * * This contains the generated digest of _either_ the Content Data or * the Authenticated Attributes [RFC2315 9.3]. If the latter, one of - * the attributes contains the digest of the the Content Data within - * it. + * the attributes contains the digest of the Content Data within it. * - * THis also contains the issuing cert serial number and issuer's name + * This also contains the issuing cert serial number and issuer's name * [PKCS#7 or CMS ver 1] or issuing cert's SKID [CMS ver 3]. */ struct public_key_signature *sig; diff --git a/crypto/asymmetric_keys/pkcs7_trust.c b/crypto/asymmetric_keys/pkcs7_trust.c index 61af3c4d82cc..b531df2013c4 100644 --- a/crypto/asymmetric_keys/pkcs7_trust.c +++ b/crypto/asymmetric_keys/pkcs7_trust.c @@ -16,7 +16,7 @@ #include #include "pkcs7_parser.h" -/** +/* * Check the trust on one PKCS#7 SignedInfo block. */ static int pkcs7_validate_trust_one(struct pkcs7_message *pkcs7, diff --git a/crypto/asymmetric_keys/pkcs7_verify.c b/crypto/asymmetric_keys/pkcs7_verify.c index ce49820caa97..0b4d07aa8811 100644 --- a/crypto/asymmetric_keys/pkcs7_verify.c +++ b/crypto/asymmetric_keys/pkcs7_verify.c @@ -141,11 +141,10 @@ int pkcs7_get_digest(struct pkcs7_message *pkcs7, const u8 **buf, u32 *len, *buf = sinfo->sig->digest; *len = sinfo->sig->digest_size; - for (i = 0; i < HASH_ALGO__LAST; i++) - if (!strcmp(hash_algo_name[i], sinfo->sig->hash_algo)) { - *hash_algo = i; - break; - } + i = match_string(hash_algo_name, HASH_ALGO__LAST, + sinfo->sig->hash_algo); + if (i >= 0) + *hash_algo = i; return 0; } diff --git a/crypto/asymmetric_keys/restrict.c b/crypto/asymmetric_keys/restrict.c index 77ebebada29c..84cefe3b3585 100644 --- a/crypto/asymmetric_keys/restrict.c +++ b/crypto/asymmetric_keys/restrict.c @@ -244,9 +244,10 @@ int restrict_link_by_key_or_keyring(struct key *dest_keyring, * @payload: The payload of the new key. * @trusted: A key or ring of keys that can be used to vouch for the new cert. * - * Check the new certificate only against the key or keys passed in the data - * parameter. If one of those is the signing key and validates the new - * certificate, then mark the new certificate as being ok to link. + * Check the new certificate against the key or keys passed in the data + * parameter and against the keys already linked to the destination keyring. If + * one of those is the signing key and validates the new certificate, then mark + * the new certificate as being ok to link. * * Returns 0 if the new certificate was accepted, -ENOKEY if we * couldn't find a matching parent certificate in the trusted list, diff --git a/crypto/blake2b_generic.c b/crypto/blake2b_generic.c index a2ffe60e06d3..6704c0355889 100644 --- a/crypto/blake2b_generic.c +++ b/crypto/blake2b_generic.c @@ -1,55 +1,27 @@ // SPDX-License-Identifier: (GPL-2.0-only OR Apache-2.0) /* - * BLAKE2b reference source code package - reference C implementations + * Generic implementation of the BLAKE2b digest algorithm. Based on the BLAKE2b + * reference implementation, but it has been heavily modified for use in the + * kernel. The reference implementation was: * - * Copyright 2012, Samuel Neves . You may use this under the - * terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at - * your option. The terms of these licenses can be found at: + * Copyright 2012, Samuel Neves . You may use this under + * the terms of the CC0, the OpenSSL Licence, or the Apache Public License + * 2.0, at your option. The terms of these licenses can be found at: * - * - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 - * - OpenSSL license : https://www.openssl.org/source/license.html - * - Apache 2.0 : https://www.apache.org/licenses/LICENSE-2.0 + * - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + * - OpenSSL license : https://www.openssl.org/source/license.html + * - Apache 2.0 : https://www.apache.org/licenses/LICENSE-2.0 * - * More information about the BLAKE2 hash function can be found at - * https://blake2.net. - * - * Note: the original sources have been modified for inclusion in linux kernel - * in terms of coding style, using generic helpers and simplifications of error - * handling. + * More information about BLAKE2 can be found at https://blake2.net. */ #include #include -#include #include #include +#include #include -#define BLAKE2B_160_DIGEST_SIZE (160 / 8) -#define BLAKE2B_256_DIGEST_SIZE (256 / 8) -#define BLAKE2B_384_DIGEST_SIZE (384 / 8) -#define BLAKE2B_512_DIGEST_SIZE (512 / 8) - -enum blake2b_constant { - BLAKE2B_BLOCKBYTES = 128, - BLAKE2B_KEYBYTES = 64, -}; - -struct blake2b_state { - u64 h[8]; - u64 t[2]; - u64 f[2]; - u8 buf[BLAKE2B_BLOCKBYTES]; - size_t buflen; -}; - -static const u64 blake2b_IV[8] = { - 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, - 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, - 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL -}; - static const u8 blake2b_sigma[12][16] = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, @@ -95,8 +67,8 @@ static void blake2b_increment_counter(struct blake2b_state *S, const u64 inc) G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ } while (0) -static void blake2b_compress(struct blake2b_state *S, - const u8 block[BLAKE2B_BLOCKBYTES]) +static void blake2b_compress_one_generic(struct blake2b_state *S, + const u8 block[BLAKE2B_BLOCK_SIZE]) { u64 m[16]; u64 v[16]; @@ -108,14 +80,14 @@ static void blake2b_compress(struct blake2b_state *S, for (i = 0; i < 8; ++i) v[i] = S->h[i]; - v[ 8] = blake2b_IV[0]; - v[ 9] = blake2b_IV[1]; - v[10] = blake2b_IV[2]; - v[11] = blake2b_IV[3]; - v[12] = blake2b_IV[4] ^ S->t[0]; - v[13] = blake2b_IV[5] ^ S->t[1]; - v[14] = blake2b_IV[6] ^ S->f[0]; - v[15] = blake2b_IV[7] ^ S->f[1]; + v[ 8] = BLAKE2B_IV0; + v[ 9] = BLAKE2B_IV1; + v[10] = BLAKE2B_IV2; + v[11] = BLAKE2B_IV3; + v[12] = BLAKE2B_IV4 ^ S->t[0]; + v[13] = BLAKE2B_IV5 ^ S->t[1]; + v[14] = BLAKE2B_IV6 ^ S->f[0]; + v[15] = BLAKE2B_IV7 ^ S->f[1]; ROUND(0); ROUND(1); @@ -139,159 +111,54 @@ static void blake2b_compress(struct blake2b_state *S, #undef G #undef ROUND -struct blake2b_tfm_ctx { - u8 key[BLAKE2B_KEYBYTES]; - unsigned int keylen; -}; - -static int blake2b_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) +void blake2b_compress_generic(struct blake2b_state *state, + const u8 *block, size_t nblocks, u32 inc) { - struct blake2b_tfm_ctx *tctx = crypto_shash_ctx(tfm); + do { + blake2b_increment_counter(state, inc); + blake2b_compress_one_generic(state, block); + block += BLAKE2B_BLOCK_SIZE; + } while (--nblocks); +} +EXPORT_SYMBOL(blake2b_compress_generic); - if (keylen == 0 || keylen > BLAKE2B_KEYBYTES) - return -EINVAL; - - memcpy(tctx->key, key, keylen); - tctx->keylen = keylen; - - return 0; +static int crypto_blake2b_update_generic(struct shash_desc *desc, + const u8 *in, unsigned int inlen) +{ + return crypto_blake2b_update(desc, in, inlen, blake2b_compress_generic); } -static int blake2b_init(struct shash_desc *desc) +static int crypto_blake2b_final_generic(struct shash_desc *desc, u8 *out) { - struct blake2b_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct blake2b_state *state = shash_desc_ctx(desc); - const int digestsize = crypto_shash_digestsize(desc->tfm); + return crypto_blake2b_final(desc, out, blake2b_compress_generic); +} - memset(state, 0, sizeof(*state)); - memcpy(state->h, blake2b_IV, sizeof(state->h)); - - /* Parameter block is all zeros except index 0, no xor for 1..7 */ - state->h[0] ^= 0x01010000 | tctx->keylen << 8 | digestsize; - - if (tctx->keylen) { - /* - * Prefill the buffer with the key, next call to _update or - * _final will process it - */ - memcpy(state->buf, tctx->key, tctx->keylen); - state->buflen = BLAKE2B_BLOCKBYTES; +#define BLAKE2B_ALG(name, driver_name, digest_size) \ + { \ + .base.cra_name = name, \ + .base.cra_driver_name = driver_name, \ + .base.cra_priority = 100, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_blocksize = BLAKE2B_BLOCK_SIZE, \ + .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), \ + .base.cra_module = THIS_MODULE, \ + .digestsize = digest_size, \ + .setkey = crypto_blake2b_setkey, \ + .init = crypto_blake2b_init, \ + .update = crypto_blake2b_update_generic, \ + .final = crypto_blake2b_final_generic, \ + .descsize = sizeof(struct blake2b_state), \ } - return 0; -} - -static int blake2b_update(struct shash_desc *desc, const u8 *in, - unsigned int inlen) -{ - struct blake2b_state *state = shash_desc_ctx(desc); - const size_t left = state->buflen; - const size_t fill = BLAKE2B_BLOCKBYTES - left; - - if (!inlen) - return 0; - - if (inlen > fill) { - state->buflen = 0; - /* Fill buffer */ - memcpy(state->buf + left, in, fill); - blake2b_increment_counter(state, BLAKE2B_BLOCKBYTES); - /* Compress */ - blake2b_compress(state, state->buf); - in += fill; - inlen -= fill; - while (inlen > BLAKE2B_BLOCKBYTES) { - blake2b_increment_counter(state, BLAKE2B_BLOCKBYTES); - blake2b_compress(state, in); - in += BLAKE2B_BLOCKBYTES; - inlen -= BLAKE2B_BLOCKBYTES; - } - } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; - - return 0; -} - -static int blake2b_final(struct shash_desc *desc, u8 *out) -{ - struct blake2b_state *state = shash_desc_ctx(desc); - const int digestsize = crypto_shash_digestsize(desc->tfm); - size_t i; - - blake2b_increment_counter(state, state->buflen); - /* Set last block */ - state->f[0] = (u64)-1; - /* Padding */ - memset(state->buf + state->buflen, 0, BLAKE2B_BLOCKBYTES - state->buflen); - blake2b_compress(state, state->buf); - - /* Avoid temporary buffer and switch the internal output to LE order */ - for (i = 0; i < ARRAY_SIZE(state->h); i++) - __cpu_to_le64s(&state->h[i]); - - memcpy(out, state->h, digestsize); - return 0; -} static struct shash_alg blake2b_algs[] = { - { - .base.cra_name = "blake2b-160", - .base.cra_driver_name = "blake2b-160-generic", - .base.cra_priority = 100, - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_blocksize = BLAKE2B_BLOCKBYTES, - .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), - .base.cra_module = THIS_MODULE, - .digestsize = BLAKE2B_160_DIGEST_SIZE, - .setkey = blake2b_setkey, - .init = blake2b_init, - .update = blake2b_update, - .final = blake2b_final, - .descsize = sizeof(struct blake2b_state), - }, { - .base.cra_name = "blake2b-256", - .base.cra_driver_name = "blake2b-256-generic", - .base.cra_priority = 100, - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_blocksize = BLAKE2B_BLOCKBYTES, - .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), - .base.cra_module = THIS_MODULE, - .digestsize = BLAKE2B_256_DIGEST_SIZE, - .setkey = blake2b_setkey, - .init = blake2b_init, - .update = blake2b_update, - .final = blake2b_final, - .descsize = sizeof(struct blake2b_state), - }, { - .base.cra_name = "blake2b-384", - .base.cra_driver_name = "blake2b-384-generic", - .base.cra_priority = 100, - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_blocksize = BLAKE2B_BLOCKBYTES, - .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), - .base.cra_module = THIS_MODULE, - .digestsize = BLAKE2B_384_DIGEST_SIZE, - .setkey = blake2b_setkey, - .init = blake2b_init, - .update = blake2b_update, - .final = blake2b_final, - .descsize = sizeof(struct blake2b_state), - }, { - .base.cra_name = "blake2b-512", - .base.cra_driver_name = "blake2b-512-generic", - .base.cra_priority = 100, - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_blocksize = BLAKE2B_BLOCKBYTES, - .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), - .base.cra_module = THIS_MODULE, - .digestsize = BLAKE2B_512_DIGEST_SIZE, - .setkey = blake2b_setkey, - .init = blake2b_init, - .update = blake2b_update, - .final = blake2b_final, - .descsize = sizeof(struct blake2b_state), - } + BLAKE2B_ALG("blake2b-160", "blake2b-160-generic", + BLAKE2B_160_HASH_SIZE), + BLAKE2B_ALG("blake2b-256", "blake2b-256-generic", + BLAKE2B_256_HASH_SIZE), + BLAKE2B_ALG("blake2b-384", "blake2b-384-generic", + BLAKE2B_384_HASH_SIZE), + BLAKE2B_ALG("blake2b-512", "blake2b-512-generic", + BLAKE2B_512_HASH_SIZE), }; static int __init blake2b_mod_init(void) diff --git a/crypto/blake2s_generic.c b/crypto/blake2s_generic.c index 005783ff45ad..72fe480f9bd6 100644 --- a/crypto/blake2s_generic.c +++ b/crypto/blake2s_generic.c @@ -1,149 +1,55 @@ // SPDX-License-Identifier: GPL-2.0 OR MIT /* + * shash interface to the generic implementation of BLAKE2s + * * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ #include -#include #include #include -#include #include #include -static int crypto_blake2s_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) +static int crypto_blake2s_update_generic(struct shash_desc *desc, + const u8 *in, unsigned int inlen) { - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen == 0 || keylen > BLAKE2S_KEY_SIZE) - return -EINVAL; - - memcpy(tctx->key, key, keylen); - tctx->keylen = keylen; - - return 0; + return crypto_blake2s_update(desc, in, inlen, blake2s_compress_generic); } -static int crypto_blake2s_init(struct shash_desc *desc) +static int crypto_blake2s_final_generic(struct shash_desc *desc, u8 *out) { - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct blake2s_state *state = shash_desc_ctx(desc); - const int outlen = crypto_shash_digestsize(desc->tfm); - - if (tctx->keylen) - blake2s_init_key(state, outlen, tctx->key, tctx->keylen); - else - blake2s_init(state, outlen); - - return 0; + return crypto_blake2s_final(desc, out, blake2s_compress_generic); } -static int crypto_blake2s_update(struct shash_desc *desc, const u8 *in, - unsigned int inlen) -{ - struct blake2s_state *state = shash_desc_ctx(desc); - const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; - - if (unlikely(!inlen)) - return 0; - if (inlen > fill) { - memcpy(state->buf + state->buflen, in, fill); - blake2s_compress_generic(state, state->buf, 1, BLAKE2S_BLOCK_SIZE); - state->buflen = 0; - in += fill; - inlen -= fill; +#define BLAKE2S_ALG(name, driver_name, digest_size) \ + { \ + .base.cra_name = name, \ + .base.cra_driver_name = driver_name, \ + .base.cra_priority = 100, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \ + .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \ + .base.cra_module = THIS_MODULE, \ + .digestsize = digest_size, \ + .setkey = crypto_blake2s_setkey, \ + .init = crypto_blake2s_init, \ + .update = crypto_blake2s_update_generic, \ + .final = crypto_blake2s_final_generic, \ + .descsize = sizeof(struct blake2s_state), \ } - if (inlen > BLAKE2S_BLOCK_SIZE) { - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); - /* Hash one less (full) block than strictly possible */ - blake2s_compress_generic(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE); - in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); - inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); - } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; - return 0; -} - -static int crypto_blake2s_final(struct shash_desc *desc, u8 *out) -{ - struct blake2s_state *state = shash_desc_ctx(desc); - - blake2s_set_lastblock(state); - memset(state->buf + state->buflen, 0, - BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ - blake2s_compress_generic(state, state->buf, 1, state->buflen); - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - memcpy(out, state->h, state->outlen); - memzero_explicit(state, sizeof(*state)); - - return 0; -} - -static struct shash_alg blake2s_algs[] = {{ - .base.cra_name = "blake2s-128", - .base.cra_driver_name = "blake2s-128-generic", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_128_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-160", - .base.cra_driver_name = "blake2s-160-generic", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_160_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-224", - .base.cra_driver_name = "blake2s-224-generic", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_224_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-256", - .base.cra_driver_name = "blake2s-256-generic", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_256_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}}; +static struct shash_alg blake2s_algs[] = { + BLAKE2S_ALG("blake2s-128", "blake2s-128-generic", + BLAKE2S_128_HASH_SIZE), + BLAKE2S_ALG("blake2s-160", "blake2s-160-generic", + BLAKE2S_160_HASH_SIZE), + BLAKE2S_ALG("blake2s-224", "blake2s-224-generic", + BLAKE2S_224_HASH_SIZE), + BLAKE2S_ALG("blake2s-256", "blake2s-256-generic", + BLAKE2S_256_HASH_SIZE), +}; static int __init blake2s_mod_init(void) { diff --git a/crypto/blowfish_generic.c b/crypto/blowfish_generic.c index c3c2041fe0c5..003b52c6880e 100644 --- a/crypto/blowfish_generic.c +++ b/crypto/blowfish_generic.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -36,12 +36,10 @@ static void bf_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct bf_ctx *ctx = crypto_tfm_ctx(tfm); - const __be32 *in_blk = (const __be32 *)src; - __be32 *const out_blk = (__be32 *)dst; const u32 *P = ctx->p; const u32 *S = ctx->s; - u32 yl = be32_to_cpu(in_blk[0]); - u32 yr = be32_to_cpu(in_blk[1]); + u32 yl = get_unaligned_be32(src); + u32 yr = get_unaligned_be32(src + 4); ROUND(yr, yl, 0); ROUND(yl, yr, 1); @@ -63,19 +61,17 @@ static void bf_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) yl ^= P[16]; yr ^= P[17]; - out_blk[0] = cpu_to_be32(yr); - out_blk[1] = cpu_to_be32(yl); + put_unaligned_be32(yr, dst); + put_unaligned_be32(yl, dst + 4); } static void bf_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct bf_ctx *ctx = crypto_tfm_ctx(tfm); - const __be32 *in_blk = (const __be32 *)src; - __be32 *const out_blk = (__be32 *)dst; const u32 *P = ctx->p; const u32 *S = ctx->s; - u32 yl = be32_to_cpu(in_blk[0]); - u32 yr = be32_to_cpu(in_blk[1]); + u32 yl = get_unaligned_be32(src); + u32 yr = get_unaligned_be32(src + 4); ROUND(yr, yl, 17); ROUND(yl, yr, 16); @@ -97,8 +93,8 @@ static void bf_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) yl ^= P[1]; yr ^= P[0]; - out_blk[0] = cpu_to_be32(yr); - out_blk[1] = cpu_to_be32(yl); + put_unaligned_be32(yr, dst); + put_unaligned_be32(yl, dst + 4); } static struct crypto_alg alg = { @@ -108,7 +104,6 @@ static struct crypto_alg alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = BF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct bf_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { .cia_min_keysize = BF_MIN_KEY_SIZE, diff --git a/crypto/camellia_generic.c b/crypto/camellia_generic.c index 0b9f409f7370..fd1a88af9e77 100644 --- a/crypto/camellia_generic.c +++ b/crypto/camellia_generic.c @@ -9,14 +9,6 @@ * https://info.isl.ntt.co.jp/crypt/eng/camellia/specifications.html */ -/* - * - * NOTE --- NOTE --- NOTE --- NOTE - * This implementation assumes that all memory addresses passed - * as parameters are four-byte aligned. - * - */ - #include #include #include @@ -994,16 +986,14 @@ camellia_set_key(struct crypto_tfm *tfm, const u8 *in_key, static void camellia_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { const struct camellia_ctx *cctx = crypto_tfm_ctx(tfm); - const __be32 *src = (const __be32 *)in; - __be32 *dst = (__be32 *)out; unsigned int max; u32 tmp[4]; - tmp[0] = be32_to_cpu(src[0]); - tmp[1] = be32_to_cpu(src[1]); - tmp[2] = be32_to_cpu(src[2]); - tmp[3] = be32_to_cpu(src[3]); + tmp[0] = get_unaligned_be32(in); + tmp[1] = get_unaligned_be32(in + 4); + tmp[2] = get_unaligned_be32(in + 8); + tmp[3] = get_unaligned_be32(in + 12); if (cctx->key_length == 16) max = 24; @@ -1013,25 +1003,23 @@ static void camellia_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) camellia_do_encrypt(cctx->key_table, tmp, max); /* do_encrypt returns 0,1 swapped with 2,3 */ - dst[0] = cpu_to_be32(tmp[2]); - dst[1] = cpu_to_be32(tmp[3]); - dst[2] = cpu_to_be32(tmp[0]); - dst[3] = cpu_to_be32(tmp[1]); + put_unaligned_be32(tmp[2], out); + put_unaligned_be32(tmp[3], out + 4); + put_unaligned_be32(tmp[0], out + 8); + put_unaligned_be32(tmp[1], out + 12); } static void camellia_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { const struct camellia_ctx *cctx = crypto_tfm_ctx(tfm); - const __be32 *src = (const __be32 *)in; - __be32 *dst = (__be32 *)out; unsigned int max; u32 tmp[4]; - tmp[0] = be32_to_cpu(src[0]); - tmp[1] = be32_to_cpu(src[1]); - tmp[2] = be32_to_cpu(src[2]); - tmp[3] = be32_to_cpu(src[3]); + tmp[0] = get_unaligned_be32(in); + tmp[1] = get_unaligned_be32(in + 4); + tmp[2] = get_unaligned_be32(in + 8); + tmp[3] = get_unaligned_be32(in + 12); if (cctx->key_length == 16) max = 24; @@ -1041,10 +1029,10 @@ static void camellia_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) camellia_do_decrypt(cctx->key_table, tmp, max); /* do_decrypt returns 0,1 swapped with 2,3 */ - dst[0] = cpu_to_be32(tmp[2]); - dst[1] = cpu_to_be32(tmp[3]); - dst[2] = cpu_to_be32(tmp[0]); - dst[3] = cpu_to_be32(tmp[1]); + put_unaligned_be32(tmp[2], out); + put_unaligned_be32(tmp[3], out + 4); + put_unaligned_be32(tmp[0], out + 8); + put_unaligned_be32(tmp[1], out + 12); } static struct crypto_alg camellia_alg = { @@ -1054,7 +1042,6 @@ static struct crypto_alg camellia_alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = CAMELLIA_BLOCK_SIZE, .cra_ctxsize = sizeof(struct camellia_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { diff --git a/crypto/cast5_generic.c b/crypto/cast5_generic.c index 4095085d4e51..0257c14cefc2 100644 --- a/crypto/cast5_generic.c +++ b/crypto/cast5_generic.c @@ -13,7 +13,7 @@ */ -#include +#include #include #include #include @@ -302,8 +302,6 @@ static const u32 sb8[256] = { void __cast5_encrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) { - const __be32 *src = (const __be32 *)inbuf; - __be32 *dst = (__be32 *)outbuf; u32 l, r, t; u32 I; /* used by the Fx macros */ u32 *Km; @@ -315,8 +313,8 @@ void __cast5_encrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) /* (L0,R0) <-- (m1...m64). (Split the plaintext into left and * right 32-bit halves L0 = m1...m32 and R0 = m33...m64.) */ - l = be32_to_cpu(src[0]); - r = be32_to_cpu(src[1]); + l = get_unaligned_be32(inbuf); + r = get_unaligned_be32(inbuf + 4); /* (16 rounds) for i from 1 to 16, compute Li and Ri as follows: * Li = Ri-1; @@ -347,8 +345,8 @@ void __cast5_encrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) /* c1...c64 <-- (R16,L16). (Exchange final blocks L16, R16 and * concatenate to form the ciphertext.) */ - dst[0] = cpu_to_be32(r); - dst[1] = cpu_to_be32(l); + put_unaligned_be32(r, outbuf); + put_unaligned_be32(l, outbuf + 4); } EXPORT_SYMBOL_GPL(__cast5_encrypt); @@ -359,8 +357,6 @@ static void cast5_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf) void __cast5_decrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) { - const __be32 *src = (const __be32 *)inbuf; - __be32 *dst = (__be32 *)outbuf; u32 l, r, t; u32 I; u32 *Km; @@ -369,8 +365,8 @@ void __cast5_decrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) Km = c->Km; Kr = c->Kr; - l = be32_to_cpu(src[0]); - r = be32_to_cpu(src[1]); + l = get_unaligned_be32(inbuf); + r = get_unaligned_be32(inbuf + 4); if (!(c->rr)) { t = l; l = r; r = t ^ F1(r, Km[15], Kr[15]); @@ -391,8 +387,8 @@ void __cast5_decrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) t = l; l = r; r = t ^ F2(r, Km[1], Kr[1]); t = l; l = r; r = t ^ F1(r, Km[0], Kr[0]); - dst[0] = cpu_to_be32(r); - dst[1] = cpu_to_be32(l); + put_unaligned_be32(r, outbuf); + put_unaligned_be32(l, outbuf + 4); } EXPORT_SYMBOL_GPL(__cast5_decrypt); @@ -513,7 +509,6 @@ static struct crypto_alg alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = CAST5_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast5_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { diff --git a/crypto/cast6_generic.c b/crypto/cast6_generic.c index c77ff6c8a2b2..75346380aa0b 100644 --- a/crypto/cast6_generic.c +++ b/crypto/cast6_generic.c @@ -10,7 +10,7 @@ */ -#include +#include #include #include #include @@ -172,16 +172,14 @@ static inline void QBAR(u32 *block, const u8 *Kr, const u32 *Km) void __cast6_encrypt(const void *ctx, u8 *outbuf, const u8 *inbuf) { const struct cast6_ctx *c = ctx; - const __be32 *src = (const __be32 *)inbuf; - __be32 *dst = (__be32 *)outbuf; u32 block[4]; const u32 *Km; const u8 *Kr; - block[0] = be32_to_cpu(src[0]); - block[1] = be32_to_cpu(src[1]); - block[2] = be32_to_cpu(src[2]); - block[3] = be32_to_cpu(src[3]); + block[0] = get_unaligned_be32(inbuf); + block[1] = get_unaligned_be32(inbuf + 4); + block[2] = get_unaligned_be32(inbuf + 8); + block[3] = get_unaligned_be32(inbuf + 12); Km = c->Km[0]; Kr = c->Kr[0]; Q(block, Kr, Km); Km = c->Km[1]; Kr = c->Kr[1]; Q(block, Kr, Km); @@ -196,10 +194,10 @@ void __cast6_encrypt(const void *ctx, u8 *outbuf, const u8 *inbuf) Km = c->Km[10]; Kr = c->Kr[10]; QBAR(block, Kr, Km); Km = c->Km[11]; Kr = c->Kr[11]; QBAR(block, Kr, Km); - dst[0] = cpu_to_be32(block[0]); - dst[1] = cpu_to_be32(block[1]); - dst[2] = cpu_to_be32(block[2]); - dst[3] = cpu_to_be32(block[3]); + put_unaligned_be32(block[0], outbuf); + put_unaligned_be32(block[1], outbuf + 4); + put_unaligned_be32(block[2], outbuf + 8); + put_unaligned_be32(block[3], outbuf + 12); } EXPORT_SYMBOL_GPL(__cast6_encrypt); @@ -211,16 +209,14 @@ static void cast6_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf) void __cast6_decrypt(const void *ctx, u8 *outbuf, const u8 *inbuf) { const struct cast6_ctx *c = ctx; - const __be32 *src = (const __be32 *)inbuf; - __be32 *dst = (__be32 *)outbuf; u32 block[4]; const u32 *Km; const u8 *Kr; - block[0] = be32_to_cpu(src[0]); - block[1] = be32_to_cpu(src[1]); - block[2] = be32_to_cpu(src[2]); - block[3] = be32_to_cpu(src[3]); + block[0] = get_unaligned_be32(inbuf); + block[1] = get_unaligned_be32(inbuf + 4); + block[2] = get_unaligned_be32(inbuf + 8); + block[3] = get_unaligned_be32(inbuf + 12); Km = c->Km[11]; Kr = c->Kr[11]; Q(block, Kr, Km); Km = c->Km[10]; Kr = c->Kr[10]; Q(block, Kr, Km); @@ -235,10 +231,10 @@ void __cast6_decrypt(const void *ctx, u8 *outbuf, const u8 *inbuf) Km = c->Km[1]; Kr = c->Kr[1]; QBAR(block, Kr, Km); Km = c->Km[0]; Kr = c->Kr[0]; QBAR(block, Kr, Km); - dst[0] = cpu_to_be32(block[0]); - dst[1] = cpu_to_be32(block[1]); - dst[2] = cpu_to_be32(block[2]); - dst[3] = cpu_to_be32(block[3]); + put_unaligned_be32(block[0], outbuf); + put_unaligned_be32(block[1], outbuf + 4); + put_unaligned_be32(block[2], outbuf + 8); + put_unaligned_be32(block[3], outbuf + 12); } EXPORT_SYMBOL_GPL(__cast6_decrypt); @@ -254,7 +250,6 @@ static struct crypto_alg alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { diff --git a/crypto/cbc.c b/crypto/cbc.c index 0d9509dff891..6c03e96b945f 100644 --- a/crypto/cbc.c +++ b/crypto/cbc.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include diff --git a/crypto/ccm.c b/crypto/ccm.c index 494d70901186..6b815ece51c6 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -954,3 +955,4 @@ MODULE_ALIAS_CRYPTO("ccm_base"); MODULE_ALIAS_CRYPTO("rfc4309"); MODULE_ALIAS_CRYPTO("ccm"); MODULE_ALIAS_CRYPTO("cbcmac"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/cfb.c b/crypto/cfb.c index 4e5219bbcd19..0d664dfb47bc 100644 --- a/crypto/cfb.c +++ b/crypto/cfb.c @@ -20,6 +20,7 @@ */ #include +#include #include #include #include @@ -250,3 +251,4 @@ module_exit(crypto_cfb_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("CFB block cipher mode of operation"); MODULE_ALIAS_CRYPTO("cfb"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/cipher.c b/crypto/cipher.c index fd78150deb1c..b47141ed4a9f 100644 --- a/crypto/cipher.c +++ b/crypto/cipher.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -53,7 +54,7 @@ int crypto_cipher_setkey(struct crypto_cipher *tfm, return cia->cia_setkey(crypto_cipher_tfm(tfm), key, keylen); } -EXPORT_SYMBOL_GPL(crypto_cipher_setkey); +EXPORT_SYMBOL_NS_GPL(crypto_cipher_setkey, CRYPTO_INTERNAL); static inline void cipher_crypt_one(struct crypto_cipher *tfm, u8 *dst, const u8 *src, bool enc) @@ -81,11 +82,11 @@ void crypto_cipher_encrypt_one(struct crypto_cipher *tfm, { cipher_crypt_one(tfm, dst, src, true); } -EXPORT_SYMBOL_GPL(crypto_cipher_encrypt_one); +EXPORT_SYMBOL_NS_GPL(crypto_cipher_encrypt_one, CRYPTO_INTERNAL); void crypto_cipher_decrypt_one(struct crypto_cipher *tfm, u8 *dst, const u8 *src) { cipher_crypt_one(tfm, dst, src, false); } -EXPORT_SYMBOL_GPL(crypto_cipher_decrypt_one); +EXPORT_SYMBOL_NS_GPL(crypto_cipher_decrypt_one, CRYPTO_INTERNAL); diff --git a/crypto/cmac.c b/crypto/cmac.c index df36be1efb81..f4a5d3bfb376 100644 --- a/crypto/cmac.c +++ b/crypto/cmac.c @@ -11,6 +11,7 @@ * Author: Kazunori Miyazawa */ +#include #include #include #include @@ -313,3 +314,4 @@ module_exit(crypto_cmac_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("CMAC keyed hash algorithm"); MODULE_ALIAS_CRYPTO("cmac"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/ctr.c b/crypto/ctr.c index c39fcffba27f..23c698b22013 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -358,3 +359,4 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("CTR block cipher mode of operation"); MODULE_ALIAS_CRYPTO("rfc3686"); MODULE_ALIAS_CRYPTO("ctr"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/drbg.c b/crypto/drbg.c index 3132967a1749..1b4587e0ddad 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -98,6 +98,7 @@ */ #include +#include #include /*************************************************************** @@ -2161,3 +2162,4 @@ MODULE_DESCRIPTION("NIST SP800-90A Deterministic Random Bit Generator (DRBG) " CRYPTO_DRBG_HMAC_STRING CRYPTO_DRBG_CTR_STRING); MODULE_ALIAS_CRYPTO("stdrng"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/ecb.c b/crypto/ecb.c index 69a687cbdf21..71fbb0543d64 100644 --- a/crypto/ecb.c +++ b/crypto/ecb.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include diff --git a/crypto/ecdh_helper.c b/crypto/ecdh_helper.c index 66fcb2ea8154..fca63b559f65 100644 --- a/crypto/ecdh_helper.c +++ b/crypto/ecdh_helper.c @@ -67,6 +67,9 @@ int crypto_ecdh_decode_key(const char *buf, unsigned int len, if (secret.type != CRYPTO_KPP_SECRET_TYPE_ECDH) return -EINVAL; + if (unlikely(len < secret.len)) + return -EINVAL; + ptr = ecdh_unpack_data(¶ms->curve_id, ptr, sizeof(params->curve_id)); ptr = ecdh_unpack_data(¶ms->key_size, ptr, sizeof(params->key_size)); if (secret.len != crypto_ecdh_key_len(params)) diff --git a/crypto/essiv.c b/crypto/essiv.c index d012be23d496..8bcc5bdcb2a9 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -643,3 +644,4 @@ module_exit(essiv_module_exit); MODULE_DESCRIPTION("ESSIV skcipher/aead wrapper for block encryption"); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("essiv"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c index 58f935315cf8..c36ea0c8be98 100644 --- a/crypto/fcrypt.c +++ b/crypto/fcrypt.c @@ -396,7 +396,6 @@ static struct crypto_alg fcrypt_alg = { .cra_blocksize = 8, .cra_ctxsize = sizeof(struct fcrypt_ctx), .cra_module = THIS_MODULE, - .cra_alignmask = 3, .cra_u = { .cipher = { .cia_min_keysize = 8, .cia_max_keysize = 8, diff --git a/crypto/keywrap.c b/crypto/keywrap.c index 0355cce21b1e..3517773bc7f7 100644 --- a/crypto/keywrap.c +++ b/crypto/keywrap.c @@ -85,6 +85,7 @@ #include #include #include +#include #include struct crypto_kw_block { @@ -316,3 +317,4 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Stephan Mueller "); MODULE_DESCRIPTION("Key Wrapping (RFC3394 / NIST SP800-38F)"); MODULE_ALIAS_CRYPTO("kw"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/michael_mic.c b/crypto/michael_mic.c index 63350c4ad461..f4c31049601c 100644 --- a/crypto/michael_mic.c +++ b/crypto/michael_mic.c @@ -7,7 +7,7 @@ * Copyright (c) 2004 Jouni Malinen */ #include -#include +#include #include #include #include @@ -19,7 +19,7 @@ struct michael_mic_ctx { }; struct michael_mic_desc_ctx { - u8 pending[4]; + __le32 pending; size_t pending_len; u32 l, r; @@ -60,13 +60,12 @@ static int michael_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct michael_mic_desc_ctx *mctx = shash_desc_ctx(desc); - const __le32 *src; if (mctx->pending_len) { int flen = 4 - mctx->pending_len; if (flen > len) flen = len; - memcpy(&mctx->pending[mctx->pending_len], data, flen); + memcpy((u8 *)&mctx->pending + mctx->pending_len, data, flen); mctx->pending_len += flen; data += flen; len -= flen; @@ -74,23 +73,21 @@ static int michael_update(struct shash_desc *desc, const u8 *data, if (mctx->pending_len < 4) return 0; - src = (const __le32 *)mctx->pending; - mctx->l ^= le32_to_cpup(src); + mctx->l ^= le32_to_cpu(mctx->pending); michael_block(mctx->l, mctx->r); mctx->pending_len = 0; } - src = (const __le32 *)data; - while (len >= 4) { - mctx->l ^= le32_to_cpup(src++); + mctx->l ^= get_unaligned_le32(data); michael_block(mctx->l, mctx->r); + data += 4; len -= 4; } if (len > 0) { mctx->pending_len = len; - memcpy(mctx->pending, src, len); + memcpy(&mctx->pending, data, len); } return 0; @@ -100,8 +97,7 @@ static int michael_update(struct shash_desc *desc, const u8 *data, static int michael_final(struct shash_desc *desc, u8 *out) { struct michael_mic_desc_ctx *mctx = shash_desc_ctx(desc); - u8 *data = mctx->pending; - __le32 *dst = (__le32 *)out; + u8 *data = (u8 *)&mctx->pending; /* Last block and padding (0x5a, 4..7 x 0) */ switch (mctx->pending_len) { @@ -123,8 +119,8 @@ static int michael_final(struct shash_desc *desc, u8 *out) /* l ^= 0; */ michael_block(mctx->l, mctx->r); - dst[0] = cpu_to_le32(mctx->l); - dst[1] = cpu_to_le32(mctx->r); + put_unaligned_le32(mctx->l, out); + put_unaligned_le32(mctx->r, out + 4); return 0; } @@ -135,13 +131,11 @@ static int michael_setkey(struct crypto_shash *tfm, const u8 *key, { struct michael_mic_ctx *mctx = crypto_shash_ctx(tfm); - const __le32 *data = (const __le32 *)key; - if (keylen != 8) return -EINVAL; - mctx->l = le32_to_cpu(data[0]); - mctx->r = le32_to_cpu(data[1]); + mctx->l = get_unaligned_le32(key); + mctx->r = get_unaligned_le32(key + 4); return 0; } @@ -156,7 +150,6 @@ static struct shash_alg alg = { .cra_name = "michael_mic", .cra_driver_name = "michael_mic-generic", .cra_blocksize = 8, - .cra_alignmask = 3, .cra_ctxsize = sizeof(struct michael_mic_ctx), .cra_module = THIS_MODULE, } diff --git a/crypto/ofb.c b/crypto/ofb.c index 2ec68e3f2c55..b630fdecceee 100644 --- a/crypto/ofb.c +++ b/crypto/ofb.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -102,3 +103,4 @@ module_exit(crypto_ofb_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("OFB block cipher mode of operation"); MODULE_ALIAS_CRYPTO("ofb"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/pcbc.c b/crypto/pcbc.c index ae921fb74dc9..7030f59e46b6 100644 --- a/crypto/pcbc.c +++ b/crypto/pcbc.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -191,3 +192,4 @@ module_exit(crypto_pcbc_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("PCBC block cipher mode of operation"); MODULE_ALIAS_CRYPTO("pcbc"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/ripemd.h b/crypto/ripemd.h index 93edbf52197d..b977785e2a62 100644 --- a/crypto/ripemd.h +++ b/crypto/ripemd.h @@ -6,29 +6,15 @@ #ifndef _CRYPTO_RMD_H #define _CRYPTO_RMD_H -#define RMD128_DIGEST_SIZE 16 -#define RMD128_BLOCK_SIZE 64 - #define RMD160_DIGEST_SIZE 20 #define RMD160_BLOCK_SIZE 64 -#define RMD256_DIGEST_SIZE 32 -#define RMD256_BLOCK_SIZE 64 - -#define RMD320_DIGEST_SIZE 40 -#define RMD320_BLOCK_SIZE 64 - /* initial values */ #define RMD_H0 0x67452301UL #define RMD_H1 0xefcdab89UL #define RMD_H2 0x98badcfeUL #define RMD_H3 0x10325476UL #define RMD_H4 0xc3d2e1f0UL -#define RMD_H5 0x76543210UL -#define RMD_H6 0xfedcba98UL -#define RMD_H7 0x89abcdefUL -#define RMD_H8 0x01234567UL -#define RMD_H9 0x3c2d1e0fUL /* constants */ #define RMD_K1 0x00000000UL diff --git a/crypto/rmd128.c b/crypto/rmd128.c deleted file mode 100644 index 29308fb97e7e..000000000000 --- a/crypto/rmd128.c +++ /dev/null @@ -1,323 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * RIPEMD-128 - RACE Integrity Primitives Evaluation Message Digest. - * - * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC - * - * Copyright (c) 2008 Adrian-Ken Rueegsegger - */ -#include -#include -#include -#include -#include -#include - -#include "ripemd.h" - -struct rmd128_ctx { - u64 byte_count; - u32 state[4]; - __le32 buffer[16]; -}; - -#define K1 RMD_K1 -#define K2 RMD_K2 -#define K3 RMD_K3 -#define K4 RMD_K4 -#define KK1 RMD_K6 -#define KK2 RMD_K7 -#define KK3 RMD_K8 -#define KK4 RMD_K1 - -#define F1(x, y, z) (x ^ y ^ z) /* XOR */ -#define F2(x, y, z) (z ^ (x & (y ^ z))) /* x ? y : z */ -#define F3(x, y, z) ((x | ~y) ^ z) -#define F4(x, y, z) (y ^ (z & (x ^ y))) /* z ? x : y */ - -#define ROUND(a, b, c, d, f, k, x, s) { \ - (a) += f((b), (c), (d)) + le32_to_cpup(&(x)) + (k); \ - (a) = rol32((a), (s)); \ -} - -static void rmd128_transform(u32 *state, const __le32 *in) -{ - u32 aa, bb, cc, dd, aaa, bbb, ccc, ddd; - - /* Initialize left lane */ - aa = state[0]; - bb = state[1]; - cc = state[2]; - dd = state[3]; - - /* Initialize right lane */ - aaa = state[0]; - bbb = state[1]; - ccc = state[2]; - ddd = state[3]; - - /* round 1: left lane */ - ROUND(aa, bb, cc, dd, F1, K1, in[0], 11); - ROUND(dd, aa, bb, cc, F1, K1, in[1], 14); - ROUND(cc, dd, aa, bb, F1, K1, in[2], 15); - ROUND(bb, cc, dd, aa, F1, K1, in[3], 12); - ROUND(aa, bb, cc, dd, F1, K1, in[4], 5); - ROUND(dd, aa, bb, cc, F1, K1, in[5], 8); - ROUND(cc, dd, aa, bb, F1, K1, in[6], 7); - ROUND(bb, cc, dd, aa, F1, K1, in[7], 9); - ROUND(aa, bb, cc, dd, F1, K1, in[8], 11); - ROUND(dd, aa, bb, cc, F1, K1, in[9], 13); - ROUND(cc, dd, aa, bb, F1, K1, in[10], 14); - ROUND(bb, cc, dd, aa, F1, K1, in[11], 15); - ROUND(aa, bb, cc, dd, F1, K1, in[12], 6); - ROUND(dd, aa, bb, cc, F1, K1, in[13], 7); - ROUND(cc, dd, aa, bb, F1, K1, in[14], 9); - ROUND(bb, cc, dd, aa, F1, K1, in[15], 8); - - /* round 2: left lane */ - ROUND(aa, bb, cc, dd, F2, K2, in[7], 7); - ROUND(dd, aa, bb, cc, F2, K2, in[4], 6); - ROUND(cc, dd, aa, bb, F2, K2, in[13], 8); - ROUND(bb, cc, dd, aa, F2, K2, in[1], 13); - ROUND(aa, bb, cc, dd, F2, K2, in[10], 11); - ROUND(dd, aa, bb, cc, F2, K2, in[6], 9); - ROUND(cc, dd, aa, bb, F2, K2, in[15], 7); - ROUND(bb, cc, dd, aa, F2, K2, in[3], 15); - ROUND(aa, bb, cc, dd, F2, K2, in[12], 7); - ROUND(dd, aa, bb, cc, F2, K2, in[0], 12); - ROUND(cc, dd, aa, bb, F2, K2, in[9], 15); - ROUND(bb, cc, dd, aa, F2, K2, in[5], 9); - ROUND(aa, bb, cc, dd, F2, K2, in[2], 11); - ROUND(dd, aa, bb, cc, F2, K2, in[14], 7); - ROUND(cc, dd, aa, bb, F2, K2, in[11], 13); - ROUND(bb, cc, dd, aa, F2, K2, in[8], 12); - - /* round 3: left lane */ - ROUND(aa, bb, cc, dd, F3, K3, in[3], 11); - ROUND(dd, aa, bb, cc, F3, K3, in[10], 13); - ROUND(cc, dd, aa, bb, F3, K3, in[14], 6); - ROUND(bb, cc, dd, aa, F3, K3, in[4], 7); - ROUND(aa, bb, cc, dd, F3, K3, in[9], 14); - ROUND(dd, aa, bb, cc, F3, K3, in[15], 9); - ROUND(cc, dd, aa, bb, F3, K3, in[8], 13); - ROUND(bb, cc, dd, aa, F3, K3, in[1], 15); - ROUND(aa, bb, cc, dd, F3, K3, in[2], 14); - ROUND(dd, aa, bb, cc, F3, K3, in[7], 8); - ROUND(cc, dd, aa, bb, F3, K3, in[0], 13); - ROUND(bb, cc, dd, aa, F3, K3, in[6], 6); - ROUND(aa, bb, cc, dd, F3, K3, in[13], 5); - ROUND(dd, aa, bb, cc, F3, K3, in[11], 12); - ROUND(cc, dd, aa, bb, F3, K3, in[5], 7); - ROUND(bb, cc, dd, aa, F3, K3, in[12], 5); - - /* round 4: left lane */ - ROUND(aa, bb, cc, dd, F4, K4, in[1], 11); - ROUND(dd, aa, bb, cc, F4, K4, in[9], 12); - ROUND(cc, dd, aa, bb, F4, K4, in[11], 14); - ROUND(bb, cc, dd, aa, F4, K4, in[10], 15); - ROUND(aa, bb, cc, dd, F4, K4, in[0], 14); - ROUND(dd, aa, bb, cc, F4, K4, in[8], 15); - ROUND(cc, dd, aa, bb, F4, K4, in[12], 9); - ROUND(bb, cc, dd, aa, F4, K4, in[4], 8); - ROUND(aa, bb, cc, dd, F4, K4, in[13], 9); - ROUND(dd, aa, bb, cc, F4, K4, in[3], 14); - ROUND(cc, dd, aa, bb, F4, K4, in[7], 5); - ROUND(bb, cc, dd, aa, F4, K4, in[15], 6); - ROUND(aa, bb, cc, dd, F4, K4, in[14], 8); - ROUND(dd, aa, bb, cc, F4, K4, in[5], 6); - ROUND(cc, dd, aa, bb, F4, K4, in[6], 5); - ROUND(bb, cc, dd, aa, F4, K4, in[2], 12); - - /* round 1: right lane */ - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[5], 8); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[14], 9); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[7], 9); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[0], 11); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[9], 13); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[2], 15); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[11], 15); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[4], 5); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[13], 7); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[6], 7); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[15], 8); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[8], 11); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[1], 14); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[10], 14); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[3], 12); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[12], 6); - - /* round 2: right lane */ - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[6], 9); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[11], 13); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[3], 15); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[7], 7); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[0], 12); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[13], 8); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[5], 9); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[10], 11); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[14], 7); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[15], 7); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[8], 12); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[12], 7); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[4], 6); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[9], 15); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[1], 13); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[2], 11); - - /* round 3: right lane */ - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[15], 9); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[5], 7); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[1], 15); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[3], 11); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[7], 8); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[14], 6); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[6], 6); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[9], 14); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[11], 12); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[8], 13); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[12], 5); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[2], 14); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[10], 13); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[0], 13); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[4], 7); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[13], 5); - - /* round 4: right lane */ - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[8], 15); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[6], 5); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[4], 8); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[1], 11); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[3], 14); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[11], 14); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[15], 6); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[0], 14); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[5], 6); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[12], 9); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[2], 12); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[13], 9); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[9], 12); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[7], 5); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[10], 15); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[14], 8); - - /* combine results */ - ddd += cc + state[1]; /* final result for state[0] */ - state[1] = state[2] + dd + aaa; - state[2] = state[3] + aa + bbb; - state[3] = state[0] + bb + ccc; - state[0] = ddd; -} - -static int rmd128_init(struct shash_desc *desc) -{ - struct rmd128_ctx *rctx = shash_desc_ctx(desc); - - rctx->byte_count = 0; - - rctx->state[0] = RMD_H0; - rctx->state[1] = RMD_H1; - rctx->state[2] = RMD_H2; - rctx->state[3] = RMD_H3; - - memset(rctx->buffer, 0, sizeof(rctx->buffer)); - - return 0; -} - -static int rmd128_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct rmd128_ctx *rctx = shash_desc_ctx(desc); - const u32 avail = sizeof(rctx->buffer) - (rctx->byte_count & 0x3f); - - rctx->byte_count += len; - - /* Enough space in buffer? If so copy and we're done */ - if (avail > len) { - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, len); - goto out; - } - - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, avail); - - rmd128_transform(rctx->state, rctx->buffer); - data += avail; - len -= avail; - - while (len >= sizeof(rctx->buffer)) { - memcpy(rctx->buffer, data, sizeof(rctx->buffer)); - rmd128_transform(rctx->state, rctx->buffer); - data += sizeof(rctx->buffer); - len -= sizeof(rctx->buffer); - } - - memcpy(rctx->buffer, data, len); - -out: - return 0; -} - -/* Add padding and return the message digest. */ -static int rmd128_final(struct shash_desc *desc, u8 *out) -{ - struct rmd128_ctx *rctx = shash_desc_ctx(desc); - u32 i, index, padlen; - __le64 bits; - __le32 *dst = (__le32 *)out; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_le64(rctx->byte_count << 3); - - /* Pad out to 56 mod 64 */ - index = rctx->byte_count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - rmd128_update(desc, padding, padlen); - - /* Append length */ - rmd128_update(desc, (const u8 *)&bits, sizeof(bits)); - - /* Store state in digest */ - for (i = 0; i < 4; i++) - dst[i] = cpu_to_le32p(&rctx->state[i]); - - /* Wipe context */ - memset(rctx, 0, sizeof(*rctx)); - - return 0; -} - -static struct shash_alg alg = { - .digestsize = RMD128_DIGEST_SIZE, - .init = rmd128_init, - .update = rmd128_update, - .final = rmd128_final, - .descsize = sizeof(struct rmd128_ctx), - .base = { - .cra_name = "rmd128", - .cra_driver_name = "rmd128-generic", - .cra_blocksize = RMD128_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init rmd128_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit rmd128_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -subsys_initcall(rmd128_mod_init); -module_exit(rmd128_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Adrian-Ken Rueegsegger "); -MODULE_DESCRIPTION("RIPEMD-128 Message Digest"); -MODULE_ALIAS_CRYPTO("rmd128"); diff --git a/crypto/rmd256.c b/crypto/rmd256.c deleted file mode 100644 index 3c730e9de5fd..000000000000 --- a/crypto/rmd256.c +++ /dev/null @@ -1,342 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * RIPEMD-256 - RACE Integrity Primitives Evaluation Message Digest. - * - * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC - * - * Copyright (c) 2008 Adrian-Ken Rueegsegger - */ -#include -#include -#include -#include -#include -#include - -#include "ripemd.h" - -struct rmd256_ctx { - u64 byte_count; - u32 state[8]; - __le32 buffer[16]; -}; - -#define K1 RMD_K1 -#define K2 RMD_K2 -#define K3 RMD_K3 -#define K4 RMD_K4 -#define KK1 RMD_K6 -#define KK2 RMD_K7 -#define KK3 RMD_K8 -#define KK4 RMD_K1 - -#define F1(x, y, z) (x ^ y ^ z) /* XOR */ -#define F2(x, y, z) (z ^ (x & (y ^ z))) /* x ? y : z */ -#define F3(x, y, z) ((x | ~y) ^ z) -#define F4(x, y, z) (y ^ (z & (x ^ y))) /* z ? x : y */ - -#define ROUND(a, b, c, d, f, k, x, s) { \ - (a) += f((b), (c), (d)) + le32_to_cpup(&(x)) + (k); \ - (a) = rol32((a), (s)); \ -} - -static void rmd256_transform(u32 *state, const __le32 *in) -{ - u32 aa, bb, cc, dd, aaa, bbb, ccc, ddd; - - /* Initialize left lane */ - aa = state[0]; - bb = state[1]; - cc = state[2]; - dd = state[3]; - - /* Initialize right lane */ - aaa = state[4]; - bbb = state[5]; - ccc = state[6]; - ddd = state[7]; - - /* round 1: left lane */ - ROUND(aa, bb, cc, dd, F1, K1, in[0], 11); - ROUND(dd, aa, bb, cc, F1, K1, in[1], 14); - ROUND(cc, dd, aa, bb, F1, K1, in[2], 15); - ROUND(bb, cc, dd, aa, F1, K1, in[3], 12); - ROUND(aa, bb, cc, dd, F1, K1, in[4], 5); - ROUND(dd, aa, bb, cc, F1, K1, in[5], 8); - ROUND(cc, dd, aa, bb, F1, K1, in[6], 7); - ROUND(bb, cc, dd, aa, F1, K1, in[7], 9); - ROUND(aa, bb, cc, dd, F1, K1, in[8], 11); - ROUND(dd, aa, bb, cc, F1, K1, in[9], 13); - ROUND(cc, dd, aa, bb, F1, K1, in[10], 14); - ROUND(bb, cc, dd, aa, F1, K1, in[11], 15); - ROUND(aa, bb, cc, dd, F1, K1, in[12], 6); - ROUND(dd, aa, bb, cc, F1, K1, in[13], 7); - ROUND(cc, dd, aa, bb, F1, K1, in[14], 9); - ROUND(bb, cc, dd, aa, F1, K1, in[15], 8); - - /* round 1: right lane */ - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[5], 8); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[14], 9); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[7], 9); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[0], 11); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[9], 13); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[2], 15); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[11], 15); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[4], 5); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[13], 7); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[6], 7); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[15], 8); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[8], 11); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[1], 14); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[10], 14); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[3], 12); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[12], 6); - - /* Swap contents of "a" registers */ - swap(aa, aaa); - - /* round 2: left lane */ - ROUND(aa, bb, cc, dd, F2, K2, in[7], 7); - ROUND(dd, aa, bb, cc, F2, K2, in[4], 6); - ROUND(cc, dd, aa, bb, F2, K2, in[13], 8); - ROUND(bb, cc, dd, aa, F2, K2, in[1], 13); - ROUND(aa, bb, cc, dd, F2, K2, in[10], 11); - ROUND(dd, aa, bb, cc, F2, K2, in[6], 9); - ROUND(cc, dd, aa, bb, F2, K2, in[15], 7); - ROUND(bb, cc, dd, aa, F2, K2, in[3], 15); - ROUND(aa, bb, cc, dd, F2, K2, in[12], 7); - ROUND(dd, aa, bb, cc, F2, K2, in[0], 12); - ROUND(cc, dd, aa, bb, F2, K2, in[9], 15); - ROUND(bb, cc, dd, aa, F2, K2, in[5], 9); - ROUND(aa, bb, cc, dd, F2, K2, in[2], 11); - ROUND(dd, aa, bb, cc, F2, K2, in[14], 7); - ROUND(cc, dd, aa, bb, F2, K2, in[11], 13); - ROUND(bb, cc, dd, aa, F2, K2, in[8], 12); - - /* round 2: right lane */ - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[6], 9); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[11], 13); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[3], 15); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[7], 7); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[0], 12); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[13], 8); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[5], 9); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[10], 11); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[14], 7); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[15], 7); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[8], 12); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[12], 7); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[4], 6); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[9], 15); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[1], 13); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[2], 11); - - /* Swap contents of "b" registers */ - swap(bb, bbb); - - /* round 3: left lane */ - ROUND(aa, bb, cc, dd, F3, K3, in[3], 11); - ROUND(dd, aa, bb, cc, F3, K3, in[10], 13); - ROUND(cc, dd, aa, bb, F3, K3, in[14], 6); - ROUND(bb, cc, dd, aa, F3, K3, in[4], 7); - ROUND(aa, bb, cc, dd, F3, K3, in[9], 14); - ROUND(dd, aa, bb, cc, F3, K3, in[15], 9); - ROUND(cc, dd, aa, bb, F3, K3, in[8], 13); - ROUND(bb, cc, dd, aa, F3, K3, in[1], 15); - ROUND(aa, bb, cc, dd, F3, K3, in[2], 14); - ROUND(dd, aa, bb, cc, F3, K3, in[7], 8); - ROUND(cc, dd, aa, bb, F3, K3, in[0], 13); - ROUND(bb, cc, dd, aa, F3, K3, in[6], 6); - ROUND(aa, bb, cc, dd, F3, K3, in[13], 5); - ROUND(dd, aa, bb, cc, F3, K3, in[11], 12); - ROUND(cc, dd, aa, bb, F3, K3, in[5], 7); - ROUND(bb, cc, dd, aa, F3, K3, in[12], 5); - - /* round 3: right lane */ - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[15], 9); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[5], 7); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[1], 15); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[3], 11); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[7], 8); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[14], 6); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[6], 6); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[9], 14); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[11], 12); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[8], 13); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[12], 5); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[2], 14); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[10], 13); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[0], 13); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[4], 7); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[13], 5); - - /* Swap contents of "c" registers */ - swap(cc, ccc); - - /* round 4: left lane */ - ROUND(aa, bb, cc, dd, F4, K4, in[1], 11); - ROUND(dd, aa, bb, cc, F4, K4, in[9], 12); - ROUND(cc, dd, aa, bb, F4, K4, in[11], 14); - ROUND(bb, cc, dd, aa, F4, K4, in[10], 15); - ROUND(aa, bb, cc, dd, F4, K4, in[0], 14); - ROUND(dd, aa, bb, cc, F4, K4, in[8], 15); - ROUND(cc, dd, aa, bb, F4, K4, in[12], 9); - ROUND(bb, cc, dd, aa, F4, K4, in[4], 8); - ROUND(aa, bb, cc, dd, F4, K4, in[13], 9); - ROUND(dd, aa, bb, cc, F4, K4, in[3], 14); - ROUND(cc, dd, aa, bb, F4, K4, in[7], 5); - ROUND(bb, cc, dd, aa, F4, K4, in[15], 6); - ROUND(aa, bb, cc, dd, F4, K4, in[14], 8); - ROUND(dd, aa, bb, cc, F4, K4, in[5], 6); - ROUND(cc, dd, aa, bb, F4, K4, in[6], 5); - ROUND(bb, cc, dd, aa, F4, K4, in[2], 12); - - /* round 4: right lane */ - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[8], 15); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[6], 5); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[4], 8); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[1], 11); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[3], 14); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[11], 14); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[15], 6); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[0], 14); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[5], 6); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[12], 9); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[2], 12); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[13], 9); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[9], 12); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[7], 5); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[10], 15); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[14], 8); - - /* Swap contents of "d" registers */ - swap(dd, ddd); - - /* combine results */ - state[0] += aa; - state[1] += bb; - state[2] += cc; - state[3] += dd; - state[4] += aaa; - state[5] += bbb; - state[6] += ccc; - state[7] += ddd; -} - -static int rmd256_init(struct shash_desc *desc) -{ - struct rmd256_ctx *rctx = shash_desc_ctx(desc); - - rctx->byte_count = 0; - - rctx->state[0] = RMD_H0; - rctx->state[1] = RMD_H1; - rctx->state[2] = RMD_H2; - rctx->state[3] = RMD_H3; - rctx->state[4] = RMD_H5; - rctx->state[5] = RMD_H6; - rctx->state[6] = RMD_H7; - rctx->state[7] = RMD_H8; - - memset(rctx->buffer, 0, sizeof(rctx->buffer)); - - return 0; -} - -static int rmd256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct rmd256_ctx *rctx = shash_desc_ctx(desc); - const u32 avail = sizeof(rctx->buffer) - (rctx->byte_count & 0x3f); - - rctx->byte_count += len; - - /* Enough space in buffer? If so copy and we're done */ - if (avail > len) { - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, len); - goto out; - } - - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, avail); - - rmd256_transform(rctx->state, rctx->buffer); - data += avail; - len -= avail; - - while (len >= sizeof(rctx->buffer)) { - memcpy(rctx->buffer, data, sizeof(rctx->buffer)); - rmd256_transform(rctx->state, rctx->buffer); - data += sizeof(rctx->buffer); - len -= sizeof(rctx->buffer); - } - - memcpy(rctx->buffer, data, len); - -out: - return 0; -} - -/* Add padding and return the message digest. */ -static int rmd256_final(struct shash_desc *desc, u8 *out) -{ - struct rmd256_ctx *rctx = shash_desc_ctx(desc); - u32 i, index, padlen; - __le64 bits; - __le32 *dst = (__le32 *)out; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_le64(rctx->byte_count << 3); - - /* Pad out to 56 mod 64 */ - index = rctx->byte_count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - rmd256_update(desc, padding, padlen); - - /* Append length */ - rmd256_update(desc, (const u8 *)&bits, sizeof(bits)); - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_le32p(&rctx->state[i]); - - /* Wipe context */ - memset(rctx, 0, sizeof(*rctx)); - - return 0; -} - -static struct shash_alg alg = { - .digestsize = RMD256_DIGEST_SIZE, - .init = rmd256_init, - .update = rmd256_update, - .final = rmd256_final, - .descsize = sizeof(struct rmd256_ctx), - .base = { - .cra_name = "rmd256", - .cra_driver_name = "rmd256-generic", - .cra_blocksize = RMD256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init rmd256_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit rmd256_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -subsys_initcall(rmd256_mod_init); -module_exit(rmd256_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Adrian-Ken Rueegsegger "); -MODULE_DESCRIPTION("RIPEMD-256 Message Digest"); -MODULE_ALIAS_CRYPTO("rmd256"); diff --git a/crypto/rmd320.c b/crypto/rmd320.c deleted file mode 100644 index c919ad6c4705..000000000000 --- a/crypto/rmd320.c +++ /dev/null @@ -1,391 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * RIPEMD-320 - RACE Integrity Primitives Evaluation Message Digest. - * - * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC - * - * Copyright (c) 2008 Adrian-Ken Rueegsegger - */ -#include -#include -#include -#include -#include -#include - -#include "ripemd.h" - -struct rmd320_ctx { - u64 byte_count; - u32 state[10]; - __le32 buffer[16]; -}; - -#define K1 RMD_K1 -#define K2 RMD_K2 -#define K3 RMD_K3 -#define K4 RMD_K4 -#define K5 RMD_K5 -#define KK1 RMD_K6 -#define KK2 RMD_K7 -#define KK3 RMD_K8 -#define KK4 RMD_K9 -#define KK5 RMD_K1 - -#define F1(x, y, z) (x ^ y ^ z) /* XOR */ -#define F2(x, y, z) (z ^ (x & (y ^ z))) /* x ? y : z */ -#define F3(x, y, z) ((x | ~y) ^ z) -#define F4(x, y, z) (y ^ (z & (x ^ y))) /* z ? x : y */ -#define F5(x, y, z) (x ^ (y | ~z)) - -#define ROUND(a, b, c, d, e, f, k, x, s) { \ - (a) += f((b), (c), (d)) + le32_to_cpup(&(x)) + (k); \ - (a) = rol32((a), (s)) + (e); \ - (c) = rol32((c), 10); \ -} - -static void rmd320_transform(u32 *state, const __le32 *in) -{ - u32 aa, bb, cc, dd, ee, aaa, bbb, ccc, ddd, eee; - - /* Initialize left lane */ - aa = state[0]; - bb = state[1]; - cc = state[2]; - dd = state[3]; - ee = state[4]; - - /* Initialize right lane */ - aaa = state[5]; - bbb = state[6]; - ccc = state[7]; - ddd = state[8]; - eee = state[9]; - - /* round 1: left lane */ - ROUND(aa, bb, cc, dd, ee, F1, K1, in[0], 11); - ROUND(ee, aa, bb, cc, dd, F1, K1, in[1], 14); - ROUND(dd, ee, aa, bb, cc, F1, K1, in[2], 15); - ROUND(cc, dd, ee, aa, bb, F1, K1, in[3], 12); - ROUND(bb, cc, dd, ee, aa, F1, K1, in[4], 5); - ROUND(aa, bb, cc, dd, ee, F1, K1, in[5], 8); - ROUND(ee, aa, bb, cc, dd, F1, K1, in[6], 7); - ROUND(dd, ee, aa, bb, cc, F1, K1, in[7], 9); - ROUND(cc, dd, ee, aa, bb, F1, K1, in[8], 11); - ROUND(bb, cc, dd, ee, aa, F1, K1, in[9], 13); - ROUND(aa, bb, cc, dd, ee, F1, K1, in[10], 14); - ROUND(ee, aa, bb, cc, dd, F1, K1, in[11], 15); - ROUND(dd, ee, aa, bb, cc, F1, K1, in[12], 6); - ROUND(cc, dd, ee, aa, bb, F1, K1, in[13], 7); - ROUND(bb, cc, dd, ee, aa, F1, K1, in[14], 9); - ROUND(aa, bb, cc, dd, ee, F1, K1, in[15], 8); - - /* round 1: right lane */ - ROUND(aaa, bbb, ccc, ddd, eee, F5, KK1, in[5], 8); - ROUND(eee, aaa, bbb, ccc, ddd, F5, KK1, in[14], 9); - ROUND(ddd, eee, aaa, bbb, ccc, F5, KK1, in[7], 9); - ROUND(ccc, ddd, eee, aaa, bbb, F5, KK1, in[0], 11); - ROUND(bbb, ccc, ddd, eee, aaa, F5, KK1, in[9], 13); - ROUND(aaa, bbb, ccc, ddd, eee, F5, KK1, in[2], 15); - ROUND(eee, aaa, bbb, ccc, ddd, F5, KK1, in[11], 15); - ROUND(ddd, eee, aaa, bbb, ccc, F5, KK1, in[4], 5); - ROUND(ccc, ddd, eee, aaa, bbb, F5, KK1, in[13], 7); - ROUND(bbb, ccc, ddd, eee, aaa, F5, KK1, in[6], 7); - ROUND(aaa, bbb, ccc, ddd, eee, F5, KK1, in[15], 8); - ROUND(eee, aaa, bbb, ccc, ddd, F5, KK1, in[8], 11); - ROUND(ddd, eee, aaa, bbb, ccc, F5, KK1, in[1], 14); - ROUND(ccc, ddd, eee, aaa, bbb, F5, KK1, in[10], 14); - ROUND(bbb, ccc, ddd, eee, aaa, F5, KK1, in[3], 12); - ROUND(aaa, bbb, ccc, ddd, eee, F5, KK1, in[12], 6); - - /* Swap contents of "a" registers */ - swap(aa, aaa); - - /* round 2: left lane" */ - ROUND(ee, aa, bb, cc, dd, F2, K2, in[7], 7); - ROUND(dd, ee, aa, bb, cc, F2, K2, in[4], 6); - ROUND(cc, dd, ee, aa, bb, F2, K2, in[13], 8); - ROUND(bb, cc, dd, ee, aa, F2, K2, in[1], 13); - ROUND(aa, bb, cc, dd, ee, F2, K2, in[10], 11); - ROUND(ee, aa, bb, cc, dd, F2, K2, in[6], 9); - ROUND(dd, ee, aa, bb, cc, F2, K2, in[15], 7); - ROUND(cc, dd, ee, aa, bb, F2, K2, in[3], 15); - ROUND(bb, cc, dd, ee, aa, F2, K2, in[12], 7); - ROUND(aa, bb, cc, dd, ee, F2, K2, in[0], 12); - ROUND(ee, aa, bb, cc, dd, F2, K2, in[9], 15); - ROUND(dd, ee, aa, bb, cc, F2, K2, in[5], 9); - ROUND(cc, dd, ee, aa, bb, F2, K2, in[2], 11); - ROUND(bb, cc, dd, ee, aa, F2, K2, in[14], 7); - ROUND(aa, bb, cc, dd, ee, F2, K2, in[11], 13); - ROUND(ee, aa, bb, cc, dd, F2, K2, in[8], 12); - - /* round 2: right lane */ - ROUND(eee, aaa, bbb, ccc, ddd, F4, KK2, in[6], 9); - ROUND(ddd, eee, aaa, bbb, ccc, F4, KK2, in[11], 13); - ROUND(ccc, ddd, eee, aaa, bbb, F4, KK2, in[3], 15); - ROUND(bbb, ccc, ddd, eee, aaa, F4, KK2, in[7], 7); - ROUND(aaa, bbb, ccc, ddd, eee, F4, KK2, in[0], 12); - ROUND(eee, aaa, bbb, ccc, ddd, F4, KK2, in[13], 8); - ROUND(ddd, eee, aaa, bbb, ccc, F4, KK2, in[5], 9); - ROUND(ccc, ddd, eee, aaa, bbb, F4, KK2, in[10], 11); - ROUND(bbb, ccc, ddd, eee, aaa, F4, KK2, in[14], 7); - ROUND(aaa, bbb, ccc, ddd, eee, F4, KK2, in[15], 7); - ROUND(eee, aaa, bbb, ccc, ddd, F4, KK2, in[8], 12); - ROUND(ddd, eee, aaa, bbb, ccc, F4, KK2, in[12], 7); - ROUND(ccc, ddd, eee, aaa, bbb, F4, KK2, in[4], 6); - ROUND(bbb, ccc, ddd, eee, aaa, F4, KK2, in[9], 15); - ROUND(aaa, bbb, ccc, ddd, eee, F4, KK2, in[1], 13); - ROUND(eee, aaa, bbb, ccc, ddd, F4, KK2, in[2], 11); - - /* Swap contents of "b" registers */ - swap(bb, bbb); - - /* round 3: left lane" */ - ROUND(dd, ee, aa, bb, cc, F3, K3, in[3], 11); - ROUND(cc, dd, ee, aa, bb, F3, K3, in[10], 13); - ROUND(bb, cc, dd, ee, aa, F3, K3, in[14], 6); - ROUND(aa, bb, cc, dd, ee, F3, K3, in[4], 7); - ROUND(ee, aa, bb, cc, dd, F3, K3, in[9], 14); - ROUND(dd, ee, aa, bb, cc, F3, K3, in[15], 9); - ROUND(cc, dd, ee, aa, bb, F3, K3, in[8], 13); - ROUND(bb, cc, dd, ee, aa, F3, K3, in[1], 15); - ROUND(aa, bb, cc, dd, ee, F3, K3, in[2], 14); - ROUND(ee, aa, bb, cc, dd, F3, K3, in[7], 8); - ROUND(dd, ee, aa, bb, cc, F3, K3, in[0], 13); - ROUND(cc, dd, ee, aa, bb, F3, K3, in[6], 6); - ROUND(bb, cc, dd, ee, aa, F3, K3, in[13], 5); - ROUND(aa, bb, cc, dd, ee, F3, K3, in[11], 12); - ROUND(ee, aa, bb, cc, dd, F3, K3, in[5], 7); - ROUND(dd, ee, aa, bb, cc, F3, K3, in[12], 5); - - /* round 3: right lane */ - ROUND(ddd, eee, aaa, bbb, ccc, F3, KK3, in[15], 9); - ROUND(ccc, ddd, eee, aaa, bbb, F3, KK3, in[5], 7); - ROUND(bbb, ccc, ddd, eee, aaa, F3, KK3, in[1], 15); - ROUND(aaa, bbb, ccc, ddd, eee, F3, KK3, in[3], 11); - ROUND(eee, aaa, bbb, ccc, ddd, F3, KK3, in[7], 8); - ROUND(ddd, eee, aaa, bbb, ccc, F3, KK3, in[14], 6); - ROUND(ccc, ddd, eee, aaa, bbb, F3, KK3, in[6], 6); - ROUND(bbb, ccc, ddd, eee, aaa, F3, KK3, in[9], 14); - ROUND(aaa, bbb, ccc, ddd, eee, F3, KK3, in[11], 12); - ROUND(eee, aaa, bbb, ccc, ddd, F3, KK3, in[8], 13); - ROUND(ddd, eee, aaa, bbb, ccc, F3, KK3, in[12], 5); - ROUND(ccc, ddd, eee, aaa, bbb, F3, KK3, in[2], 14); - ROUND(bbb, ccc, ddd, eee, aaa, F3, KK3, in[10], 13); - ROUND(aaa, bbb, ccc, ddd, eee, F3, KK3, in[0], 13); - ROUND(eee, aaa, bbb, ccc, ddd, F3, KK3, in[4], 7); - ROUND(ddd, eee, aaa, bbb, ccc, F3, KK3, in[13], 5); - - /* Swap contents of "c" registers */ - swap(cc, ccc); - - /* round 4: left lane" */ - ROUND(cc, dd, ee, aa, bb, F4, K4, in[1], 11); - ROUND(bb, cc, dd, ee, aa, F4, K4, in[9], 12); - ROUND(aa, bb, cc, dd, ee, F4, K4, in[11], 14); - ROUND(ee, aa, bb, cc, dd, F4, K4, in[10], 15); - ROUND(dd, ee, aa, bb, cc, F4, K4, in[0], 14); - ROUND(cc, dd, ee, aa, bb, F4, K4, in[8], 15); - ROUND(bb, cc, dd, ee, aa, F4, K4, in[12], 9); - ROUND(aa, bb, cc, dd, ee, F4, K4, in[4], 8); - ROUND(ee, aa, bb, cc, dd, F4, K4, in[13], 9); - ROUND(dd, ee, aa, bb, cc, F4, K4, in[3], 14); - ROUND(cc, dd, ee, aa, bb, F4, K4, in[7], 5); - ROUND(bb, cc, dd, ee, aa, F4, K4, in[15], 6); - ROUND(aa, bb, cc, dd, ee, F4, K4, in[14], 8); - ROUND(ee, aa, bb, cc, dd, F4, K4, in[5], 6); - ROUND(dd, ee, aa, bb, cc, F4, K4, in[6], 5); - ROUND(cc, dd, ee, aa, bb, F4, K4, in[2], 12); - - /* round 4: right lane */ - ROUND(ccc, ddd, eee, aaa, bbb, F2, KK4, in[8], 15); - ROUND(bbb, ccc, ddd, eee, aaa, F2, KK4, in[6], 5); - ROUND(aaa, bbb, ccc, ddd, eee, F2, KK4, in[4], 8); - ROUND(eee, aaa, bbb, ccc, ddd, F2, KK4, in[1], 11); - ROUND(ddd, eee, aaa, bbb, ccc, F2, KK4, in[3], 14); - ROUND(ccc, ddd, eee, aaa, bbb, F2, KK4, in[11], 14); - ROUND(bbb, ccc, ddd, eee, aaa, F2, KK4, in[15], 6); - ROUND(aaa, bbb, ccc, ddd, eee, F2, KK4, in[0], 14); - ROUND(eee, aaa, bbb, ccc, ddd, F2, KK4, in[5], 6); - ROUND(ddd, eee, aaa, bbb, ccc, F2, KK4, in[12], 9); - ROUND(ccc, ddd, eee, aaa, bbb, F2, KK4, in[2], 12); - ROUND(bbb, ccc, ddd, eee, aaa, F2, KK4, in[13], 9); - ROUND(aaa, bbb, ccc, ddd, eee, F2, KK4, in[9], 12); - ROUND(eee, aaa, bbb, ccc, ddd, F2, KK4, in[7], 5); - ROUND(ddd, eee, aaa, bbb, ccc, F2, KK4, in[10], 15); - ROUND(ccc, ddd, eee, aaa, bbb, F2, KK4, in[14], 8); - - /* Swap contents of "d" registers */ - swap(dd, ddd); - - /* round 5: left lane" */ - ROUND(bb, cc, dd, ee, aa, F5, K5, in[4], 9); - ROUND(aa, bb, cc, dd, ee, F5, K5, in[0], 15); - ROUND(ee, aa, bb, cc, dd, F5, K5, in[5], 5); - ROUND(dd, ee, aa, bb, cc, F5, K5, in[9], 11); - ROUND(cc, dd, ee, aa, bb, F5, K5, in[7], 6); - ROUND(bb, cc, dd, ee, aa, F5, K5, in[12], 8); - ROUND(aa, bb, cc, dd, ee, F5, K5, in[2], 13); - ROUND(ee, aa, bb, cc, dd, F5, K5, in[10], 12); - ROUND(dd, ee, aa, bb, cc, F5, K5, in[14], 5); - ROUND(cc, dd, ee, aa, bb, F5, K5, in[1], 12); - ROUND(bb, cc, dd, ee, aa, F5, K5, in[3], 13); - ROUND(aa, bb, cc, dd, ee, F5, K5, in[8], 14); - ROUND(ee, aa, bb, cc, dd, F5, K5, in[11], 11); - ROUND(dd, ee, aa, bb, cc, F5, K5, in[6], 8); - ROUND(cc, dd, ee, aa, bb, F5, K5, in[15], 5); - ROUND(bb, cc, dd, ee, aa, F5, K5, in[13], 6); - - /* round 5: right lane */ - ROUND(bbb, ccc, ddd, eee, aaa, F1, KK5, in[12], 8); - ROUND(aaa, bbb, ccc, ddd, eee, F1, KK5, in[15], 5); - ROUND(eee, aaa, bbb, ccc, ddd, F1, KK5, in[10], 12); - ROUND(ddd, eee, aaa, bbb, ccc, F1, KK5, in[4], 9); - ROUND(ccc, ddd, eee, aaa, bbb, F1, KK5, in[1], 12); - ROUND(bbb, ccc, ddd, eee, aaa, F1, KK5, in[5], 5); - ROUND(aaa, bbb, ccc, ddd, eee, F1, KK5, in[8], 14); - ROUND(eee, aaa, bbb, ccc, ddd, F1, KK5, in[7], 6); - ROUND(ddd, eee, aaa, bbb, ccc, F1, KK5, in[6], 8); - ROUND(ccc, ddd, eee, aaa, bbb, F1, KK5, in[2], 13); - ROUND(bbb, ccc, ddd, eee, aaa, F1, KK5, in[13], 6); - ROUND(aaa, bbb, ccc, ddd, eee, F1, KK5, in[14], 5); - ROUND(eee, aaa, bbb, ccc, ddd, F1, KK5, in[0], 15); - ROUND(ddd, eee, aaa, bbb, ccc, F1, KK5, in[3], 13); - ROUND(ccc, ddd, eee, aaa, bbb, F1, KK5, in[9], 11); - ROUND(bbb, ccc, ddd, eee, aaa, F1, KK5, in[11], 11); - - /* Swap contents of "e" registers */ - swap(ee, eee); - - /* combine results */ - state[0] += aa; - state[1] += bb; - state[2] += cc; - state[3] += dd; - state[4] += ee; - state[5] += aaa; - state[6] += bbb; - state[7] += ccc; - state[8] += ddd; - state[9] += eee; -} - -static int rmd320_init(struct shash_desc *desc) -{ - struct rmd320_ctx *rctx = shash_desc_ctx(desc); - - rctx->byte_count = 0; - - rctx->state[0] = RMD_H0; - rctx->state[1] = RMD_H1; - rctx->state[2] = RMD_H2; - rctx->state[3] = RMD_H3; - rctx->state[4] = RMD_H4; - rctx->state[5] = RMD_H5; - rctx->state[6] = RMD_H6; - rctx->state[7] = RMD_H7; - rctx->state[8] = RMD_H8; - rctx->state[9] = RMD_H9; - - memset(rctx->buffer, 0, sizeof(rctx->buffer)); - - return 0; -} - -static int rmd320_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct rmd320_ctx *rctx = shash_desc_ctx(desc); - const u32 avail = sizeof(rctx->buffer) - (rctx->byte_count & 0x3f); - - rctx->byte_count += len; - - /* Enough space in buffer? If so copy and we're done */ - if (avail > len) { - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, len); - goto out; - } - - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, avail); - - rmd320_transform(rctx->state, rctx->buffer); - data += avail; - len -= avail; - - while (len >= sizeof(rctx->buffer)) { - memcpy(rctx->buffer, data, sizeof(rctx->buffer)); - rmd320_transform(rctx->state, rctx->buffer); - data += sizeof(rctx->buffer); - len -= sizeof(rctx->buffer); - } - - memcpy(rctx->buffer, data, len); - -out: - return 0; -} - -/* Add padding and return the message digest. */ -static int rmd320_final(struct shash_desc *desc, u8 *out) -{ - struct rmd320_ctx *rctx = shash_desc_ctx(desc); - u32 i, index, padlen; - __le64 bits; - __le32 *dst = (__le32 *)out; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_le64(rctx->byte_count << 3); - - /* Pad out to 56 mod 64 */ - index = rctx->byte_count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - rmd320_update(desc, padding, padlen); - - /* Append length */ - rmd320_update(desc, (const u8 *)&bits, sizeof(bits)); - - /* Store state in digest */ - for (i = 0; i < 10; i++) - dst[i] = cpu_to_le32p(&rctx->state[i]); - - /* Wipe context */ - memset(rctx, 0, sizeof(*rctx)); - - return 0; -} - -static struct shash_alg alg = { - .digestsize = RMD320_DIGEST_SIZE, - .init = rmd320_init, - .update = rmd320_update, - .final = rmd320_final, - .descsize = sizeof(struct rmd320_ctx), - .base = { - .cra_name = "rmd320", - .cra_driver_name = "rmd320-generic", - .cra_blocksize = RMD320_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init rmd320_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit rmd320_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -subsys_initcall(rmd320_mod_init); -module_exit(rmd320_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Adrian-Ken Rueegsegger "); -MODULE_DESCRIPTION("RIPEMD-320 Message Digest"); -MODULE_ALIAS_CRYPTO("rmd320"); diff --git a/crypto/salsa20_generic.c b/crypto/salsa20_generic.c deleted file mode 100644 index 3418869dabef..000000000000 --- a/crypto/salsa20_generic.c +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Salsa20: Salsa20 stream cipher algorithm - * - * Copyright (c) 2007 Tan Swee Heng - * - * Derived from: - * - salsa20.c: Public domain C code by Daniel J. Bernstein - * - * Salsa20 is a stream cipher candidate in eSTREAM, the ECRYPT Stream - * Cipher Project. It is designed by Daniel J. Bernstein . - * More information about eSTREAM and Salsa20 can be found here: - * https://www.ecrypt.eu.org/stream/ - * https://cr.yp.to/snuffle.html - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - */ - -#include -#include -#include - -#define SALSA20_IV_SIZE 8 -#define SALSA20_MIN_KEY_SIZE 16 -#define SALSA20_MAX_KEY_SIZE 32 -#define SALSA20_BLOCK_SIZE 64 - -struct salsa20_ctx { - u32 initial_state[16]; -}; - -static void salsa20_block(u32 *state, __le32 *stream) -{ - u32 x[16]; - int i; - - memcpy(x, state, sizeof(x)); - - for (i = 0; i < 20; i += 2) { - x[ 4] ^= rol32((x[ 0] + x[12]), 7); - x[ 8] ^= rol32((x[ 4] + x[ 0]), 9); - x[12] ^= rol32((x[ 8] + x[ 4]), 13); - x[ 0] ^= rol32((x[12] + x[ 8]), 18); - x[ 9] ^= rol32((x[ 5] + x[ 1]), 7); - x[13] ^= rol32((x[ 9] + x[ 5]), 9); - x[ 1] ^= rol32((x[13] + x[ 9]), 13); - x[ 5] ^= rol32((x[ 1] + x[13]), 18); - x[14] ^= rol32((x[10] + x[ 6]), 7); - x[ 2] ^= rol32((x[14] + x[10]), 9); - x[ 6] ^= rol32((x[ 2] + x[14]), 13); - x[10] ^= rol32((x[ 6] + x[ 2]), 18); - x[ 3] ^= rol32((x[15] + x[11]), 7); - x[ 7] ^= rol32((x[ 3] + x[15]), 9); - x[11] ^= rol32((x[ 7] + x[ 3]), 13); - x[15] ^= rol32((x[11] + x[ 7]), 18); - x[ 1] ^= rol32((x[ 0] + x[ 3]), 7); - x[ 2] ^= rol32((x[ 1] + x[ 0]), 9); - x[ 3] ^= rol32((x[ 2] + x[ 1]), 13); - x[ 0] ^= rol32((x[ 3] + x[ 2]), 18); - x[ 6] ^= rol32((x[ 5] + x[ 4]), 7); - x[ 7] ^= rol32((x[ 6] + x[ 5]), 9); - x[ 4] ^= rol32((x[ 7] + x[ 6]), 13); - x[ 5] ^= rol32((x[ 4] + x[ 7]), 18); - x[11] ^= rol32((x[10] + x[ 9]), 7); - x[ 8] ^= rol32((x[11] + x[10]), 9); - x[ 9] ^= rol32((x[ 8] + x[11]), 13); - x[10] ^= rol32((x[ 9] + x[ 8]), 18); - x[12] ^= rol32((x[15] + x[14]), 7); - x[13] ^= rol32((x[12] + x[15]), 9); - x[14] ^= rol32((x[13] + x[12]), 13); - x[15] ^= rol32((x[14] + x[13]), 18); - } - - for (i = 0; i < 16; i++) - stream[i] = cpu_to_le32(x[i] + state[i]); - - if (++state[8] == 0) - state[9]++; -} - -static void salsa20_docrypt(u32 *state, u8 *dst, const u8 *src, - unsigned int bytes) -{ - __le32 stream[SALSA20_BLOCK_SIZE / sizeof(__le32)]; - - while (bytes >= SALSA20_BLOCK_SIZE) { - salsa20_block(state, stream); - crypto_xor_cpy(dst, src, (const u8 *)stream, - SALSA20_BLOCK_SIZE); - bytes -= SALSA20_BLOCK_SIZE; - dst += SALSA20_BLOCK_SIZE; - src += SALSA20_BLOCK_SIZE; - } - if (bytes) { - salsa20_block(state, stream); - crypto_xor_cpy(dst, src, (const u8 *)stream, bytes); - } -} - -static void salsa20_init(u32 *state, const struct salsa20_ctx *ctx, - const u8 *iv) -{ - memcpy(state, ctx->initial_state, sizeof(ctx->initial_state)); - state[6] = get_unaligned_le32(iv + 0); - state[7] = get_unaligned_le32(iv + 4); -} - -static int salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize) -{ - static const char sigma[16] = "expand 32-byte k"; - static const char tau[16] = "expand 16-byte k"; - struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm); - const char *constants; - - if (keysize != SALSA20_MIN_KEY_SIZE && - keysize != SALSA20_MAX_KEY_SIZE) - return -EINVAL; - - ctx->initial_state[1] = get_unaligned_le32(key + 0); - ctx->initial_state[2] = get_unaligned_le32(key + 4); - ctx->initial_state[3] = get_unaligned_le32(key + 8); - ctx->initial_state[4] = get_unaligned_le32(key + 12); - if (keysize == 32) { /* recommended */ - key += 16; - constants = sigma; - } else { /* keysize == 16 */ - constants = tau; - } - ctx->initial_state[11] = get_unaligned_le32(key + 0); - ctx->initial_state[12] = get_unaligned_le32(key + 4); - ctx->initial_state[13] = get_unaligned_le32(key + 8); - ctx->initial_state[14] = get_unaligned_le32(key + 12); - ctx->initial_state[0] = get_unaligned_le32(constants + 0); - ctx->initial_state[5] = get_unaligned_le32(constants + 4); - ctx->initial_state[10] = get_unaligned_le32(constants + 8); - ctx->initial_state[15] = get_unaligned_le32(constants + 12); - - /* space for the nonce; it will be overridden for each request */ - ctx->initial_state[6] = 0; - ctx->initial_state[7] = 0; - - /* initial block number */ - ctx->initial_state[8] = 0; - ctx->initial_state[9] = 0; - - return 0; -} - -static int salsa20_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - const struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - salsa20_init(state, ctx, req->iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - salsa20_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr, - nbytes); - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static struct skcipher_alg alg = { - .base.cra_name = "salsa20", - .base.cra_driver_name = "salsa20-generic", - .base.cra_priority = 100, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct salsa20_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = SALSA20_MIN_KEY_SIZE, - .max_keysize = SALSA20_MAX_KEY_SIZE, - .ivsize = SALSA20_IV_SIZE, - .chunksize = SALSA20_BLOCK_SIZE, - .setkey = salsa20_setkey, - .encrypt = salsa20_crypt, - .decrypt = salsa20_crypt, -}; - -static int __init salsa20_generic_mod_init(void) -{ - return crypto_register_skcipher(&alg); -} - -static void __exit salsa20_generic_mod_fini(void) -{ - crypto_unregister_skcipher(&alg); -} - -subsys_initcall(salsa20_generic_mod_init); -module_exit(salsa20_generic_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION ("Salsa20 stream cipher algorithm"); -MODULE_ALIAS_CRYPTO("salsa20"); -MODULE_ALIAS_CRYPTO("salsa20-generic"); diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index 492c1d0bfe06..236c87547a17 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -5,17 +5,12 @@ * Serpent Cipher Algorithm. * * Copyright (C) 2002 Dag Arne Osvik - * 2003 Herbert Valerio Riedel - * - * Added tnepres support: - * Ruben Jesus Garcia Hernandez , 18.10.2004 - * Based on code by hvr */ #include #include #include -#include +#include #include #include #include @@ -453,19 +448,12 @@ void __serpent_encrypt(const void *c, u8 *dst, const u8 *src) { const struct serpent_ctx *ctx = c; const u32 *k = ctx->expkey; - const __le32 *s = (const __le32 *)src; - __le32 *d = (__le32 *)dst; u32 r0, r1, r2, r3, r4; -/* - * Note: The conversions between u8* and u32* might cause trouble - * on architectures with stricter alignment rules than x86 - */ - - r0 = le32_to_cpu(s[0]); - r1 = le32_to_cpu(s[1]); - r2 = le32_to_cpu(s[2]); - r3 = le32_to_cpu(s[3]); + r0 = get_unaligned_le32(src); + r1 = get_unaligned_le32(src + 4); + r2 = get_unaligned_le32(src + 8); + r3 = get_unaligned_le32(src + 12); K(r0, r1, r2, r3, 0); S0(r0, r1, r2, r3, r4); LK(r2, r1, r3, r0, r4, 1); @@ -501,10 +489,10 @@ void __serpent_encrypt(const void *c, u8 *dst, const u8 *src) S6(r0, r1, r3, r2, r4); LK(r3, r4, r1, r2, r0, 31); S7(r3, r4, r1, r2, r0); K(r0, r1, r2, r3, 32); - d[0] = cpu_to_le32(r0); - d[1] = cpu_to_le32(r1); - d[2] = cpu_to_le32(r2); - d[3] = cpu_to_le32(r3); + put_unaligned_le32(r0, dst); + put_unaligned_le32(r1, dst + 4); + put_unaligned_le32(r2, dst + 8); + put_unaligned_le32(r3, dst + 12); } EXPORT_SYMBOL_GPL(__serpent_encrypt); @@ -519,14 +507,12 @@ void __serpent_decrypt(const void *c, u8 *dst, const u8 *src) { const struct serpent_ctx *ctx = c; const u32 *k = ctx->expkey; - const __le32 *s = (const __le32 *)src; - __le32 *d = (__le32 *)dst; u32 r0, r1, r2, r3, r4; - r0 = le32_to_cpu(s[0]); - r1 = le32_to_cpu(s[1]); - r2 = le32_to_cpu(s[2]); - r3 = le32_to_cpu(s[3]); + r0 = get_unaligned_le32(src); + r1 = get_unaligned_le32(src + 4); + r2 = get_unaligned_le32(src + 8); + r3 = get_unaligned_le32(src + 12); K(r0, r1, r2, r3, 32); SI7(r0, r1, r2, r3, r4); KL(r1, r3, r0, r4, r2, 31); @@ -562,10 +548,10 @@ void __serpent_decrypt(const void *c, u8 *dst, const u8 *src) SI1(r3, r1, r2, r0, r4); KL(r4, r1, r2, r0, r3, 1); SI0(r4, r1, r2, r0, r3); K(r2, r3, r1, r4, 0); - d[0] = cpu_to_le32(r2); - d[1] = cpu_to_le32(r3); - d[2] = cpu_to_le32(r1); - d[3] = cpu_to_le32(r4); + put_unaligned_le32(r2, dst); + put_unaligned_le32(r3, dst + 4); + put_unaligned_le32(r1, dst + 8); + put_unaligned_le32(r4, dst + 12); } EXPORT_SYMBOL_GPL(__serpent_decrypt); @@ -576,66 +562,13 @@ static void serpent_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) __serpent_decrypt(ctx, dst, src); } -static int tnepres_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) -{ - u8 rev_key[SERPENT_MAX_KEY_SIZE]; - int i; - - for (i = 0; i < keylen; ++i) - rev_key[keylen - i - 1] = key[i]; - - return serpent_setkey(tfm, rev_key, keylen); -} - -static void tnepres_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - const u32 * const s = (const u32 * const)src; - u32 * const d = (u32 * const)dst; - - u32 rs[4], rd[4]; - - rs[0] = swab32(s[3]); - rs[1] = swab32(s[2]); - rs[2] = swab32(s[1]); - rs[3] = swab32(s[0]); - - serpent_encrypt(tfm, (u8 *)rd, (u8 *)rs); - - d[0] = swab32(rd[3]); - d[1] = swab32(rd[2]); - d[2] = swab32(rd[1]); - d[3] = swab32(rd[0]); -} - -static void tnepres_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - const u32 * const s = (const u32 * const)src; - u32 * const d = (u32 * const)dst; - - u32 rs[4], rd[4]; - - rs[0] = swab32(s[3]); - rs[1] = swab32(s[2]); - rs[2] = swab32(s[1]); - rs[3] = swab32(s[0]); - - serpent_decrypt(tfm, (u8 *)rd, (u8 *)rs); - - d[0] = swab32(rd[3]); - d[1] = swab32(rd[2]); - d[2] = swab32(rd[1]); - d[3] = swab32(rd[0]); -} - -static struct crypto_alg srp_algs[2] = { { +static struct crypto_alg srp_alg = { .cra_name = "serpent", .cra_driver_name = "serpent-generic", .cra_priority = 100, .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { .cia_min_keysize = SERPENT_MIN_KEY_SIZE, @@ -643,38 +576,23 @@ static struct crypto_alg srp_algs[2] = { { .cia_setkey = serpent_setkey, .cia_encrypt = serpent_encrypt, .cia_decrypt = serpent_decrypt } } -}, { - .cra_name = "tnepres", - .cra_driver_name = "tnepres-generic", - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = SERPENT_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct serpent_ctx), - .cra_alignmask = 3, - .cra_module = THIS_MODULE, - .cra_u = { .cipher = { - .cia_min_keysize = SERPENT_MIN_KEY_SIZE, - .cia_max_keysize = SERPENT_MAX_KEY_SIZE, - .cia_setkey = tnepres_setkey, - .cia_encrypt = tnepres_encrypt, - .cia_decrypt = tnepres_decrypt } } -} }; +}; static int __init serpent_mod_init(void) { - return crypto_register_algs(srp_algs, ARRAY_SIZE(srp_algs)); + return crypto_register_alg(&srp_alg); } static void __exit serpent_mod_fini(void) { - crypto_unregister_algs(srp_algs, ARRAY_SIZE(srp_algs)); + crypto_unregister_alg(&srp_alg); } subsys_initcall(serpent_mod_init); module_exit(serpent_mod_fini); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Serpent and tnepres (kerneli compatible serpent reversed) Cipher Algorithm"); +MODULE_DESCRIPTION("Serpent Cipher Algorithm"); MODULE_AUTHOR("Dag Arne Osvik "); -MODULE_ALIAS_CRYPTO("tnepres"); MODULE_ALIAS_CRYPTO("serpent"); MODULE_ALIAS_CRYPTO("serpent-generic"); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index b4dae640de9f..a15376245416 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -490,12 +491,6 @@ int skcipher_walk_virt(struct skcipher_walk *walk, } EXPORT_SYMBOL_GPL(skcipher_walk_virt); -void skcipher_walk_atomise(struct skcipher_walk *walk) -{ - walk->flags &= ~SKCIPHER_WALK_SLEEP; -} -EXPORT_SYMBOL_GPL(skcipher_walk_atomise); - int skcipher_walk_async(struct skcipher_walk *walk, struct skcipher_request *req) { @@ -986,3 +981,4 @@ EXPORT_SYMBOL_GPL(skcipher_alloc_instance_simple); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Symmetric key cipher type"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index a647bb298fbc..6b7c158dc508 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -70,8 +70,8 @@ static const char *check[] = { "des", "md5", "des3_ede", "rot13", "sha1", "sha224", "sha256", "sm3", "blowfish", "twofish", "serpent", "sha384", "sha512", "md4", "aes", "cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea", - "khazad", "wp512", "wp384", "wp256", "tnepres", "xeta", "fcrypt", - "camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320", + "khazad", "wp512", "wp384", "wp256", "xeta", "fcrypt", + "camellia", "seed", "rmd160", "lzo", "lzo-rle", "cts", "sha3-224", "sha3-256", "sha3-384", "sha3-512", "streebog256", "streebog512", NULL @@ -199,8 +199,8 @@ static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc, goto out; } - pr_cont("%d operations in %d seconds (%ld bytes)\n", - bcount * num_mb, secs, (long)bcount * blen * num_mb); + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount * num_mb, secs, (u64)bcount * blen * num_mb); out: kfree(rc); @@ -471,8 +471,8 @@ static int test_aead_jiffies(struct aead_request *req, int enc, return ret; } - printk("%d operations in %d seconds (%ld bytes)\n", - bcount, secs, (long)bcount * blen); + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount, secs, (u64)bcount * blen); return 0; } @@ -764,8 +764,8 @@ static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen, goto out; } - pr_cont("%d operations in %d seconds (%ld bytes)\n", - bcount * num_mb, secs, (long)bcount * blen * num_mb); + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount * num_mb, secs, (u64)bcount * blen * num_mb); out: kfree(rc); @@ -1201,8 +1201,8 @@ static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc, goto out; } - pr_cont("%d operations in %d seconds (%ld bytes)\n", - bcount * num_mb, secs, (long)bcount * blen * num_mb); + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount * num_mb, secs, (u64)bcount * blen * num_mb); out: kfree(rc); @@ -1441,8 +1441,8 @@ static int test_acipher_jiffies(struct skcipher_request *req, int enc, return ret; } - pr_cont("%d operations in %d seconds (%ld bytes)\n", - bcount, secs, (long)bcount * blen); + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount, secs, (u64)bcount * blen); return 0; } @@ -1806,27 +1806,11 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("wp256"); break; - case 25: - ret += tcrypt_test("ecb(tnepres)"); - break; - case 26: ret += tcrypt_test("ecb(anubis)"); ret += tcrypt_test("cbc(anubis)"); break; - case 27: - ret += tcrypt_test("tgr192"); - break; - - case 28: - ret += tcrypt_test("tgr160"); - break; - - case 29: - ret += tcrypt_test("tgr128"); - break; - case 30: ret += tcrypt_test("ecb(xeta)"); break; @@ -1847,10 +1831,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("sha224"); break; - case 34: - ret += tcrypt_test("salsa20"); - break; - case 35: ret += tcrypt_test("gcm(aes)"); break; @@ -1867,22 +1847,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("cts(cbc(aes))"); break; - case 39: - ret += tcrypt_test("rmd128"); - break; - case 40: ret += tcrypt_test("rmd160"); break; - case 41: - ret += tcrypt_test("rmd256"); - break; - - case 42: - ret += tcrypt_test("rmd320"); - break; - case 43: ret += tcrypt_test("ecb(seed)"); break; @@ -1955,10 +1923,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("xcbc(aes)"); break; - case 107: - ret += tcrypt_test("hmac(rmd128)"); - break; - case 108: ret += tcrypt_test("hmac(rmd160)"); break; @@ -2181,11 +2145,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_32_48_64); break; - case 206: - test_cipher_speed("salsa20", ENCRYPT, sec, NULL, 0, - speed_template_16_32); - break; - case 207: test_cipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0, speed_template_16_32); @@ -2393,38 +2352,14 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_hash_speed("wp512", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; - case 310: - test_hash_speed("tgr128", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; - case 311: - test_hash_speed("tgr160", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; - case 312: - test_hash_speed("tgr192", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; case 313: test_hash_speed("sha224", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; - case 314: - test_hash_speed("rmd128", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; case 315: test_hash_speed("rmd160", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; - case 316: - test_hash_speed("rmd256", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; - case 317: - test_hash_speed("rmd320", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; case 318: klen = 16; test_hash_speed("ghash", sec, generic_hash_speed_template); @@ -2517,38 +2452,14 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_ahash_speed("wp512", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; - case 410: - test_ahash_speed("tgr128", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; - case 411: - test_ahash_speed("tgr160", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; - case 412: - test_ahash_speed("tgr192", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; case 413: test_ahash_speed("sha224", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; - case 414: - test_ahash_speed("rmd128", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; case 415: test_ahash_speed("rmd160", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; - case 416: - test_ahash_speed("rmd256", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; - case 417: - test_ahash_speed("rmd320", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; case 418: test_ahash_speed("sha3-224", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 321e38eef51b..93359999c94b 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -33,10 +33,13 @@ #include #include #include +#include #include #include "internal.h" +MODULE_IMPORT_NS(CRYPTO_INTERNAL); + static bool notests; module_param(notests, bool, 0644); MODULE_PARM_DESC(notests, "disable crypto self-tests"); @@ -4873,12 +4876,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .cipher = __VECS(tea_tv_template) } - }, { - .alg = "ecb(tnepres)", - .test = alg_test_skcipher, - .suite = { - .cipher = __VECS(tnepres_tv_template) - } }, { .alg = "ecb(twofish)", .test = alg_test_skcipher, @@ -4954,12 +4951,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .hash = __VECS(hmac_md5_tv_template) } - }, { - .alg = "hmac(rmd128)", - .test = alg_test_hash, - .suite = { - .hash = __VECS(hmac_rmd128_tv_template) - } }, { .alg = "hmac(rmd160)", .test = alg_test_hash, @@ -5272,30 +5263,12 @@ static const struct alg_test_desc alg_test_descs[] = { .aad_iv = 1, } } - }, { - .alg = "rmd128", - .test = alg_test_hash, - .suite = { - .hash = __VECS(rmd128_tv_template) - } }, { .alg = "rmd160", .test = alg_test_hash, .suite = { .hash = __VECS(rmd160_tv_template) } - }, { - .alg = "rmd256", - .test = alg_test_hash, - .suite = { - .hash = __VECS(rmd256_tv_template) - } - }, { - .alg = "rmd320", - .test = alg_test_hash, - .suite = { - .hash = __VECS(rmd320_tv_template) - } }, { .alg = "rsa", .test = alg_test_akcipher, @@ -5303,12 +5276,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .akcipher = __VECS(rsa_tv_template) } - }, { - .alg = "salsa20", - .test = alg_test_skcipher, - .suite = { - .cipher = __VECS(salsa20_stream_tv_template) - } }, { .alg = "sha1", .test = alg_test_hash, @@ -5396,24 +5363,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .hash = __VECS(streebog512_tv_template) } - }, { - .alg = "tgr128", - .test = alg_test_hash, - .suite = { - .hash = __VECS(tgr128_tv_template) - } - }, { - .alg = "tgr160", - .test = alg_test_hash, - .suite = { - .hash = __VECS(tgr160_tv_template) - } - }, { - .alg = "tgr192", - .test = alg_test_hash, - .suite = { - .hash = __VECS(tgr192_tv_template) - } }, { .alg = "vmac64(aes)", .test = alg_test_hash, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 8c83811c0e35..ced56ea0c9b4 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -3140,66 +3140,6 @@ static const struct hash_testvec md5_tv_template[] = { }; -/* - * RIPEMD-128 test vectors from ISO/IEC 10118-3:2004(E) - */ -static const struct hash_testvec rmd128_tv_template[] = { - { - .digest = "\xcd\xf2\x62\x13\xa1\x50\xdc\x3e" - "\xcb\x61\x0f\x18\xf6\xb3\x8b\x46", - }, { - .plaintext = "a", - .psize = 1, - .digest = "\x86\xbe\x7a\xfa\x33\x9d\x0f\xc7" - "\xcf\xc7\x85\xe7\x2f\x57\x8d\x33", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xc1\x4a\x12\x19\x9c\x66\xe4\xba" - "\x84\x63\x6b\x0f\x69\x14\x4c\x77", - }, { - .plaintext = "message digest", - .psize = 14, - .digest = "\x9e\x32\x7b\x3d\x6e\x52\x30\x62" - "\xaf\xc1\x13\x2d\x7d\xf9\xd1\xb8", - }, { - .plaintext = "abcdefghijklmnopqrstuvwxyz", - .psize = 26, - .digest = "\xfd\x2a\xa6\x07\xf7\x1d\xc8\xf5" - "\x10\x71\x49\x22\xb3\x71\x83\x4e", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" - "fghijklmnopqrstuvwxyz0123456789", - .psize = 62, - .digest = "\xd1\xe9\x59\xeb\x17\x9c\x91\x1f" - "\xae\xa4\x62\x4c\x60\xc5\xc7\x02", - }, { - .plaintext = "1234567890123456789012345678901234567890" - "1234567890123456789012345678901234567890", - .psize = 80, - .digest = "\x3f\x45\xef\x19\x47\x32\xc2\xdb" - "\xb2\xc4\xa2\xc7\x69\x79\x5f\xa3", - }, { - .plaintext = "abcdbcdecdefdefgefghfghighij" - "hijkijkljklmklmnlmnomnopnopq", - .psize = 56, - .digest = "\xa1\xaa\x06\x89\xd0\xfa\xfa\x2d" - "\xdc\x22\xe8\x8b\x49\x13\x3a\x06", - }, { - .plaintext = "abcdefghbcdefghicdefghijdefghijkefghijklfghi" - "jklmghijklmnhijklmnoijklmnopjklmnopqklmnopqr" - "lmnopqrsmnopqrstnopqrstu", - .psize = 112, - .digest = "\xd4\xec\xc9\x13\xe1\xdf\x77\x6b" - "\xf4\x8d\xe9\xd5\x5b\x1f\x25\x46", - }, { - .plaintext = "abcdbcdecdefdefgefghfghighijhijk", - .psize = 32, - .digest = "\x13\xfc\x13\xe8\xef\xff\x34\x7d" - "\xe1\x93\xff\x46\xdb\xac\xcf\xd4", - } -}; - /* * RIPEMD-160 test vectors from ISO/IEC 10118-3:2004(E) */ @@ -3260,134 +3200,6 @@ static const struct hash_testvec rmd160_tv_template[] = { } }; -/* - * RIPEMD-256 test vectors - */ -static const struct hash_testvec rmd256_tv_template[] = { - { - .digest = "\x02\xba\x4c\x4e\x5f\x8e\xcd\x18" - "\x77\xfc\x52\xd6\x4d\x30\xe3\x7a" - "\x2d\x97\x74\xfb\x1e\x5d\x02\x63" - "\x80\xae\x01\x68\xe3\xc5\x52\x2d", - }, { - .plaintext = "a", - .psize = 1, - .digest = "\xf9\x33\x3e\x45\xd8\x57\xf5\xd9" - "\x0a\x91\xba\xb7\x0a\x1e\xba\x0c" - "\xfb\x1b\xe4\xb0\x78\x3c\x9a\xcf" - "\xcd\x88\x3a\x91\x34\x69\x29\x25", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xaf\xbd\x6e\x22\x8b\x9d\x8c\xbb" - "\xce\xf5\xca\x2d\x03\xe6\xdb\xa1" - "\x0a\xc0\xbc\x7d\xcb\xe4\x68\x0e" - "\x1e\x42\xd2\xe9\x75\x45\x9b\x65", - }, { - .plaintext = "message digest", - .psize = 14, - .digest = "\x87\xe9\x71\x75\x9a\x1c\xe4\x7a" - "\x51\x4d\x5c\x91\x4c\x39\x2c\x90" - "\x18\xc7\xc4\x6b\xc1\x44\x65\x55" - "\x4a\xfc\xdf\x54\xa5\x07\x0c\x0e", - }, { - .plaintext = "abcdefghijklmnopqrstuvwxyz", - .psize = 26, - .digest = "\x64\x9d\x30\x34\x75\x1e\xa2\x16" - "\x77\x6b\xf9\xa1\x8a\xcc\x81\xbc" - "\x78\x96\x11\x8a\x51\x97\x96\x87" - "\x82\xdd\x1f\xd9\x7d\x8d\x51\x33", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" - "fghijklmnopqrstuvwxyz0123456789", - .psize = 62, - .digest = "\x57\x40\xa4\x08\xac\x16\xb7\x20" - "\xb8\x44\x24\xae\x93\x1c\xbb\x1f" - "\xe3\x63\xd1\xd0\xbf\x40\x17\xf1" - "\xa8\x9f\x7e\xa6\xde\x77\xa0\xb8", - }, { - .plaintext = "1234567890123456789012345678901234567890" - "1234567890123456789012345678901234567890", - .psize = 80, - .digest = "\x06\xfd\xcc\x7a\x40\x95\x48\xaa" - "\xf9\x13\x68\xc0\x6a\x62\x75\xb5" - "\x53\xe3\xf0\x99\xbf\x0e\xa4\xed" - "\xfd\x67\x78\xdf\x89\xa8\x90\xdd", - }, { - .plaintext = "abcdbcdecdefdefgefghfghighij" - "hijkijkljklmklmnlmnomnopnopq", - .psize = 56, - .digest = "\x38\x43\x04\x55\x83\xaa\xc6\xc8" - "\xc8\xd9\x12\x85\x73\xe7\xa9\x80" - "\x9a\xfb\x2a\x0f\x34\xcc\xc3\x6e" - "\xa9\xe7\x2f\x16\xf6\x36\x8e\x3f", - } -}; - -/* - * RIPEMD-320 test vectors - */ -static const struct hash_testvec rmd320_tv_template[] = { - { - .digest = "\x22\xd6\x5d\x56\x61\x53\x6c\xdc\x75\xc1" - "\xfd\xf5\xc6\xde\x7b\x41\xb9\xf2\x73\x25" - "\xeb\xc6\x1e\x85\x57\x17\x7d\x70\x5a\x0e" - "\xc8\x80\x15\x1c\x3a\x32\xa0\x08\x99\xb8", - }, { - .plaintext = "a", - .psize = 1, - .digest = "\xce\x78\x85\x06\x38\xf9\x26\x58\xa5\xa5" - "\x85\x09\x75\x79\x92\x6d\xda\x66\x7a\x57" - "\x16\x56\x2c\xfc\xf6\xfb\xe7\x7f\x63\x54" - "\x2f\x99\xb0\x47\x05\xd6\x97\x0d\xff\x5d", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xde\x4c\x01\xb3\x05\x4f\x89\x30\xa7\x9d" - "\x09\xae\x73\x8e\x92\x30\x1e\x5a\x17\x08" - "\x5b\xef\xfd\xc1\xb8\xd1\x16\x71\x3e\x74" - "\xf8\x2f\xa9\x42\xd6\x4c\xdb\xc4\x68\x2d", - }, { - .plaintext = "message digest", - .psize = 14, - .digest = "\x3a\x8e\x28\x50\x2e\xd4\x5d\x42\x2f\x68" - "\x84\x4f\x9d\xd3\x16\xe7\xb9\x85\x33\xfa" - "\x3f\x2a\x91\xd2\x9f\x84\xd4\x25\xc8\x8d" - "\x6b\x4e\xff\x72\x7d\xf6\x6a\x7c\x01\x97", - }, { - .plaintext = "abcdefghijklmnopqrstuvwxyz", - .psize = 26, - .digest = "\xca\xbd\xb1\x81\x0b\x92\x47\x0a\x20\x93" - "\xaa\x6b\xce\x05\x95\x2c\x28\x34\x8c\xf4" - "\x3f\xf6\x08\x41\x97\x51\x66\xbb\x40\xed" - "\x23\x40\x04\xb8\x82\x44\x63\xe6\xb0\x09", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" - "fghijklmnopqrstuvwxyz0123456789", - .psize = 62, - .digest = "\xed\x54\x49\x40\xc8\x6d\x67\xf2\x50\xd2" - "\x32\xc3\x0b\x7b\x3e\x57\x70\xe0\xc6\x0c" - "\x8c\xb9\xa4\xca\xfe\x3b\x11\x38\x8a\xf9" - "\x92\x0e\x1b\x99\x23\x0b\x84\x3c\x86\xa4", - }, { - .plaintext = "1234567890123456789012345678901234567890" - "1234567890123456789012345678901234567890", - .psize = 80, - .digest = "\x55\x78\x88\xaf\x5f\x6d\x8e\xd6\x2a\xb6" - "\x69\x45\xc6\xd2\xa0\xa4\x7e\xcd\x53\x41" - "\xe9\x15\xeb\x8f\xea\x1d\x05\x24\x95\x5f" - "\x82\x5d\xc7\x17\xe4\xa0\x08\xab\x2d\x42", - }, { - .plaintext = "abcdbcdecdefdefgefghfghighij" - "hijkijkljklmklmnlmnomnopnopq", - .psize = 56, - .digest = "\xd0\x34\xa7\x95\x0c\xf7\x22\x02\x1b\xa4" - "\xb8\x4d\xf7\x69\xa5\xde\x20\x60\xe2\x59" - "\xdf\x4c\x9b\xb4\xa4\x26\x8c\x0e\x93\x5b" - "\xbc\x74\x70\xa9\x69\xc9\xd0\x72\xa1\xac", - } -}; - static const struct hash_testvec crct10dif_tv_template[] = { { .plaintext = "abc", @@ -5138,132 +4950,6 @@ static const struct hash_testvec wp256_tv_template[] = { }, }; -/* - * TIGER test vectors from Tiger website - */ -static const struct hash_testvec tgr192_tv_template[] = { - { - .plaintext = "", - .psize = 0, - .digest = "\x24\xf0\x13\x0c\x63\xac\x93\x32" - "\x16\x16\x6e\x76\xb1\xbb\x92\x5f" - "\xf3\x73\xde\x2d\x49\x58\x4e\x7a", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xf2\x58\xc1\xe8\x84\x14\xab\x2a" - "\x52\x7a\xb5\x41\xff\xc5\xb8\xbf" - "\x93\x5f\x7b\x95\x1c\x13\x29\x51", - }, { - .plaintext = "Tiger", - .psize = 5, - .digest = "\x9f\x00\xf5\x99\x07\x23\x00\xdd" - "\x27\x6a\xbb\x38\xc8\xeb\x6d\xec" - "\x37\x79\x0c\x11\x6f\x9d\x2b\xdf", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-", - .psize = 64, - .digest = "\x87\xfb\x2a\x90\x83\x85\x1c\xf7" - "\x47\x0d\x2c\xf8\x10\xe6\xdf\x9e" - "\xb5\x86\x44\x50\x34\xa5\xa3\x86", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZ=abcdefghijklmnopqrstuvwxyz+0123456789", - .psize = 64, - .digest = "\x46\x7d\xb8\x08\x63\xeb\xce\x48" - "\x8d\xf1\xcd\x12\x61\x65\x5d\xe9" - "\x57\x89\x65\x65\x97\x5f\x91\x97", - }, { - .plaintext = "Tiger - A Fast New Hash Function, " - "by Ross Anderson and Eli Biham, " - "proceedings of Fast Software Encryption 3, " - "Cambridge, 1996.", - .psize = 125, - .digest = "\x3d\x9a\xeb\x03\xd1\xbd\x1a\x63" - "\x57\xb2\x77\x4d\xfd\x6d\x5b\x24" - "\xdd\x68\x15\x1d\x50\x39\x74\xfc", - }, -}; - -static const struct hash_testvec tgr160_tv_template[] = { - { - .plaintext = "", - .psize = 0, - .digest = "\x24\xf0\x13\x0c\x63\xac\x93\x32" - "\x16\x16\x6e\x76\xb1\xbb\x92\x5f" - "\xf3\x73\xde\x2d", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xf2\x58\xc1\xe8\x84\x14\xab\x2a" - "\x52\x7a\xb5\x41\xff\xc5\xb8\xbf" - "\x93\x5f\x7b\x95", - }, { - .plaintext = "Tiger", - .psize = 5, - .digest = "\x9f\x00\xf5\x99\x07\x23\x00\xdd" - "\x27\x6a\xbb\x38\xc8\xeb\x6d\xec" - "\x37\x79\x0c\x11", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-", - .psize = 64, - .digest = "\x87\xfb\x2a\x90\x83\x85\x1c\xf7" - "\x47\x0d\x2c\xf8\x10\xe6\xdf\x9e" - "\xb5\x86\x44\x50", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZ=abcdefghijklmnopqrstuvwxyz+0123456789", - .psize = 64, - .digest = "\x46\x7d\xb8\x08\x63\xeb\xce\x48" - "\x8d\xf1\xcd\x12\x61\x65\x5d\xe9" - "\x57\x89\x65\x65", - }, { - .plaintext = "Tiger - A Fast New Hash Function, " - "by Ross Anderson and Eli Biham, " - "proceedings of Fast Software Encryption 3, " - "Cambridge, 1996.", - .psize = 125, - .digest = "\x3d\x9a\xeb\x03\xd1\xbd\x1a\x63" - "\x57\xb2\x77\x4d\xfd\x6d\x5b\x24" - "\xdd\x68\x15\x1d", - }, -}; - -static const struct hash_testvec tgr128_tv_template[] = { - { - .plaintext = "", - .psize = 0, - .digest = "\x24\xf0\x13\x0c\x63\xac\x93\x32" - "\x16\x16\x6e\x76\xb1\xbb\x92\x5f", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xf2\x58\xc1\xe8\x84\x14\xab\x2a" - "\x52\x7a\xb5\x41\xff\xc5\xb8\xbf", - }, { - .plaintext = "Tiger", - .psize = 5, - .digest = "\x9f\x00\xf5\x99\x07\x23\x00\xdd" - "\x27\x6a\xbb\x38\xc8\xeb\x6d\xec", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-", - .psize = 64, - .digest = "\x87\xfb\x2a\x90\x83\x85\x1c\xf7" - "\x47\x0d\x2c\xf8\x10\xe6\xdf\x9e", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZ=abcdefghijklmnopqrstuvwxyz+0123456789", - .psize = 64, - .digest = "\x46\x7d\xb8\x08\x63\xeb\xce\x48" - "\x8d\xf1\xcd\x12\x61\x65\x5d\xe9", - }, { - .plaintext = "Tiger - A Fast New Hash Function, " - "by Ross Anderson and Eli Biham, " - "proceedings of Fast Software Encryption 3, " - "Cambridge, 1996.", - .psize = 125, - .digest = "\x3d\x9a\xeb\x03\xd1\xbd\x1a\x63" - "\x57\xb2\x77\x4d\xfd\x6d\x5b\x24", - }, -}; - static const struct hash_testvec ghash_tv_template[] = { { @@ -5452,83 +5138,6 @@ static const struct hash_testvec hmac_md5_tv_template[] = }, }; -/* - * HMAC-RIPEMD128 test vectors from RFC2286 - */ -static const struct hash_testvec hmac_rmd128_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", - .ksize = 16, - .plaintext = "Hi There", - .psize = 8, - .digest = "\xfb\xf6\x1f\x94\x92\xaa\x4b\xbf" - "\x81\xc1\x72\xe8\x4e\x07\x34\xdb", - }, { - .key = "Jefe", - .ksize = 4, - .plaintext = "what do ya want for nothing?", - .psize = 28, - .digest = "\x87\x5f\x82\x88\x62\xb6\xb3\x34" - "\xb4\x27\xc5\x5f\x9f\x7f\xf0\x9b", - }, { - .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", - .ksize = 16, - .plaintext = "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", - .psize = 50, - .digest = "\x09\xf0\xb2\x84\x6d\x2f\x54\x3d" - "\xa3\x63\xcb\xec\x8d\x62\xa3\x8d", - }, { - .key = "\x01\x02\x03\x04\x05\x06\x07\x08" - "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" - "\x11\x12\x13\x14\x15\x16\x17\x18\x19", - .ksize = 25, - .plaintext = "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", - .psize = 50, - .digest = "\xbd\xbb\xd7\xcf\x03\xe4\x4b\x5a" - "\xa6\x0a\xf8\x15\xbe\x4d\x22\x94", - }, { - .key = "\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c", - .ksize = 16, - .plaintext = "Test With Truncation", - .psize = 20, - .digest = "\xe7\x98\x08\xf2\x4b\x25\xfd\x03" - "\x1c\x15\x5f\x0d\x55\x1d\x9a\x3a", - }, { - .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa", - .ksize = 80, - .plaintext = "Test Using Larger Than Block-Size Key - Hash Key First", - .psize = 54, - .digest = "\xdc\x73\x29\x28\xde\x98\x10\x4a" - "\x1f\x59\xd3\x73\xc1\x50\xac\xbb", - }, { - .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa", - .ksize = 80, - .plaintext = "Test Using Larger Than Block-Size Key and Larger Than One " - "Block-Size Data", - .psize = 73, - .digest = "\x5c\x6b\xec\x96\x79\x3e\x16\xd4" - "\x06\x90\xc2\x37\x63\x5f\x30\xc5", - }, -}; - /* * HMAC-RIPEMD160 test vectors from RFC2286 */ @@ -11806,85 +11415,6 @@ static const struct cipher_testvec serpent_tv_template[] = { }, }; -static const struct cipher_testvec tnepres_tv_template[] = { - { /* KeySize=0 */ - .ptext = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ctext = "\x41\xcc\x6b\x31\x59\x31\x45\x97" - "\x6d\x6f\xbb\x38\x4b\x37\x21\x28", - .len = 16, - }, - { /* KeySize=128, PT=0, I=1 */ - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .key = "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 16, - .ctext = "\x49\xaf\xbf\xad\x9d\x5a\x34\x05" - "\x2c\xd8\xff\xa5\x98\x6b\xd2\xdd", - .len = 16, - }, { /* KeySize=128 */ - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .klen = 16, - .ptext = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ctext = "\xea\xf4\xd7\xfc\xd8\x01\x34\x47" - "\x81\x45\x0b\xfa\x0c\xd6\xad\x6e", - .len = 16, - }, { /* KeySize=128, I=121 */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80", - .klen = 16, - .ptext = zeroed_string, - .ctext = "\x3d\xda\xbf\xc0\x06\xda\xab\x06" - "\x46\x2a\xf4\xef\x81\x54\x4e\x26", - .len = 16, - }, { /* KeySize=192, PT=0, I=1 */ - .key = "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 24, - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .ctext = "\xe7\x8e\x54\x02\xc7\x19\x55\x68" - "\xac\x36\x78\xf7\xa3\xf6\x0c\x66", - .len = 16, - }, { /* KeySize=256, PT=0, I=1 */ - .key = "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 32, - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .ctext = "\xab\xed\x96\xe7\x66\xbf\x28\xcb" - "\xc0\xeb\xd2\x1a\x82\xef\x08\x19", - .len = 16, - }, { /* KeySize=256, I=257 */ - .key = "\x1f\x1e\x1d\x1c\x1b\x1a\x19\x18" - "\x17\x16\x15\x14\x13\x12\x11\x10" - "\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08" - "\x07\x06\x05\x04\x03\x02\x01\x00", - .klen = 32, - .ptext = "\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08" - "\x07\x06\x05\x04\x03\x02\x01\x00", - .ctext = "\x5c\xe7\x1c\x70\xd2\x88\x2e\x5b" - "\xb8\x32\xe4\x33\xf8\x9f\x26\xde", - .len = 16, - }, { /* KeySize=256 */ - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x10\x11\x12\x13\x14\x15\x16\x17" - "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", - .klen = 32, - .ptext = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ctext = "\x64\xa9\x1a\x37\xed\x9f\xe7\x49" - "\xa8\x4e\x76\xd6\xf5\x0d\x78\xee", - .len = 16, - } -}; - static const struct cipher_testvec serpent_cbc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" @@ -24800,1168 +24330,6 @@ static const struct cipher_testvec seed_tv_template[] = { } }; -static const struct cipher_testvec salsa20_stream_tv_template[] = { - /* - * Testvectors from verified.test-vectors submitted to ECRYPT. - * They are truncated to size 39, 64, 111, 129 to test a variety - * of input length. - */ - { /* Set 3, vector 0 */ - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F", - .klen = 16, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x00", - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00", - .ctext = "\x2D\xD5\xC3\xF7\xBA\x2B\x20\xF7" - "\x68\x02\x41\x0C\x68\x86\x88\x89" - "\x5A\xD8\xC1\xBD\x4E\xA6\xC9\xB1" - "\x40\xFB\x9B\x90\xE2\x10\x49\xBF" - "\x58\x3F\x52\x79\x70\xEB\xC1", - .len = 39, - }, { /* Set 5, vector 0 */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 16, - .iv = "\x80\x00\x00\x00\x00\x00\x00\x00", - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .ctext = "\xB6\x6C\x1E\x44\x46\xDD\x95\x57" - "\xE5\x78\xE2\x23\xB0\xB7\x68\x01" - "\x7B\x23\xB2\x67\xBB\x02\x34\xAE" - "\x46\x26\xBF\x44\x3F\x21\x97\x76" - "\x43\x6F\xB1\x9F\xD0\xE8\x86\x6F" - "\xCD\x0D\xE9\xA9\x53\x8F\x4A\x09" - "\xCA\x9A\xC0\x73\x2E\x30\xBC\xF9" - "\x8E\x4F\x13\xE4\xB9\xE2\x01\xD9", - .len = 64, - }, { /* Set 3, vector 27 */ - .key = "\x1B\x1C\x1D\x1E\x1F\x20\x21\x22" - "\x23\x24\x25\x26\x27\x28\x29\x2A" - "\x2B\x2C\x2D\x2E\x2F\x30\x31\x32" - "\x33\x34\x35\x36\x37\x38\x39\x3A", - .klen = 32, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x00", - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00", - .ctext = "\xAE\x39\x50\x8E\xAC\x9A\xEC\xE7" - "\xBF\x97\xBB\x20\xB9\xDE\xE4\x1F" - "\x87\xD9\x47\xF8\x28\x91\x35\x98" - "\xDB\x72\xCC\x23\x29\x48\x56\x5E" - "\x83\x7E\x0B\xF3\x7D\x5D\x38\x7B" - "\x2D\x71\x02\xB4\x3B\xB5\xD8\x23" - "\xB0\x4A\xDF\x3C\xEC\xB6\xD9\x3B" - "\x9B\xA7\x52\xBE\xC5\xD4\x50\x59" - "\x15\x14\xB4\x0E\x40\xE6\x53\xD1" - "\x83\x9C\x5B\xA0\x92\x29\x6B\x5E" - "\x96\x5B\x1E\x2F\xD3\xAC\xC1\x92" - "\xB1\x41\x3F\x19\x2F\xC4\x3B\xC6" - "\x95\x46\x45\x54\xE9\x75\x03\x08" - "\x44\xAF\xE5\x8A\x81\x12\x09", - .len = 111, - }, { /* Set 5, vector 27 */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 32, - .iv = "\x00\x00\x00\x10\x00\x00\x00\x00", - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00", - .ctext = "\xD2\xDB\x1A\x5C\xF1\xC1\xAC\xDB" - "\xE8\x1A\x7A\x43\x40\xEF\x53\x43" - "\x5E\x7F\x4B\x1A\x50\x52\x3F\x8D" - "\x28\x3D\xCF\x85\x1D\x69\x6E\x60" - "\xF2\xDE\x74\x56\x18\x1B\x84\x10" - "\xD4\x62\xBA\x60\x50\xF0\x61\xF2" - "\x1C\x78\x7F\xC1\x24\x34\xAF\x58" - "\xBF\x2C\x59\xCA\x90\x77\xF3\xB0" - "\x5B\x4A\xDF\x89\xCE\x2C\x2F\xFC" - "\x67\xF0\xE3\x45\xE8\xB3\xB3\x75" - "\xA0\x95\x71\xA1\x29\x39\x94\xCA" - "\x45\x2F\xBD\xCB\x10\xB6\xBE\x9F" - "\x8E\xF9\xB2\x01\x0A\x5A\x0A\xB7" - "\x6B\x9D\x70\x8E\x4B\xD6\x2F\xCD" - "\x2E\x40\x48\x75\xE9\xE2\x21\x45" - "\x0B\xC9\xB6\xB5\x66\xBC\x9A\x59" - "\x5A", - .len = 129, - }, { /* large test vector generated using Crypto++ */ - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x10\x11\x12\x13\x14\x15\x16\x17" - "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", - .klen = 32, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .ptext = - "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x10\x11\x12\x13\x14\x15\x16\x17" - "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" - "\x20\x21\x22\x23\x24\x25\x26\x27" - "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" - "\x30\x31\x32\x33\x34\x35\x36\x37" - "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f" - "\x40\x41\x42\x43\x44\x45\x46\x47" - "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f" - "\x50\x51\x52\x53\x54\x55\x56\x57" - "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f" - "\x60\x61\x62\x63\x64\x65\x66\x67" - "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f" - "\x70\x71\x72\x73\x74\x75\x76\x77" - "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f" - "\x80\x81\x82\x83\x84\x85\x86\x87" - "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" - "\x90\x91\x92\x93\x94\x95\x96\x97" - "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f" - "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" - "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" - "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" - "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf" - "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" - "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf" - "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7" - "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf" - "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7" - "\xe8\xe9\xea\xeb\xec\xed\xee\xef" - "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7" - "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" - "\x00\x03\x06\x09\x0c\x0f\x12\x15" - "\x18\x1b\x1e\x21\x24\x27\x2a\x2d" - "\x30\x33\x36\x39\x3c\x3f\x42\x45" - "\x48\x4b\x4e\x51\x54\x57\x5a\x5d" - "\x60\x63\x66\x69\x6c\x6f\x72\x75" - "\x78\x7b\x7e\x81\x84\x87\x8a\x8d" - "\x90\x93\x96\x99\x9c\x9f\xa2\xa5" - "\xa8\xab\xae\xb1\xb4\xb7\xba\xbd" - "\xc0\xc3\xc6\xc9\xcc\xcf\xd2\xd5" - "\xd8\xdb\xde\xe1\xe4\xe7\xea\xed" - "\xf0\xf3\xf6\xf9\xfc\xff\x02\x05" - "\x08\x0b\x0e\x11\x14\x17\x1a\x1d" - "\x20\x23\x26\x29\x2c\x2f\x32\x35" - "\x38\x3b\x3e\x41\x44\x47\x4a\x4d" - "\x50\x53\x56\x59\x5c\x5f\x62\x65" - "\x68\x6b\x6e\x71\x74\x77\x7a\x7d" - "\x80\x83\x86\x89\x8c\x8f\x92\x95" - "\x98\x9b\x9e\xa1\xa4\xa7\xaa\xad" - "\xb0\xb3\xb6\xb9\xbc\xbf\xc2\xc5" - "\xc8\xcb\xce\xd1\xd4\xd7\xda\xdd" - "\xe0\xe3\xe6\xe9\xec\xef\xf2\xf5" - "\xf8\xfb\xfe\x01\x04\x07\x0a\x0d" - "\x10\x13\x16\x19\x1c\x1f\x22\x25" - "\x28\x2b\x2e\x31\x34\x37\x3a\x3d" - "\x40\x43\x46\x49\x4c\x4f\x52\x55" - "\x58\x5b\x5e\x61\x64\x67\x6a\x6d" - "\x70\x73\x76\x79\x7c\x7f\x82\x85" - "\x88\x8b\x8e\x91\x94\x97\x9a\x9d" - "\xa0\xa3\xa6\xa9\xac\xaf\xb2\xb5" - "\xb8\xbb\xbe\xc1\xc4\xc7\xca\xcd" - "\xd0\xd3\xd6\xd9\xdc\xdf\xe2\xe5" - "\xe8\xeb\xee\xf1\xf4\xf7\xfa\xfd" - "\x00\x05\x0a\x0f\x14\x19\x1e\x23" - "\x28\x2d\x32\x37\x3c\x41\x46\x4b" - "\x50\x55\x5a\x5f\x64\x69\x6e\x73" - "\x78\x7d\x82\x87\x8c\x91\x96\x9b" - "\xa0\xa5\xaa\xaf\xb4\xb9\xbe\xc3" - "\xc8\xcd\xd2\xd7\xdc\xe1\xe6\xeb" - "\xf0\xf5\xfa\xff\x04\x09\x0e\x13" - "\x18\x1d\x22\x27\x2c\x31\x36\x3b" - "\x40\x45\x4a\x4f\x54\x59\x5e\x63" - "\x68\x6d\x72\x77\x7c\x81\x86\x8b" - "\x90\x95\x9a\x9f\xa4\xa9\xae\xb3" - "\xb8\xbd\xc2\xc7\xcc\xd1\xd6\xdb" - "\xe0\xe5\xea\xef\xf4\xf9\xfe\x03" - "\x08\x0d\x12\x17\x1c\x21\x26\x2b" - "\x30\x35\x3a\x3f\x44\x49\x4e\x53" - "\x58\x5d\x62\x67\x6c\x71\x76\x7b" - "\x80\x85\x8a\x8f\x94\x99\x9e\xa3" - "\xa8\xad\xb2\xb7\xbc\xc1\xc6\xcb" - "\xd0\xd5\xda\xdf\xe4\xe9\xee\xf3" - "\xf8\xfd\x02\x07\x0c\x11\x16\x1b" - "\x20\x25\x2a\x2f\x34\x39\x3e\x43" - "\x48\x4d\x52\x57\x5c\x61\x66\x6b" - "\x70\x75\x7a\x7f\x84\x89\x8e\x93" - "\x98\x9d\xa2\xa7\xac\xb1\xb6\xbb" - "\xc0\xc5\xca\xcf\xd4\xd9\xde\xe3" - "\xe8\xed\xf2\xf7\xfc\x01\x06\x0b" - "\x10\x15\x1a\x1f\x24\x29\x2e\x33" - "\x38\x3d\x42\x47\x4c\x51\x56\x5b" - "\x60\x65\x6a\x6f\x74\x79\x7e\x83" - "\x88\x8d\x92\x97\x9c\xa1\xa6\xab" - "\xb0\xb5\xba\xbf\xc4\xc9\xce\xd3" - "\xd8\xdd\xe2\xe7\xec\xf1\xf6\xfb" - "\x00\x07\x0e\x15\x1c\x23\x2a\x31" - "\x38\x3f\x46\x4d\x54\x5b\x62\x69" - "\x70\x77\x7e\x85\x8c\x93\x9a\xa1" - "\xa8\xaf\xb6\xbd\xc4\xcb\xd2\xd9" - "\xe0\xe7\xee\xf5\xfc\x03\x0a\x11" - "\x18\x1f\x26\x2d\x34\x3b\x42\x49" - "\x50\x57\x5e\x65\x6c\x73\x7a\x81" - "\x88\x8f\x96\x9d\xa4\xab\xb2\xb9" - "\xc0\xc7\xce\xd5\xdc\xe3\xea\xf1" - "\xf8\xff\x06\x0d\x14\x1b\x22\x29" - "\x30\x37\x3e\x45\x4c\x53\x5a\x61" - "\x68\x6f\x76\x7d\x84\x8b\x92\x99" - "\xa0\xa7\xae\xb5\xbc\xc3\xca\xd1" - "\xd8\xdf\xe6\xed\xf4\xfb\x02\x09" - "\x10\x17\x1e\x25\x2c\x33\x3a\x41" - "\x48\x4f\x56\x5d\x64\x6b\x72\x79" - "\x80\x87\x8e\x95\x9c\xa3\xaa\xb1" - "\xb8\xbf\xc6\xcd\xd4\xdb\xe2\xe9" - "\xf0\xf7\xfe\x05\x0c\x13\x1a\x21" - "\x28\x2f\x36\x3d\x44\x4b\x52\x59" - "\x60\x67\x6e\x75\x7c\x83\x8a\x91" - "\x98\x9f\xa6\xad\xb4\xbb\xc2\xc9" - "\xd0\xd7\xde\xe5\xec\xf3\xfa\x01" - "\x08\x0f\x16\x1d\x24\x2b\x32\x39" - "\x40\x47\x4e\x55\x5c\x63\x6a\x71" - "\x78\x7f\x86\x8d\x94\x9b\xa2\xa9" - "\xb0\xb7\xbe\xc5\xcc\xd3\xda\xe1" - "\xe8\xef\xf6\xfd\x04\x0b\x12\x19" - "\x20\x27\x2e\x35\x3c\x43\x4a\x51" - "\x58\x5f\x66\x6d\x74\x7b\x82\x89" - "\x90\x97\x9e\xa5\xac\xb3\xba\xc1" - "\xc8\xcf\xd6\xdd\xe4\xeb\xf2\xf9" - "\x00\x09\x12\x1b\x24\x2d\x36\x3f" - "\x48\x51\x5a\x63\x6c\x75\x7e\x87" - "\x90\x99\xa2\xab\xb4\xbd\xc6\xcf" - "\xd8\xe1\xea\xf3\xfc\x05\x0e\x17" - "\x20\x29\x32\x3b\x44\x4d\x56\x5f" - "\x68\x71\x7a\x83\x8c\x95\x9e\xa7" - "\xb0\xb9\xc2\xcb\xd4\xdd\xe6\xef" - "\xf8\x01\x0a\x13\x1c\x25\x2e\x37" - "\x40\x49\x52\x5b\x64\x6d\x76\x7f" - "\x88\x91\x9a\xa3\xac\xb5\xbe\xc7" - "\xd0\xd9\xe2\xeb\xf4\xfd\x06\x0f" - "\x18\x21\x2a\x33\x3c\x45\x4e\x57" - "\x60\x69\x72\x7b\x84\x8d\x96\x9f" - "\xa8\xb1\xba\xc3\xcc\xd5\xde\xe7" - "\xf0\xf9\x02\x0b\x14\x1d\x26\x2f" - "\x38\x41\x4a\x53\x5c\x65\x6e\x77" - "\x80\x89\x92\x9b\xa4\xad\xb6\xbf" - "\xc8\xd1\xda\xe3\xec\xf5\xfe\x07" - "\x10\x19\x22\x2b\x34\x3d\x46\x4f" - "\x58\x61\x6a\x73\x7c\x85\x8e\x97" - "\xa0\xa9\xb2\xbb\xc4\xcd\xd6\xdf" - "\xe8\xf1\xfa\x03\x0c\x15\x1e\x27" - "\x30\x39\x42\x4b\x54\x5d\x66\x6f" - "\x78\x81\x8a\x93\x9c\xa5\xae\xb7" - "\xc0\xc9\xd2\xdb\xe4\xed\xf6\xff" - "\x08\x11\x1a\x23\x2c\x35\x3e\x47" - "\x50\x59\x62\x6b\x74\x7d\x86\x8f" - "\x98\xa1\xaa\xb3\xbc\xc5\xce\xd7" - "\xe0\xe9\xf2\xfb\x04\x0d\x16\x1f" - "\x28\x31\x3a\x43\x4c\x55\x5e\x67" - "\x70\x79\x82\x8b\x94\x9d\xa6\xaf" - "\xb8\xc1\xca\xd3\xdc\xe5\xee\xf7" - "\x00\x0b\x16\x21\x2c\x37\x42\x4d" - "\x58\x63\x6e\x79\x84\x8f\x9a\xa5" - "\xb0\xbb\xc6\xd1\xdc\xe7\xf2\xfd" - "\x08\x13\x1e\x29\x34\x3f\x4a\x55" - "\x60\x6b\x76\x81\x8c\x97\xa2\xad" - "\xb8\xc3\xce\xd9\xe4\xef\xfa\x05" - "\x10\x1b\x26\x31\x3c\x47\x52\x5d" - "\x68\x73\x7e\x89\x94\x9f\xaa\xb5" - "\xc0\xcb\xd6\xe1\xec\xf7\x02\x0d" - "\x18\x23\x2e\x39\x44\x4f\x5a\x65" - "\x70\x7b\x86\x91\x9c\xa7\xb2\xbd" - "\xc8\xd3\xde\xe9\xf4\xff\x0a\x15" - "\x20\x2b\x36\x41\x4c\x57\x62\x6d" - "\x78\x83\x8e\x99\xa4\xaf\xba\xc5" - "\xd0\xdb\xe6\xf1\xfc\x07\x12\x1d" - "\x28\x33\x3e\x49\x54\x5f\x6a\x75" - "\x80\x8b\x96\xa1\xac\xb7\xc2\xcd" - "\xd8\xe3\xee\xf9\x04\x0f\x1a\x25" - "\x30\x3b\x46\x51\x5c\x67\x72\x7d" - "\x88\x93\x9e\xa9\xb4\xbf\xca\xd5" - "\xe0\xeb\xf6\x01\x0c\x17\x22\x2d" - "\x38\x43\x4e\x59\x64\x6f\x7a\x85" - "\x90\x9b\xa6\xb1\xbc\xc7\xd2\xdd" - "\xe8\xf3\xfe\x09\x14\x1f\x2a\x35" - "\x40\x4b\x56\x61\x6c\x77\x82\x8d" - "\x98\xa3\xae\xb9\xc4\xcf\xda\xe5" - "\xf0\xfb\x06\x11\x1c\x27\x32\x3d" - "\x48\x53\x5e\x69\x74\x7f\x8a\x95" - "\xa0\xab\xb6\xc1\xcc\xd7\xe2\xed" - "\xf8\x03\x0e\x19\x24\x2f\x3a\x45" - "\x50\x5b\x66\x71\x7c\x87\x92\x9d" - "\xa8\xb3\xbe\xc9\xd4\xdf\xea\xf5" - "\x00\x0d\x1a\x27\x34\x41\x4e\x5b" - "\x68\x75\x82\x8f\x9c\xa9\xb6\xc3" - "\xd0\xdd\xea\xf7\x04\x11\x1e\x2b" - "\x38\x45\x52\x5f\x6c\x79\x86\x93" - "\xa0\xad\xba\xc7\xd4\xe1\xee\xfb" - "\x08\x15\x22\x2f\x3c\x49\x56\x63" - "\x70\x7d\x8a\x97\xa4\xb1\xbe\xcb" - "\xd8\xe5\xf2\xff\x0c\x19\x26\x33" - "\x40\x4d\x5a\x67\x74\x81\x8e\x9b" - "\xa8\xb5\xc2\xcf\xdc\xe9\xf6\x03" - "\x10\x1d\x2a\x37\x44\x51\x5e\x6b" - "\x78\x85\x92\x9f\xac\xb9\xc6\xd3" - "\xe0\xed\xfa\x07\x14\x21\x2e\x3b" - "\x48\x55\x62\x6f\x7c\x89\x96\xa3" - "\xb0\xbd\xca\xd7\xe4\xf1\xfe\x0b" - "\x18\x25\x32\x3f\x4c\x59\x66\x73" - "\x80\x8d\x9a\xa7\xb4\xc1\xce\xdb" - "\xe8\xf5\x02\x0f\x1c\x29\x36\x43" - "\x50\x5d\x6a\x77\x84\x91\x9e\xab" - "\xb8\xc5\xd2\xdf\xec\xf9\x06\x13" - "\x20\x2d\x3a\x47\x54\x61\x6e\x7b" - "\x88\x95\xa2\xaf\xbc\xc9\xd6\xe3" - "\xf0\xfd\x0a\x17\x24\x31\x3e\x4b" - "\x58\x65\x72\x7f\x8c\x99\xa6\xb3" - "\xc0\xcd\xda\xe7\xf4\x01\x0e\x1b" - "\x28\x35\x42\x4f\x5c\x69\x76\x83" - "\x90\x9d\xaa\xb7\xc4\xd1\xde\xeb" - "\xf8\x05\x12\x1f\x2c\x39\x46\x53" - "\x60\x6d\x7a\x87\x94\xa1\xae\xbb" - "\xc8\xd5\xe2\xef\xfc\x09\x16\x23" - "\x30\x3d\x4a\x57\x64\x71\x7e\x8b" - "\x98\xa5\xb2\xbf\xcc\xd9\xe6\xf3" - "\x00\x0f\x1e\x2d\x3c\x4b\x5a\x69" - "\x78\x87\x96\xa5\xb4\xc3\xd2\xe1" - "\xf0\xff\x0e\x1d\x2c\x3b\x4a\x59" - "\x68\x77\x86\x95\xa4\xb3\xc2\xd1" - "\xe0\xef\xfe\x0d\x1c\x2b\x3a\x49" - "\x58\x67\x76\x85\x94\xa3\xb2\xc1" - "\xd0\xdf\xee\xfd\x0c\x1b\x2a\x39" - "\x48\x57\x66\x75\x84\x93\xa2\xb1" - "\xc0\xcf\xde\xed\xfc\x0b\x1a\x29" - "\x38\x47\x56\x65\x74\x83\x92\xa1" - "\xb0\xbf\xce\xdd\xec\xfb\x0a\x19" - "\x28\x37\x46\x55\x64\x73\x82\x91" - "\xa0\xaf\xbe\xcd\xdc\xeb\xfa\x09" - "\x18\x27\x36\x45\x54\x63\x72\x81" - "\x90\x9f\xae\xbd\xcc\xdb\xea\xf9" - "\x08\x17\x26\x35\x44\x53\x62\x71" - "\x80\x8f\x9e\xad\xbc\xcb\xda\xe9" - "\xf8\x07\x16\x25\x34\x43\x52\x61" - "\x70\x7f\x8e\x9d\xac\xbb\xca\xd9" - "\xe8\xf7\x06\x15\x24\x33\x42\x51" - "\x60\x6f\x7e\x8d\x9c\xab\xba\xc9" - "\xd8\xe7\xf6\x05\x14\x23\x32\x41" - "\x50\x5f\x6e\x7d\x8c\x9b\xaa\xb9" - "\xc8\xd7\xe6\xf5\x04\x13\x22\x31" - "\x40\x4f\x5e\x6d\x7c\x8b\x9a\xa9" - "\xb8\xc7\xd6\xe5\xf4\x03\x12\x21" - "\x30\x3f\x4e\x5d\x6c\x7b\x8a\x99" - "\xa8\xb7\xc6\xd5\xe4\xf3\x02\x11" - "\x20\x2f\x3e\x4d\x5c\x6b\x7a\x89" - "\x98\xa7\xb6\xc5\xd4\xe3\xf2\x01" - "\x10\x1f\x2e\x3d\x4c\x5b\x6a\x79" - "\x88\x97\xa6\xb5\xc4\xd3\xe2\xf1" - "\x00\x11\x22\x33\x44\x55\x66\x77" - "\x88\x99\xaa\xbb\xcc\xdd\xee\xff" - "\x10\x21\x32\x43\x54\x65\x76\x87" - "\x98\xa9\xba\xcb\xdc\xed\xfe\x0f" - "\x20\x31\x42\x53\x64\x75\x86\x97" - "\xa8\xb9\xca\xdb\xec\xfd\x0e\x1f" - "\x30\x41\x52\x63\x74\x85\x96\xa7" - "\xb8\xc9\xda\xeb\xfc\x0d\x1e\x2f" - "\x40\x51\x62\x73\x84\x95\xa6\xb7" - "\xc8\xd9\xea\xfb\x0c\x1d\x2e\x3f" - "\x50\x61\x72\x83\x94\xa5\xb6\xc7" - "\xd8\xe9\xfa\x0b\x1c\x2d\x3e\x4f" - "\x60\x71\x82\x93\xa4\xb5\xc6\xd7" - "\xe8\xf9\x0a\x1b\x2c\x3d\x4e\x5f" - "\x70\x81\x92\xa3\xb4\xc5\xd6\xe7" - "\xf8\x09\x1a\x2b\x3c\x4d\x5e\x6f" - "\x80\x91\xa2\xb3\xc4\xd5\xe6\xf7" - "\x08\x19\x2a\x3b\x4c\x5d\x6e\x7f" - "\x90\xa1\xb2\xc3\xd4\xe5\xf6\x07" - "\x18\x29\x3a\x4b\x5c\x6d\x7e\x8f" - "\xa0\xb1\xc2\xd3\xe4\xf5\x06\x17" - "\x28\x39\x4a\x5b\x6c\x7d\x8e\x9f" - "\xb0\xc1\xd2\xe3\xf4\x05\x16\x27" - "\x38\x49\x5a\x6b\x7c\x8d\x9e\xaf" - "\xc0\xd1\xe2\xf3\x04\x15\x26\x37" - "\x48\x59\x6a\x7b\x8c\x9d\xae\xbf" - "\xd0\xe1\xf2\x03\x14\x25\x36\x47" - "\x58\x69\x7a\x8b\x9c\xad\xbe\xcf" - "\xe0\xf1\x02\x13\x24\x35\x46\x57" - "\x68\x79\x8a\x9b\xac\xbd\xce\xdf" - "\xf0\x01\x12\x23\x34\x45\x56\x67" - "\x78\x89\x9a\xab\xbc\xcd\xde\xef" - "\x00\x13\x26\x39\x4c\x5f\x72\x85" - "\x98\xab\xbe\xd1\xe4\xf7\x0a\x1d" - "\x30\x43\x56\x69\x7c\x8f\xa2\xb5" - "\xc8\xdb\xee\x01\x14\x27\x3a\x4d" - "\x60\x73\x86\x99\xac\xbf\xd2\xe5" - "\xf8\x0b\x1e\x31\x44\x57\x6a\x7d" - "\x90\xa3\xb6\xc9\xdc\xef\x02\x15" - "\x28\x3b\x4e\x61\x74\x87\x9a\xad" - "\xc0\xd3\xe6\xf9\x0c\x1f\x32\x45" - "\x58\x6b\x7e\x91\xa4\xb7\xca\xdd" - "\xf0\x03\x16\x29\x3c\x4f\x62\x75" - "\x88\x9b\xae\xc1\xd4\xe7\xfa\x0d" - "\x20\x33\x46\x59\x6c\x7f\x92\xa5" - "\xb8\xcb\xde\xf1\x04\x17\x2a\x3d" - "\x50\x63\x76\x89\x9c\xaf\xc2\xd5" - "\xe8\xfb\x0e\x21\x34\x47\x5a\x6d" - "\x80\x93\xa6\xb9\xcc\xdf\xf2\x05" - "\x18\x2b\x3e\x51\x64\x77\x8a\x9d" - "\xb0\xc3\xd6\xe9\xfc\x0f\x22\x35" - "\x48\x5b\x6e\x81\x94\xa7\xba\xcd" - "\xe0\xf3\x06\x19\x2c\x3f\x52\x65" - "\x78\x8b\x9e\xb1\xc4\xd7\xea\xfd" - "\x10\x23\x36\x49\x5c\x6f\x82\x95" - "\xa8\xbb\xce\xe1\xf4\x07\x1a\x2d" - "\x40\x53\x66\x79\x8c\x9f\xb2\xc5" - "\xd8\xeb\xfe\x11\x24\x37\x4a\x5d" - "\x70\x83\x96\xa9\xbc\xcf\xe2\xf5" - "\x08\x1b\x2e\x41\x54\x67\x7a\x8d" - "\xa0\xb3\xc6\xd9\xec\xff\x12\x25" - "\x38\x4b\x5e\x71\x84\x97\xaa\xbd" - "\xd0\xe3\xf6\x09\x1c\x2f\x42\x55" - "\x68\x7b\x8e\xa1\xb4\xc7\xda\xed" - "\x00\x15\x2a\x3f\x54\x69\x7e\x93" - "\xa8\xbd\xd2\xe7\xfc\x11\x26\x3b" - "\x50\x65\x7a\x8f\xa4\xb9\xce\xe3" - "\xf8\x0d\x22\x37\x4c\x61\x76\x8b" - "\xa0\xb5\xca\xdf\xf4\x09\x1e\x33" - "\x48\x5d\x72\x87\x9c\xb1\xc6\xdb" - "\xf0\x05\x1a\x2f\x44\x59\x6e\x83" - "\x98\xad\xc2\xd7\xec\x01\x16\x2b" - "\x40\x55\x6a\x7f\x94\xa9\xbe\xd3" - "\xe8\xfd\x12\x27\x3c\x51\x66\x7b" - "\x90\xa5\xba\xcf\xe4\xf9\x0e\x23" - "\x38\x4d\x62\x77\x8c\xa1\xb6\xcb" - "\xe0\xf5\x0a\x1f\x34\x49\x5e\x73" - "\x88\x9d\xb2\xc7\xdc\xf1\x06\x1b" - "\x30\x45\x5a\x6f\x84\x99\xae\xc3" - "\xd8\xed\x02\x17\x2c\x41\x56\x6b" - "\x80\x95\xaa\xbf\xd4\xe9\xfe\x13" - "\x28\x3d\x52\x67\x7c\x91\xa6\xbb" - "\xd0\xe5\xfa\x0f\x24\x39\x4e\x63" - "\x78\x8d\xa2\xb7\xcc\xe1\xf6\x0b" - "\x20\x35\x4a\x5f\x74\x89\x9e\xb3" - "\xc8\xdd\xf2\x07\x1c\x31\x46\x5b" - "\x70\x85\x9a\xaf\xc4\xd9\xee\x03" - "\x18\x2d\x42\x57\x6c\x81\x96\xab" - "\xc0\xd5\xea\xff\x14\x29\x3e\x53" - "\x68\x7d\x92\xa7\xbc\xd1\xe6\xfb" - "\x10\x25\x3a\x4f\x64\x79\x8e\xa3" - "\xb8\xcd\xe2\xf7\x0c\x21\x36\x4b" - "\x60\x75\x8a\x9f\xb4\xc9\xde\xf3" - "\x08\x1d\x32\x47\x5c\x71\x86\x9b" - "\xb0\xc5\xda\xef\x04\x19\x2e\x43" - "\x58\x6d\x82\x97\xac\xc1\xd6\xeb" - "\x00\x17\x2e\x45\x5c\x73\x8a\xa1" - "\xb8\xcf\xe6\xfd\x14\x2b\x42\x59" - "\x70\x87\x9e\xb5\xcc\xe3\xfa\x11" - "\x28\x3f\x56\x6d\x84\x9b\xb2\xc9" - "\xe0\xf7\x0e\x25\x3c\x53\x6a\x81" - "\x98\xaf\xc6\xdd\xf4\x0b\x22\x39" - "\x50\x67\x7e\x95\xac\xc3\xda\xf1" - "\x08\x1f\x36\x4d\x64\x7b\x92\xa9" - "\xc0\xd7\xee\x05\x1c\x33\x4a\x61" - "\x78\x8f\xa6\xbd\xd4\xeb\x02\x19" - "\x30\x47\x5e\x75\x8c\xa3\xba\xd1" - "\xe8\xff\x16\x2d\x44\x5b\x72\x89" - "\xa0\xb7\xce\xe5\xfc\x13\x2a\x41" - "\x58\x6f\x86\x9d\xb4\xcb\xe2\xf9" - "\x10\x27\x3e\x55\x6c\x83\x9a\xb1" - "\xc8\xdf\xf6\x0d\x24\x3b\x52\x69" - "\x80\x97\xae\xc5\xdc\xf3\x0a\x21" - "\x38\x4f\x66\x7d\x94\xab\xc2\xd9" - "\xf0\x07\x1e\x35\x4c\x63\x7a\x91" - "\xa8\xbf\xd6\xed\x04\x1b\x32\x49" - "\x60\x77\x8e\xa5\xbc\xd3\xea\x01" - "\x18\x2f\x46\x5d\x74\x8b\xa2\xb9" - "\xd0\xe7\xfe\x15\x2c\x43\x5a\x71" - "\x88\x9f\xb6\xcd\xe4\xfb\x12\x29" - "\x40\x57\x6e\x85\x9c\xb3\xca\xe1" - "\xf8\x0f\x26\x3d\x54\x6b\x82\x99" - "\xb0\xc7\xde\xf5\x0c\x23\x3a\x51" - "\x68\x7f\x96\xad\xc4\xdb\xf2\x09" - "\x20\x37\x4e\x65\x7c\x93\xaa\xc1" - "\xd8\xef\x06\x1d\x34\x4b\x62\x79" - "\x90\xa7\xbe\xd5\xec\x03\x1a\x31" - "\x48\x5f\x76\x8d\xa4\xbb\xd2\xe9" - "\x00\x19\x32\x4b\x64\x7d\x96\xaf" - "\xc8\xe1\xfa\x13\x2c\x45\x5e\x77" - "\x90\xa9\xc2\xdb\xf4\x0d\x26\x3f" - "\x58\x71\x8a\xa3\xbc\xd5\xee\x07" - "\x20\x39\x52\x6b\x84\x9d\xb6\xcf" - "\xe8\x01\x1a\x33\x4c\x65\x7e\x97" - "\xb0\xc9\xe2\xfb\x14\x2d\x46\x5f" - "\x78\x91\xaa\xc3\xdc\xf5\x0e\x27" - "\x40\x59\x72\x8b\xa4\xbd\xd6\xef" - "\x08\x21\x3a\x53\x6c\x85\x9e\xb7" - "\xd0\xe9\x02\x1b\x34\x4d\x66\x7f" - "\x98\xb1\xca\xe3\xfc\x15\x2e\x47" - "\x60\x79\x92\xab\xc4\xdd\xf6\x0f" - "\x28\x41\x5a\x73\x8c\xa5\xbe\xd7" - "\xf0\x09\x22\x3b\x54\x6d\x86\x9f" - "\xb8\xd1\xea\x03\x1c\x35\x4e\x67" - "\x80\x99\xb2\xcb\xe4\xfd\x16\x2f" - "\x48\x61\x7a\x93\xac\xc5\xde\xf7" - "\x10\x29\x42\x5b\x74\x8d\xa6\xbf" - "\xd8\xf1\x0a\x23\x3c\x55\x6e\x87" - "\xa0\xb9\xd2\xeb\x04\x1d\x36\x4f" - "\x68\x81\x9a\xb3\xcc\xe5\xfe\x17" - "\x30\x49\x62\x7b\x94\xad\xc6\xdf" - "\xf8\x11\x2a\x43\x5c\x75\x8e\xa7" - "\xc0\xd9\xf2\x0b\x24\x3d\x56\x6f" - "\x88\xa1\xba\xd3\xec\x05\x1e\x37" - "\x50\x69\x82\x9b\xb4\xcd\xe6\xff" - "\x18\x31\x4a\x63\x7c\x95\xae\xc7" - "\xe0\xf9\x12\x2b\x44\x5d\x76\x8f" - "\xa8\xc1\xda\xf3\x0c\x25\x3e\x57" - "\x70\x89\xa2\xbb\xd4\xed\x06\x1f" - "\x38\x51\x6a\x83\x9c\xb5\xce\xe7" - "\x00\x1b\x36\x51\x6c\x87\xa2\xbd" - "\xd8\xf3\x0e\x29\x44\x5f\x7a\x95" - "\xb0\xcb\xe6\x01\x1c\x37\x52\x6d" - "\x88\xa3\xbe\xd9\xf4\x0f\x2a\x45" - "\x60\x7b\x96\xb1\xcc\xe7\x02\x1d" - "\x38\x53\x6e\x89\xa4\xbf\xda\xf5" - "\x10\x2b\x46\x61\x7c\x97\xb2\xcd" - "\xe8\x03\x1e\x39\x54\x6f\x8a\xa5" - "\xc0\xdb\xf6\x11\x2c\x47\x62\x7d" - "\x98\xb3\xce\xe9\x04\x1f\x3a\x55" - "\x70\x8b\xa6\xc1\xdc\xf7\x12\x2d" - "\x48\x63\x7e\x99\xb4\xcf\xea\x05" - "\x20\x3b\x56\x71\x8c\xa7\xc2\xdd" - "\xf8\x13\x2e\x49\x64\x7f\x9a\xb5" - "\xd0\xeb\x06\x21\x3c\x57\x72\x8d" - "\xa8\xc3\xde\xf9\x14\x2f\x4a\x65" - "\x80\x9b\xb6\xd1\xec\x07\x22\x3d" - "\x58\x73\x8e\xa9\xc4\xdf\xfa\x15" - "\x30\x4b\x66\x81\x9c\xb7\xd2\xed" - "\x08\x23\x3e\x59\x74\x8f\xaa\xc5" - "\xe0\xfb\x16\x31\x4c\x67\x82\x9d" - "\xb8\xd3\xee\x09\x24\x3f\x5a\x75" - "\x90\xab\xc6\xe1\xfc\x17\x32\x4d" - "\x68\x83\x9e\xb9\xd4\xef\x0a\x25" - "\x40\x5b\x76\x91\xac\xc7\xe2\xfd" - "\x18\x33\x4e\x69\x84\x9f\xba\xd5" - "\xf0\x0b\x26\x41\x5c\x77\x92\xad" - "\xc8\xe3\xfe\x19\x34\x4f\x6a\x85" - "\xa0\xbb\xd6\xf1\x0c\x27\x42\x5d" - "\x78\x93\xae\xc9\xe4\xff\x1a\x35" - "\x50\x6b\x86\xa1\xbc\xd7\xf2\x0d" - "\x28\x43\x5e\x79\x94\xaf\xca\xe5" - "\x00\x1d\x3a\x57\x74\x91\xae\xcb" - "\xe8\x05\x22\x3f\x5c\x79\x96\xb3" - "\xd0\xed\x0a\x27\x44\x61\x7e\x9b" - "\xb8\xd5\xf2\x0f\x2c\x49\x66\x83" - "\xa0\xbd\xda\xf7\x14\x31\x4e\x6b" - "\x88\xa5\xc2\xdf\xfc\x19\x36\x53" - "\x70\x8d\xaa\xc7\xe4\x01\x1e\x3b" - "\x58\x75\x92\xaf\xcc\xe9\x06\x23" - "\x40\x5d\x7a\x97\xb4\xd1\xee\x0b" - "\x28\x45\x62\x7f\x9c\xb9\xd6\xf3" - "\x10\x2d\x4a\x67\x84\xa1\xbe\xdb" - "\xf8\x15\x32\x4f\x6c\x89\xa6\xc3" - "\xe0\xfd\x1a\x37\x54\x71\x8e\xab" - "\xc8\xe5\x02\x1f\x3c\x59\x76\x93" - "\xb0\xcd\xea\x07\x24\x41\x5e\x7b" - "\x98\xb5\xd2\xef\x0c\x29\x46\x63" - "\x80\x9d\xba\xd7\xf4\x11\x2e\x4b" - "\x68\x85\xa2\xbf\xdc\xf9\x16\x33" - "\x50\x6d\x8a\xa7\xc4\xe1\xfe\x1b" - "\x38\x55\x72\x8f\xac\xc9\xe6\x03" - "\x20\x3d\x5a\x77\x94\xb1\xce\xeb" - "\x08\x25\x42\x5f\x7c\x99\xb6\xd3" - "\xf0\x0d\x2a\x47\x64\x81\x9e\xbb" - "\xd8\xf5\x12\x2f\x4c\x69\x86\xa3" - "\xc0\xdd\xfa\x17\x34\x51\x6e\x8b" - "\xa8\xc5\xe2\xff\x1c\x39\x56\x73" - "\x90\xad\xca\xe7\x04\x21\x3e\x5b" - "\x78\x95\xb2\xcf\xec\x09\x26\x43" - "\x60\x7d\x9a\xb7\xd4\xf1\x0e\x2b" - "\x48\x65\x82\x9f\xbc\xd9\xf6\x13" - "\x30\x4d\x6a\x87\xa4\xc1\xde\xfb" - "\x18\x35\x52\x6f\x8c\xa9\xc6\xe3" - "\x00\x1f\x3e\x5d\x7c\x9b\xba\xd9" - "\xf8\x17\x36\x55\x74\x93\xb2\xd1" - "\xf0\x0f\x2e\x4d\x6c\x8b\xaa\xc9" - "\xe8\x07\x26\x45\x64\x83\xa2\xc1" - "\xe0\xff\x1e\x3d\x5c\x7b\x9a\xb9" - "\xd8\xf7\x16\x35\x54\x73\x92\xb1" - "\xd0\xef\x0e\x2d\x4c\x6b\x8a\xa9" - "\xc8\xe7\x06\x25\x44\x63\x82\xa1" - "\xc0\xdf\xfe\x1d\x3c\x5b\x7a\x99" - "\xb8\xd7\xf6\x15\x34\x53\x72\x91" - "\xb0\xcf\xee\x0d\x2c\x4b\x6a\x89" - "\xa8\xc7\xe6\x05\x24\x43\x62\x81" - "\xa0\xbf\xde\xfd\x1c\x3b\x5a\x79" - "\x98\xb7\xd6\xf5\x14\x33\x52\x71" - "\x90\xaf\xce\xed\x0c\x2b\x4a\x69" - "\x88\xa7\xc6\xe5\x04\x23\x42\x61" - "\x80\x9f\xbe\xdd\xfc\x1b\x3a\x59" - "\x78\x97\xb6\xd5\xf4\x13\x32\x51" - "\x70\x8f\xae\xcd\xec\x0b\x2a\x49" - "\x68\x87\xa6\xc5\xe4\x03\x22\x41" - "\x60\x7f\x9e\xbd\xdc\xfb\x1a\x39" - "\x58\x77\x96\xb5\xd4\xf3\x12\x31" - "\x50\x6f\x8e\xad\xcc\xeb\x0a\x29" - "\x48\x67\x86\xa5\xc4\xe3\x02\x21" - "\x40\x5f\x7e\x9d\xbc\xdb\xfa\x19" - "\x38\x57\x76\x95\xb4\xd3\xf2\x11" - "\x30\x4f\x6e\x8d\xac\xcb\xea\x09" - "\x28\x47\x66\x85\xa4\xc3\xe2\x01" - "\x20\x3f\x5e\x7d\x9c\xbb\xda\xf9" - "\x18\x37\x56\x75\x94\xb3\xd2\xf1" - "\x10\x2f\x4e\x6d\x8c\xab\xca\xe9" - "\x08\x27\x46\x65\x84\xa3\xc2\xe1" - "\x00\x21\x42\x63", - .ctext = - "\xb5\x81\xf5\x64\x18\x73\xe3\xf0" - "\x4c\x13\xf2\x77\x18\x60\x65\x5e" - "\x29\x01\xce\x98\x55\x53\xf9\x0c" - "\x2a\x08\xd5\x09\xb3\x57\x55\x56" - "\xc5\xe9\x56\x90\xcb\x6a\xa3\xc0" - "\xff\xc4\x79\xb4\xd2\x97\x5d\xc4" - "\x43\xd1\xfe\x94\x7b\x88\x06\x5a" - "\xb2\x9e\x2c\xfc\x44\x03\xb7\x90" - "\xa0\xc1\xba\x6a\x33\xb8\xc7\xb2" - "\x9d\xe1\x12\x4f\xc0\x64\xd4\x01" - "\xfe\x8c\x7a\x66\xf7\xe6\x5a\x91" - "\xbb\xde\x56\x86\xab\x65\x21\x30" - "\x00\x84\x65\x24\xa5\x7d\x85\xb4" - "\xe3\x17\xed\x3a\xb7\x6f\xb4\x0b" - "\x0b\xaf\x15\xae\x5a\x8f\xf2\x0c" - "\x2f\x27\xf4\x09\xd8\xd2\x96\xb7" - "\x71\xf2\xc5\x99\x4d\x7e\x7f\x75" - "\x77\x89\x30\x8b\x59\xdb\xa2\xb2" - "\xa0\xf3\x19\x39\x2b\xc5\x7e\x3f" - "\x4f\xd9\xd3\x56\x28\x97\x44\xdc" - "\xc0\x8b\x77\x24\xd9\x52\xe7\xc5" - "\xaf\xf6\x7d\x59\xb2\x44\x05\x1d" - "\xb1\xb0\x11\xa5\x0f\xec\x33\xe1" - "\x6d\x1b\x4e\x1f\xff\x57\x91\xb4" - "\x5b\x9a\x96\xc5\x53\xbc\xae\x20" - "\x3c\xbb\x14\xe2\xe8\x22\x33\xc1" - "\x5e\x76\x9e\x46\x99\xf6\x2a\x15" - "\xc6\x97\x02\xa0\x66\x43\xd1\xa6" - "\x31\xa6\x9f\xfb\xf4\xd3\x69\xe5" - "\xcd\x76\x95\xb8\x7a\x82\x7f\x21" - "\x45\xff\x3f\xce\x55\xf6\x95\x10" - "\x08\x77\x10\x43\xc6\xf3\x09\xe5" - "\x68\xe7\x3c\xad\x00\x52\x45\x0d" - "\xfe\x2d\xc6\xc2\x94\x8c\x12\x1d" - "\xe6\x25\xae\x98\x12\x8e\x19\x9c" - "\x81\x68\xb1\x11\xf6\x69\xda\xe3" - "\x62\x08\x18\x7a\x25\x49\x28\xac" - "\xba\x71\x12\x0b\xe4\xa2\xe5\xc7" - "\x5d\x8e\xec\x49\x40\x21\xbf\x5a" - "\x98\xf3\x02\x68\x55\x03\x7f\x8a" - "\xe5\x94\x0c\x32\x5c\x07\x82\x63" - "\xaf\x6f\x91\x40\x84\x8e\x52\x25" - "\xd0\xb0\x29\x53\x05\xe2\x50\x7a" - "\x34\xeb\xc9\x46\x20\xa8\x3d\xde" - "\x7f\x16\x5f\x36\xc5\x2e\xdc\xd1" - "\x15\x47\xc7\x50\x40\x6d\x91\xc5" - "\xe7\x93\x95\x1a\xd3\x57\xbc\x52" - "\x33\xee\x14\x19\x22\x52\x89\xa7" - "\x4a\x25\x56\x77\x4b\xca\xcf\x0a" - "\xe1\xf5\x35\x85\x30\x7e\x59\x4a" - "\xbd\x14\x5b\xdf\xe3\x46\xcb\xac" - "\x1f\x6c\x96\x0e\xf4\x81\xd1\x99" - "\xca\x88\x63\x3d\x02\x58\x6b\xa9" - "\xe5\x9f\xb3\x00\xb2\x54\xc6\x74" - "\x1c\xbf\x46\xab\x97\xcc\xf8\x54" - "\x04\x07\x08\x52\xe6\xc0\xda\x93" - "\x74\x7d\x93\x99\x5d\x78\x68\xa6" - "\x2e\x6b\xd3\x6a\x69\xcc\x12\x6b" - "\xd4\xc7\xa5\xc6\xe7\xf6\x03\x04" - "\x5d\xcd\x61\x5e\x17\x40\xdc\xd1" - "\x5c\xf5\x08\xdf\x5c\x90\x85\xa4" - "\xaf\xf6\x78\xbb\x0d\xf1\xf4\xa4" - "\x54\x26\x72\x9e\x61\xfa\x86\xcf" - "\xe8\x9e\xa1\xe0\xc7\x48\x23\xae" - "\x5a\x90\xae\x75\x0a\x74\x18\x89" - "\x05\xb1\x92\xb2\x7f\xd0\x1b\xa6" - "\x62\x07\x25\x01\xc7\xc2\x4f\xf9" - "\xe8\xfe\x63\x95\x80\x07\xb4\x26" - "\xcc\xd1\x26\xb6\xc4\x3f\x9e\xcb" - "\x8e\x3b\x2e\x44\x16\xd3\x10\x9a" - "\x95\x08\xeb\xc8\xcb\xeb\xbf\x6f" - "\x0b\xcd\x1f\xc8\xca\x86\xaa\xec" - "\x33\xe6\x69\xf4\x45\x25\x86\x3a" - "\x22\x94\x4f\x00\x23\x6a\x44\xc2" - "\x49\x97\x33\xab\x36\x14\x0a\x70" - "\x24\xc3\xbe\x04\x3b\x79\xa0\xf9" - "\xb8\xe7\x76\x29\x22\x83\xd7\xf2" - "\x94\xf4\x41\x49\xba\x5f\x7b\x07" - "\xb5\xfb\xdb\x03\x1a\x9f\xb6\x4c" - "\xc2\x2e\x37\x40\x49\xc3\x38\x16" - "\xe2\x4f\x77\x82\xb0\x68\x4c\x71" - "\x1d\x57\x61\x9c\xd9\x4e\x54\x99" - "\x47\x13\x28\x73\x3c\xbb\x00\x90" - "\xf3\x4d\xc9\x0e\xfd\xe7\xb1\x71" - "\xd3\x15\x79\xbf\xcc\x26\x2f\xbd" - "\xad\x6c\x50\x69\x6c\x3e\x6d\x80" - "\x9a\xea\x78\xaf\x19\xb2\x0d\x4d" - "\xad\x04\x07\xae\x22\x90\x4a\x93" - "\x32\x0e\x36\x9b\x1b\x46\xba\x3b" - "\xb4\xac\xc6\xd1\xa2\x31\x53\x3b" - "\x2a\x3d\x45\xfe\x03\x61\x10\x85" - "\x17\x69\xa6\x78\xcc\x6c\x87\x49" - "\x53\xf9\x80\x10\xde\x80\xa2\x41" - "\x6a\xc3\x32\x02\xad\x6d\x3c\x56" - "\x00\x71\x51\x06\xa7\xbd\xfb\xef" - "\x3c\xb5\x9f\xfc\x48\x7d\x53\x7c" - "\x66\xb0\x49\x23\xc4\x47\x10\x0e" - "\xe5\x6c\x74\x13\xe6\xc5\x3f\xaa" - "\xde\xff\x07\x44\xdd\x56\x1b\xad" - "\x09\x77\xfb\x5b\x12\xb8\x0d\x38" - "\x17\x37\x35\x7b\x9b\xbc\xfe\xd4" - "\x7e\x8b\xda\x7e\x5b\x04\xa7\x22" - "\xa7\x31\xa1\x20\x86\xc7\x1b\x99" - "\xdb\xd1\x89\xf4\x94\xa3\x53\x69" - "\x8d\xe7\xe8\x74\x11\x8d\x74\xd6" - "\x07\x37\x91\x9f\xfd\x67\x50\x3a" - "\xc9\xe1\xf4\x36\xd5\xa0\x47\xd1" - "\xf9\xe5\x39\xa3\x31\xac\x07\x36" - "\x23\xf8\x66\x18\x14\x28\x34\x0f" - "\xb8\xd0\xe7\x29\xb3\x04\x4b\x55" - "\x01\x41\xb2\x75\x8d\xcb\x96\x85" - "\x3a\xfb\xab\x2b\x9e\xfa\x58\x20" - "\x44\x1f\xc0\x14\x22\x75\x61\xe8" - "\xaa\x19\xcf\xf1\x82\x56\xf4\xd7" - "\x78\x7b\x3d\x5f\xb3\x9e\x0b\x8a" - "\x57\x50\xdb\x17\x41\x65\x4d\xa3" - "\x02\xc9\x9c\x9c\x53\xfb\x39\x39" - "\x9b\x1d\x72\x24\xda\xb7\x39\xbe" - "\x13\x3b\xfa\x29\xda\x9e\x54\x64" - "\x6e\xba\xd8\xa1\xcb\xb3\x36\xfa" - "\xcb\x47\x85\xe9\x61\x38\xbc\xbe" - "\xc5\x00\x38\x2a\x54\xf7\xc4\xb9" - "\xb3\xd3\x7b\xa0\xa0\xf8\x72\x7f" - "\x8c\x8e\x82\x0e\xc6\x1c\x75\x9d" - "\xca\x8e\x61\x87\xde\xad\x80\xd2" - "\xf5\xf9\x80\xef\x15\x75\xaf\xf5" - "\x80\xfb\xff\x6d\x1e\x25\xb7\x40" - "\x61\x6a\x39\x5a\x6a\xb5\x31\xab" - "\x97\x8a\x19\x89\x44\x40\xc0\xa6" - "\xb4\x4e\x30\x32\x7b\x13\xe7\x67" - "\xa9\x8b\x57\x04\xc2\x01\xa6\xf4" - "\x28\x99\xad\x2c\x76\xa3\x78\xc2" - "\x4a\xe6\xca\x5c\x50\x6a\xc1\xb0" - "\x62\x4b\x10\x8e\x7c\x17\x43\xb3" - "\x17\x66\x1c\x3e\x8d\x69\xf0\x5a" - "\x71\xf5\x97\xdc\xd1\x45\xdd\x28" - "\xf3\x5d\xdf\x53\x7b\x11\xe5\xbc" - "\x4c\xdb\x1b\x51\x6b\xe9\xfb\x3d" - "\xc1\xc3\x2c\xb9\x71\xf5\xb6\xb2" - "\x13\x36\x79\x80\x53\xe8\xd3\xa6" - "\x0a\xaf\xfd\x56\x97\xf7\x40\x8e" - "\x45\xce\xf8\xb0\x9e\x5c\x33\x82" - "\xb0\x44\x56\xfc\x05\x09\xe9\x2a" - "\xac\x26\x80\x14\x1d\xc8\x3a\x35" - "\x4c\x82\x97\xfd\x76\xb7\xa9\x0a" - "\x35\x58\x79\x8e\x0f\x66\xea\xaf" - "\x51\x6c\x09\xa9\x6e\x9b\xcb\x9a" - "\x31\x47\xa0\x2f\x7c\x71\xb4\x4a" - "\x11\xaa\x8c\x66\xc5\x64\xe6\x3a" - "\x54\xda\x24\x6a\xc4\x41\x65\x46" - "\x82\xa0\x0a\x0f\x5f\xfb\x25\xd0" - "\x2c\x91\xa7\xee\xc4\x81\x07\x86" - "\x75\x5e\x33\x69\x97\xe4\x2c\xa8" - "\x9d\x9f\x0b\x6a\xbe\xad\x98\xda" - "\x6d\x94\x41\xda\x2c\x1e\x89\xc4" - "\xc2\xaf\x1e\x00\x05\x0b\x83\x60" - "\xbd\x43\xea\x15\x23\x7f\xb9\xac" - "\xee\x4f\x2c\xaf\x2a\xf3\xdf\xd0" - "\xf3\x19\x31\xbb\x4a\x74\x84\x17" - "\x52\x32\x2c\x7d\x61\xe4\xcb\xeb" - "\x80\x38\x15\x52\xcb\x6f\xea\xe5" - "\x73\x9c\xd9\x24\x69\xc6\x95\x32" - "\x21\xc8\x11\xe4\xdc\x36\xd7\x93" - "\x38\x66\xfb\xb2\x7f\x3a\xb9\xaf" - "\x31\xdd\x93\x75\x78\x8a\x2c\x94" - "\x87\x1a\x58\xec\x9e\x7d\x4d\xba" - "\xe1\xe5\x4d\xfc\xbc\xa4\x2a\x14" - "\xef\xcc\xa7\xec\xab\x43\x09\x18" - "\xd3\xab\x68\xd1\x07\x99\x44\x47" - "\xd6\x83\x85\x3b\x30\xea\xa9\x6b" - "\x63\xea\xc4\x07\xfb\x43\x2f\xa4" - "\xaa\xb0\xab\x03\x89\xce\x3f\x8c" - "\x02\x7c\x86\x54\xbc\x88\xaf\x75" - "\xd2\xdc\x63\x17\xd3\x26\xf6\x96" - "\xa9\x3c\xf1\x61\x8c\x11\x18\xcc" - "\xd6\xea\x5b\xe2\xcd\xf0\xf1\xb2" - "\xe5\x35\x90\x1f\x85\x4c\x76\x5b" - "\x66\xce\x44\xa4\x32\x9f\xe6\x7b" - "\x71\x6e\x9f\x58\x15\x67\x72\x87" - "\x64\x8e\x3a\x44\x45\xd4\x76\xfa" - "\xc2\xf6\xef\x85\x05\x18\x7a\x9b" - "\xba\x41\x54\xac\xf0\xfc\x59\x12" - "\x3f\xdf\xa0\xe5\x8a\x65\xfd\x3a" - "\x62\x8d\x83\x2c\x03\xbe\x05\x76" - "\x2e\x53\x49\x97\x94\x33\xae\x40" - "\x81\x15\xdb\x6e\xad\xaa\xf5\x4b" - "\xe3\x98\x70\xdf\xe0\x7c\xcd\xdb" - "\x02\xd4\x7d\x2f\xc1\xe6\xb4\xf3" - "\xd7\x0d\x7a\xd9\x23\x9e\x87\x2d" - "\xce\x87\xad\xcc\x72\x05\x00\x29" - "\xdc\x73\x7f\x64\xc1\x15\x0e\xc2" - "\xdf\xa7\x5f\xeb\x41\xa1\xcd\xef" - "\x5c\x50\x79\x2a\x56\x56\x71\x8c" - "\xac\xc0\x79\x50\x69\xca\x59\x32" - "\x65\xf2\x54\xe4\x52\x38\x76\xd1" - "\x5e\xde\x26\x9e\xfb\x75\x2e\x11" - "\xb5\x10\xf4\x17\x73\xf5\x89\xc7" - "\x4f\x43\x5c\x8e\x7c\xb9\x05\x52" - "\x24\x40\x99\xfe\x9b\x85\x0b\x6c" - "\x22\x3e\x8b\xae\x86\xa1\xd2\x79" - "\x05\x68\x6b\xab\xe3\x41\x49\xed" - "\x15\xa1\x8d\x40\x2d\x61\xdf\x1a" - "\x59\xc9\x26\x8b\xef\x30\x4c\x88" - "\x4b\x10\xf8\x8d\xa6\x92\x9f\x4b" - "\xf3\xc4\x53\x0b\x89\x5d\x28\x92" - "\xcf\x78\xb2\xc0\x5d\xed\x7e\xfc" - "\xc0\x12\x23\x5f\x5a\x78\x86\x43" - "\x6e\x27\xf7\x5a\xa7\x6a\xed\x19" - "\x04\xf0\xb3\x12\xd1\xbd\x0e\x89" - "\x6e\xbc\x96\xa8\xd8\x49\x39\x9f" - "\x7e\x67\xf0\x2e\x3e\x01\xa9\xba" - "\xec\x8b\x62\x8e\xcb\x4a\x70\x43" - "\xc7\xc2\xc4\xca\x82\x03\x73\xe9" - "\x11\xdf\xcf\x54\xea\xc9\xb0\x95" - "\x51\xc0\x13\x3d\x92\x05\xfa\xf4" - "\xa9\x34\xc8\xce\x6c\x3d\x54\xcc" - "\xc4\xaf\xf1\xdc\x11\x44\x26\xa2" - "\xaf\xf1\x85\x75\x7d\x03\x61\x68" - "\x4e\x78\xc6\x92\x7d\x86\x7d\x77" - "\xdc\x71\x72\xdb\xc6\xae\xa1\xcb" - "\x70\x9a\x0b\x19\xbe\x4a\x6c\x2a" - "\xe2\xba\x6c\x64\x9a\x13\x28\xdf" - "\x85\x75\xe6\x43\xf6\x87\x08\x68" - "\x6e\xba\x6e\x79\x9f\x04\xbc\x23" - "\x50\xf6\x33\x5c\x1f\x24\x25\xbe" - "\x33\x47\x80\x45\x56\xa3\xa7\xd7" - "\x7a\xb1\x34\x0b\x90\x3c\x9c\xad" - "\x44\x5f\x9e\x0e\x9d\xd4\xbd\x93" - "\x5e\xfa\x3c\xe0\xb0\xd9\xed\xf3" - "\xd6\x2e\xff\x24\xd8\x71\x6c\xed" - "\xaf\x55\xeb\x22\xac\x93\x68\x32" - "\x05\x5b\x47\xdd\xc6\x4a\xcb\xc7" - "\x10\xe1\x3c\x92\x1a\xf3\x23\x78" - "\x2b\xa1\xd2\x80\xf4\x12\xb1\x20" - "\x8f\xff\x26\x35\xdd\xfb\xc7\x4e" - "\x78\xf1\x2d\x50\x12\x77\xa8\x60" - "\x7c\x0f\xf5\x16\x2f\x63\x70\x2a" - "\xc0\x96\x80\x4e\x0a\xb4\x93\x35" - "\x5d\x1d\x3f\x56\xf7\x2f\xbb\x90" - "\x11\x16\x8f\xa2\xec\x47\xbe\xac" - "\x56\x01\x26\x56\xb1\x8c\xb2\x10" - "\xf9\x1a\xca\xf5\xd1\xb7\x39\x20" - "\x63\xf1\x69\x20\x4f\x13\x12\x1f" - "\x5b\x65\xfc\x98\xf7\xc4\x7a\xbe" - "\xf7\x26\x4d\x2b\x84\x7b\x42\xad" - "\xd8\x7a\x0a\xb4\xd8\x74\xbf\xc1" - "\xf0\x6e\xb4\x29\xa3\xbb\xca\x46" - "\x67\x70\x6a\x2d\xce\x0e\xa2\x8a" - "\xa9\x87\xbf\x05\xc4\xc1\x04\xa3" - "\xab\xd4\x45\x43\x8c\xb6\x02\xb0" - "\x41\xc8\xfc\x44\x3d\x59\xaa\x2e" - "\x44\x21\x2a\x8d\x88\x9d\x57\xf4" - "\xa0\x02\x77\xb8\xa6\xa0\xe6\x75" - "\x5c\x82\x65\x3e\x03\x5c\x29\x8f" - "\x38\x55\xab\x33\x26\xef\x9f\x43" - "\x52\xfd\x68\xaf\x36\xb4\xbb\x9a" - "\x58\x09\x09\x1b\xc3\x65\x46\x46" - "\x1d\xa7\x94\x18\x23\x50\x2c\xca" - "\x2c\x55\x19\x97\x01\x9d\x93\x3b" - "\x63\x86\xf2\x03\x67\x45\xd2\x72" - "\x28\x52\x6c\xf4\xe3\x1c\xb5\x11" - "\x13\xf1\xeb\x21\xc7\xd9\x56\x82" - "\x2b\x82\x39\xbd\x69\x54\xed\x62" - "\xc3\xe2\xde\x73\xd4\x6a\x12\xae" - "\x13\x21\x7f\x4b\x5b\xfc\xbf\xe8" - "\x2b\xbe\x56\xba\x68\x8b\x9a\xb1" - "\x6e\xfa\xbf\x7e\x5a\x4b\xf1\xac" - "\x98\x65\x85\xd1\x93\x53\xd3\x7b" - "\x09\xdd\x4b\x10\x6d\x84\xb0\x13" - "\x65\xbd\xcf\x52\x09\xc4\x85\xe2" - "\x84\x74\x15\x65\xb7\xf7\x51\xaf" - "\x55\xad\xa4\xd1\x22\x54\x70\x94" - "\xa0\x1c\x90\x41\xfd\x99\xd7\x5a" - "\x31\xef\xaa\x25\xd0\x7f\x4f\xea" - "\x1d\x55\x42\xe5\x49\xb0\xd0\x46" - "\x62\x36\x43\xb2\x82\x15\x75\x50" - "\xa4\x72\xeb\x54\x27\x1f\x8a\xe4" - "\x7d\xe9\x66\xc5\xf1\x53\xa4\xd1" - "\x0c\xeb\xb8\xf8\xbc\xd4\xe2\xe7" - "\xe1\xf8\x4b\xcb\xa9\xa1\xaf\x15" - "\x83\xcb\x72\xd0\x33\x79\x00\x2d" - "\x9f\xd7\xf1\x2e\x1e\x10\xe4\x45" - "\xc0\x75\x3a\x39\xea\x68\xf7\x5d" - "\x1b\x73\x8f\xe9\x8e\x0f\x72\x47" - "\xae\x35\x0a\x31\x7a\x14\x4d\x4a" - "\x6f\x47\xf7\x7e\x91\x6e\x74\x8b" - "\x26\x47\xf9\xc3\xf9\xde\x70\xf5" - "\x61\xab\xa9\x27\x9f\x82\xe4\x9c" - "\x89\x91\x3f\x2e\x6a\xfd\xb5\x49" - "\xe9\xfd\x59\x14\x36\x49\x40\x6d" - "\x32\xd8\x85\x42\xf3\xa5\xdf\x0c" - "\xa8\x27\xd7\x54\xe2\x63\x2f\xf2" - "\x7e\x8b\x8b\xe7\xf1\x9a\x95\x35" - "\x43\xdc\x3a\xe4\xb6\xf4\xd0\xdf" - "\x9c\xcb\x94\xf3\x21\xa0\x77\x50" - "\xe2\xc6\xc4\xc6\x5f\x09\x64\x5b" - "\x92\x90\xd8\xe1\xd1\xed\x4b\x42" - "\xd7\x37\xaf\x65\x3d\x11\x39\xb6" - "\x24\x8a\x60\xae\xd6\x1e\xbf\x0e" - "\x0d\xd7\xdc\x96\x0e\x65\x75\x4e" - "\x29\x06\x9d\xa4\x51\x3a\x10\x63" - "\x8f\x17\x07\xd5\x8e\x3c\xf4\x28" - "\x00\x5a\x5b\x05\x19\xd8\xc0\x6c" - "\xe5\x15\xe4\x9c\x9d\x71\x9d\x5e" - "\x94\x29\x1a\xa7\x80\xfa\x0e\x33" - "\x03\xdd\xb7\x3e\x9a\xa9\x26\x18" - "\x37\xa9\x64\x08\x4d\x94\x5a\x88" - "\xca\x35\xce\x81\x02\xe3\x1f\x1b" - "\x89\x1a\x77\x85\xe3\x41\x6d\x32" - "\x42\x19\x23\x7d\xc8\x73\xee\x25" - "\x85\x0d\xf8\x31\x25\x79\x1b\x6f" - "\x79\x25\xd2\xd8\xd4\x23\xfd\xf7" - "\x82\x36\x6a\x0c\x46\x22\x15\xe9" - "\xff\x72\x41\x91\x91\x7d\x3a\xb7" - "\xdd\x65\x99\x70\xf6\x8d\x84\xf8" - "\x67\x15\x20\x11\xd6\xb2\x55\x7b" - "\xdb\x87\xee\xef\x55\x89\x2a\x59" - "\x2b\x07\x8f\x43\x8a\x59\x3c\x01" - "\x8b\x65\x54\xa1\x66\xd5\x38\xbd" - "\xc6\x30\xa9\xcc\x49\xb6\xa8\x1b" - "\xb8\xc0\x0e\xe3\x45\x28\xe2\xff" - "\x41\x9f\x7e\x7c\xd1\xae\x9e\x25" - "\x3f\x4c\x7c\x7c\xf4\xa8\x26\x4d" - "\x5c\xfd\x4b\x27\x18\xf9\x61\x76" - "\x48\xba\x0c\x6b\xa9\x4d\xfc\xf5" - "\x3b\x35\x7e\x2f\x4a\xa9\xc2\x9a" - "\xae\xab\x86\x09\x89\xc9\xc2\x40" - "\x39\x2c\x81\xb3\xb8\x17\x67\xc2" - "\x0d\x32\x4a\x3a\x67\x81\xd7\x1a" - "\x34\x52\xc5\xdb\x0a\xf5\x63\x39" - "\xea\x1f\xe1\x7c\xa1\x9e\xc1\x35" - "\xe3\xb1\x18\x45\x67\xf9\x22\x38" - "\x95\xd9\x34\x34\x86\xc6\x41\x94" - "\x15\xf9\x5b\x41\xa6\x87\x8b\xf8" - "\xd5\xe1\x1b\xe2\x5b\xf3\x86\x10" - "\xff\xe6\xae\x69\x76\xbc\x0d\xb4" - "\x09\x90\x0c\xa2\x65\x0c\xad\x74" - "\xf5\xd7\xff\xda\xc1\xce\x85\xbe" - "\x00\xa7\xff\x4d\x2f\x65\xd3\x8c" - "\x86\x2d\x05\xe8\xed\x3e\x6b\x8b" - "\x0f\x3d\x83\x8c\xf1\x1d\x5b\x96" - "\x2e\xb1\x9c\xc2\x98\xe1\x70\xb9" - "\xba\x5c\x8a\x43\xd6\x34\xa7\x2d" - "\xc9\x92\xae\xf2\xa5\x7b\x05\x49" - "\xa7\x33\x34\x86\xca\xe4\x96\x23" - "\x76\x5b\xf2\xc6\xf1\x51\x28\x42" - "\x7b\xcc\x76\x8f\xfa\xa2\xad\x31" - "\xd4\xd6\x7a\x6d\x25\x25\x54\xe4" - "\x3f\x50\x59\xe1\x5c\x05\xb7\x27" - "\x48\xbf\x07\xec\x1b\x13\xbe\x2b" - "\xa1\x57\x2b\xd5\xab\xd7\xd0\x4c" - "\x1e\xcb\x71\x9b\xc5\x90\x85\xd3" - "\xde\x59\xec\x71\xeb\x89\xbb\xd0" - "\x09\x50\xe1\x16\x3f\xfd\x1c\x34" - "\xc3\x1c\xa1\x10\x77\x53\x98\xef" - "\xf2\xfd\xa5\x01\x59\xc2\x9b\x26" - "\xc7\x42\xd9\x49\xda\x58\x2b\x6e" - "\x9f\x53\x19\x76\x7e\xd9\xc9\x0e" - "\x68\xc8\x7f\x51\x22\x42\xef\x49" - "\xa4\x55\xb6\x36\xac\x09\xc7\x31" - "\x88\x15\x4b\x2e\x8f\x3a\x08\xf7" - "\xd8\xf7\xa8\xc5\xa9\x33\xa6\x45" - "\xe4\xc4\x94\x76\xf3\x0d\x8f\x7e" - "\xc8\xf6\xbc\x23\x0a\xb6\x4c\xd3" - "\x6a\xcd\x36\xc2\x90\x5c\x5c\x3c" - "\x65\x7b\xc2\xd6\xcc\xe6\x0d\x87" - "\x73\x2e\x71\x79\x16\x06\x63\x28" - "\x09\x15\xd8\x89\x38\x38\x3d\xb5" - "\x42\x1c\x08\x24\xf7\x2a\xd2\x9d" - "\xc8\xca\xef\xf9\x27\xd8\x07\x86" - "\xf7\x43\x0b\x55\x15\x3f\x9f\x83" - "\xef\xdc\x49\x9d\x2a\xc1\x54\x62" - "\xbd\x9b\x66\x55\x9f\xb7\x12\xf3" - "\x1b\x4d\x9d\x2a\x5c\xed\x87\x75" - "\x87\x26\xec\x61\x2c\xb4\x0f\x89" - "\xb0\xfb\x2e\x68\x5d\x15\xc7\x8d" - "\x2e\xc0\xd9\xec\xaf\x4f\xd2\x25" - "\x29\xe8\xd2\x26\x2b\x67\xe9\xfc" - "\x2b\xa8\x67\x96\x12\x1f\x5b\x96" - "\xc6\x14\x53\xaf\x44\xea\xd6\xe2" - "\x94\x98\xe4\x12\x93\x4c\x92\xe0" - "\x18\xa5\x8d\x2d\xe4\x71\x3c\x47" - "\x4c\xf7\xe6\x47\x9e\xc0\x68\xdf" - "\xd4\xf5\x5a\x74\xb1\x2b\x29\x03" - "\x19\x07\xaf\x90\x62\x5c\x68\x98" - "\x48\x16\x11\x02\x9d\xee\xb4\x9b" - "\xe5\x42\x7f\x08\xfd\x16\x32\x0b" - "\xd0\xb3\xfa\x2b\xb7\x99\xf9\x29" - "\xcd\x20\x45\x9f\xb3\x1a\x5d\xa2" - "\xaf\x4d\xe0\xbd\x42\x0d\xbc\x74" - "\x99\x9c\x8e\x53\x1a\xb4\x3e\xbd" - "\xa2\x9a\x2d\xf7\xf8\x39\x0f\x67" - "\x63\xfc\x6b\xc0\xaf\xb3\x4b\x4f" - "\x55\xc4\xcf\xa7\xc8\x04\x11\x3e" - "\x14\x32\xbb\x1b\x38\x77\xd6\x7f" - "\x54\x4c\xdf\x75\xf3\x07\x2d\x33" - "\x9b\xa8\x20\xe1\x7b\x12\xb5\xf3" - "\xef\x2f\xce\x72\xe5\x24\x60\xc1" - "\x30\xe2\xab\xa1\x8e\x11\x09\xa8" - "\x21\x33\x44\xfe\x7f\x35\x32\x93" - "\x39\xa7\xad\x8b\x79\x06\xb2\xcb" - "\x4e\xa9\x5f\xc7\xba\x74\x29\xec" - "\x93\xa0\x4e\x54\x93\xc0\xbc\x55" - "\x64\xf0\x48\xe5\x57\x99\xee\x75" - "\xd6\x79\x0f\x66\xb7\xc6\x57\x76" - "\xf7\xb7\xf3\x9c\xc5\x60\xe8\x7f" - "\x83\x76\xd6\x0e\xaa\xe6\x90\x39" - "\x1d\xa6\x32\x6a\x34\xe3\x55\xf8" - "\x58\xa0\x58\x7d\x33\xe0\x22\x39" - "\x44\x64\x87\x86\x5a\x2f\xa7\x7e" - "\x0f\x38\xea\xb0\x30\xcc\x61\xa5" - "\x6a\x32\xae\x1e\xf7\xe9\xd0\xa9" - "\x0c\x32\x4b\xb5\x49\x28\xab\x85" - "\x2f\x8e\x01\x36\x38\x52\xd0\xba" - "\xd6\x02\x78\xf8\x0e\x3e\x9c\x8b" - "\x6b\x45\x99\x3f\x5c\xfe\x58\xf1" - "\x5c\x94\x04\xe1\xf5\x18\x6d\x51" - "\xb2\x5d\x18\x20\xb6\xc2\x9a\x42" - "\x1d\xb3\xab\x3c\xb6\x3a\x13\x03" - "\xb2\x46\x82\x4f\xfc\x64\xbc\x4f" - "\xca\xfa\x9c\xc0\xd5\xa7\xbd\x11" - "\xb7\xe4\x5a\xf6\x6f\x4d\x4d\x54" - "\xea\xa4\x98\x66\xd4\x22\x3b\xd3" - "\x8f\x34\x47\xd9\x7c\xf4\x72\x3b" - "\x4d\x02\x77\xf6\xd6\xdd\x08\x0a" - "\x81\xe1\x86\x89\x3e\x56\x10\x3c" - "\xba\xd7\x81\x8c\x08\xbc\x8b\xe2" - "\x53\xec\xa7\x89\xee\xc8\x56\xb5" - "\x36\x2c\xb2\x03\xba\x99\xdd\x7c" - "\x48\xa0\xb0\xbc\x91\x33\xe9\xa8" - "\xcb\xcd\xcf\x59\x5f\x1f\x15\xe2" - "\x56\xf5\x4e\x01\x35\x27\x45\x77" - "\x47\xc8\xbc\xcb\x7e\x39\xc1\x97" - "\x28\xd3\x84\xfc\x2c\x3e\xc8\xad" - "\x9c\xf8\x8a\x61\x9c\x28\xaa\xc5" - "\x99\x20\x43\x85\x9d\xa5\xe2\x8b" - "\xb8\xae\xeb\xd0\x32\x0d\x52\x78" - "\x09\x56\x3f\xc7\xd8\x7e\x26\xfc" - "\x37\xfb\x6f\x04\xfc\xfa\x92\x10" - "\xac\xf8\x3e\x21\xdc\x8c\x21\x16" - "\x7d\x67\x6e\xf6\xcd\xda\xb6\x98" - "\x23\xab\x23\x3c\xb2\x10\xa0\x53" - "\x5a\x56\x9f\xc5\xd0\xff\xbb\xe4" - "\x98\x3c\x69\x1e\xdb\x38\x8f\x7e" - "\x0f\xd2\x98\x88\x81\x8b\x45\x67" - "\xea\x33\xf1\xeb\xe9\x97\x55\x2e" - "\xd9\xaa\xeb\x5a\xec\xda\xe1\x68" - "\xa8\x9d\x3c\x84\x7c\x05\x3d\x62" - "\x87\x8f\x03\x21\x28\x95\x0c\x89" - "\x25\x22\x4a\xb0\x93\xa9\x50\xa2" - "\x2f\x57\x6e\x18\x42\x19\x54\x0c" - "\x55\x67\xc6\x11\x49\xf4\x5c\xd2" - "\xe9\x3d\xdd\x8b\x48\x71\x21\x00" - "\xc3\x9a\x6c\x85\x74\x28\x83\x4a" - "\x1b\x31\x05\xe1\x06\x92\xe7\xda" - "\x85\x73\x78\x45\x20\x7f\xae\x13" - "\x7c\x33\x06\x22\xf4\x83\xf9\x35" - "\x3f\x6c\x71\xa8\x4e\x48\xbe\x9b" - "\xce\x8a\xba\xda\xbe\x28\x08\xf7" - "\xe2\x14\x8c\x71\xea\x72\xf9\x33" - "\xf2\x88\x3f\xd7\xbb\x69\x6c\x29" - "\x19\xdc\x84\xce\x1f\x12\x4f\xc8" - "\xaf\xa5\x04\xba\x5a\xab\xb0\xd9" - "\x14\x1f\x6c\x68\x98\x39\x89\x7a" - "\xd9\xd8\x2f\xdf\xa8\x47\x4a\x25" - "\xe2\xfb\x33\xf4\x59\x78\xe1\x68" - "\x85\xcf\xfe\x59\x20\xd4\x05\x1d" - "\x80\x99\xae\xbc\xca\xae\x0f\x2f" - "\x65\x43\x34\x8e\x7e\xac\xd3\x93" - "\x2f\xac\x6d\x14\x3d\x02\x07\x70" - "\x9d\xa4\xf3\x1b\x5c\x36\xfc\x01" - "\x73\x34\x85\x0c\x6c\xd6\xf1\xbd" - "\x3f\xdf\xee\xf5\xd9\xba\x56\xef" - "\xf4\x9b\x6b\xee\x9f\x5a\x78\x6d" - "\x32\x19\xf4\xf7\xf8\x4c\x69\x0b" - "\x4b\xbc\xbb\xb7\xf2\x85\xaf\x70" - "\x75\x24\x6c\x54\xa7\x0e\x4d\x1d" - "\x01\xbf\x08\xac\xcf\x7f\x2c\xe3" - "\x14\x89\x5e\x70\x5a\x99\x92\xcd" - "\x01\x84\xc8\xd2\xab\xe5\x4f\x58" - "\xe7\x0f\x2f\x0e\xff\x68\xea\xfd" - "\x15\xb3\x17\xe6\xb0\xe7\x85\xd8" - "\x23\x2e\x05\xc7\xc9\xc4\x46\x1f" - "\xe1\x9e\x49\x20\x23\x24\x4d\x7e" - "\x29\x65\xff\xf4\xb6\xfd\x1a\x85" - "\xc4\x16\xec\xfc\xea\x7b\xd6\x2c" - "\x43\xf8\xb7\xbf\x79\xc0\x85\xcd" - "\xef\xe1\x98\xd3\xa5\xf7\x90\x8c" - "\xe9\x7f\x80\x6b\xd2\xac\x4c\x30" - "\xa7\xc6\x61\x6c\xd2\xf9\x2c\xff" - "\x30\xbc\x22\x81\x7d\x93\x12\xe4" - "\x0a\xcd\xaf\xdd\xe8\xab\x0a\x1e" - "\x13\xa4\x27\xc3\x5f\xf7\x4b\xbb" - "\x37\x09\x4b\x91\x6f\x92\x4f\xaf" - "\x52\xee\xdf\xef\x09\x6f\xf7\x5c" - "\x6e\x12\x17\x72\x63\x57\xc7\xba" - "\x3b\x6b\x38\x32\x73\x1b\x9c\x80" - "\xc1\x7a\xc6\xcf\xcd\x35\xc0\x6b" - "\x31\x1a\x6b\xe9\xd8\x2c\x29\x3f" - "\x96\xfb\xb6\xcd\x13\x91\x3b\xc2" - "\xd2\xa3\x31\x8d\xa4\xcd\x57\xcd" - "\x13\x3d\x64\xfd\x06\xce\xe6\xdc" - "\x0c\x24\x43\x31\x40\x57\xf1\x72" - "\x17\xe3\x3a\x63\x6d\x35\xcf\x5d" - "\x97\x40\x59\xdd\xf7\x3c\x02\xf7" - "\x1c\x7e\x05\xbb\xa9\x0d\x01\xb1" - "\x8e\xc0\x30\xa9\x53\x24\xc9\x89" - "\x84\x6d\xaa\xd0\xcd\x91\xc2\x4d" - "\x91\xb0\x89\xe2\xbf\x83\x44\xaa" - "\x28\x72\x23\xa0\xc2\xad\xad\x1c" - "\xfc\x3f\x09\x7a\x0b\xdc\xc5\x1b" - "\x87\x13\xc6\x5b\x59\x8d\xf2\xc8" - "\xaf\xdf\x11\x95", - .len = 4100, - }, -}; - static const struct cipher_testvec chacha20_tv_template[] = { { /* RFC7539 A.2. Test Vector #1 */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00" diff --git a/crypto/tgr192.c b/crypto/tgr192.c deleted file mode 100644 index aa29c529b44e..000000000000 --- a/crypto/tgr192.c +++ /dev/null @@ -1,682 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * Tiger hashing Algorithm - * - * Copyright (C) 1998 Free Software Foundation, Inc. - * - * The Tiger algorithm was developed by Ross Anderson and Eli Biham. - * It was optimized for 64-bit processors while still delievering - * decent performance on 32 and 16-bit processors. - * - * This version is derived from the GnuPG implementation and the - * Tiger-Perl interface written by Rafael Sevilla - * - * Adapted for Linux Kernel Crypto by Aaron Grothe - * ajgrothe@yahoo.com, February 22, 2005 - */ -#include -#include -#include -#include -#include -#include -#include - -#define TGR192_DIGEST_SIZE 24 -#define TGR160_DIGEST_SIZE 20 -#define TGR128_DIGEST_SIZE 16 - -#define TGR192_BLOCK_SIZE 64 - -struct tgr192_ctx { - u64 a, b, c; - u8 hash[64]; - int count; - u32 nblocks; -}; - -static const u64 sbox1[256] = { - 0x02aab17cf7e90c5eULL, 0xac424b03e243a8ecULL, 0x72cd5be30dd5fcd3ULL, - 0x6d019b93f6f97f3aULL, 0xcd9978ffd21f9193ULL, 0x7573a1c9708029e2ULL, - 0xb164326b922a83c3ULL, 0x46883eee04915870ULL, 0xeaace3057103ece6ULL, - 0xc54169b808a3535cULL, 0x4ce754918ddec47cULL, 0x0aa2f4dfdc0df40cULL, - 0x10b76f18a74dbefaULL, 0xc6ccb6235ad1ab6aULL, 0x13726121572fe2ffULL, - 0x1a488c6f199d921eULL, 0x4bc9f9f4da0007caULL, 0x26f5e6f6e85241c7ULL, - 0x859079dbea5947b6ULL, 0x4f1885c5c99e8c92ULL, 0xd78e761ea96f864bULL, - 0x8e36428c52b5c17dULL, 0x69cf6827373063c1ULL, 0xb607c93d9bb4c56eULL, - 0x7d820e760e76b5eaULL, 0x645c9cc6f07fdc42ULL, 0xbf38a078243342e0ULL, - 0x5f6b343c9d2e7d04ULL, 0xf2c28aeb600b0ec6ULL, 0x6c0ed85f7254bcacULL, - 0x71592281a4db4fe5ULL, 0x1967fa69ce0fed9fULL, 0xfd5293f8b96545dbULL, - 0xc879e9d7f2a7600bULL, 0x860248920193194eULL, 0xa4f9533b2d9cc0b3ULL, - 0x9053836c15957613ULL, 0xdb6dcf8afc357bf1ULL, 0x18beea7a7a370f57ULL, - 0x037117ca50b99066ULL, 0x6ab30a9774424a35ULL, 0xf4e92f02e325249bULL, - 0x7739db07061ccae1ULL, 0xd8f3b49ceca42a05ULL, 0xbd56be3f51382f73ULL, - 0x45faed5843b0bb28ULL, 0x1c813d5c11bf1f83ULL, 0x8af0e4b6d75fa169ULL, - 0x33ee18a487ad9999ULL, 0x3c26e8eab1c94410ULL, 0xb510102bc0a822f9ULL, - 0x141eef310ce6123bULL, 0xfc65b90059ddb154ULL, 0xe0158640c5e0e607ULL, - 0x884e079826c3a3cfULL, 0x930d0d9523c535fdULL, 0x35638d754e9a2b00ULL, - 0x4085fccf40469dd5ULL, 0xc4b17ad28be23a4cULL, 0xcab2f0fc6a3e6a2eULL, - 0x2860971a6b943fcdULL, 0x3dde6ee212e30446ULL, 0x6222f32ae01765aeULL, - 0x5d550bb5478308feULL, 0xa9efa98da0eda22aULL, 0xc351a71686c40da7ULL, - 0x1105586d9c867c84ULL, 0xdcffee85fda22853ULL, 0xccfbd0262c5eef76ULL, - 0xbaf294cb8990d201ULL, 0xe69464f52afad975ULL, 0x94b013afdf133e14ULL, - 0x06a7d1a32823c958ULL, 0x6f95fe5130f61119ULL, 0xd92ab34e462c06c0ULL, - 0xed7bde33887c71d2ULL, 0x79746d6e6518393eULL, 0x5ba419385d713329ULL, - 0x7c1ba6b948a97564ULL, 0x31987c197bfdac67ULL, 0xde6c23c44b053d02ULL, - 0x581c49fed002d64dULL, 0xdd474d6338261571ULL, 0xaa4546c3e473d062ULL, - 0x928fce349455f860ULL, 0x48161bbacaab94d9ULL, 0x63912430770e6f68ULL, - 0x6ec8a5e602c6641cULL, 0x87282515337ddd2bULL, 0x2cda6b42034b701bULL, - 0xb03d37c181cb096dULL, 0xe108438266c71c6fULL, 0x2b3180c7eb51b255ULL, - 0xdf92b82f96c08bbcULL, 0x5c68c8c0a632f3baULL, 0x5504cc861c3d0556ULL, - 0xabbfa4e55fb26b8fULL, 0x41848b0ab3baceb4ULL, 0xb334a273aa445d32ULL, - 0xbca696f0a85ad881ULL, 0x24f6ec65b528d56cULL, 0x0ce1512e90f4524aULL, - 0x4e9dd79d5506d35aULL, 0x258905fac6ce9779ULL, 0x2019295b3e109b33ULL, - 0xf8a9478b73a054ccULL, 0x2924f2f934417eb0ULL, 0x3993357d536d1bc4ULL, - 0x38a81ac21db6ff8bULL, 0x47c4fbf17d6016bfULL, 0x1e0faadd7667e3f5ULL, - 0x7abcff62938beb96ULL, 0xa78dad948fc179c9ULL, 0x8f1f98b72911e50dULL, - 0x61e48eae27121a91ULL, 0x4d62f7ad31859808ULL, 0xeceba345ef5ceaebULL, - 0xf5ceb25ebc9684ceULL, 0xf633e20cb7f76221ULL, 0xa32cdf06ab8293e4ULL, - 0x985a202ca5ee2ca4ULL, 0xcf0b8447cc8a8fb1ULL, 0x9f765244979859a3ULL, - 0xa8d516b1a1240017ULL, 0x0bd7ba3ebb5dc726ULL, 0xe54bca55b86adb39ULL, - 0x1d7a3afd6c478063ULL, 0x519ec608e7669eddULL, 0x0e5715a2d149aa23ULL, - 0x177d4571848ff194ULL, 0xeeb55f3241014c22ULL, 0x0f5e5ca13a6e2ec2ULL, - 0x8029927b75f5c361ULL, 0xad139fabc3d6e436ULL, 0x0d5df1a94ccf402fULL, - 0x3e8bd948bea5dfc8ULL, 0xa5a0d357bd3ff77eULL, 0xa2d12e251f74f645ULL, - 0x66fd9e525e81a082ULL, 0x2e0c90ce7f687a49ULL, 0xc2e8bcbeba973bc5ULL, - 0x000001bce509745fULL, 0x423777bbe6dab3d6ULL, 0xd1661c7eaef06eb5ULL, - 0xa1781f354daacfd8ULL, 0x2d11284a2b16affcULL, 0xf1fc4f67fa891d1fULL, - 0x73ecc25dcb920adaULL, 0xae610c22c2a12651ULL, 0x96e0a810d356b78aULL, - 0x5a9a381f2fe7870fULL, 0xd5ad62ede94e5530ULL, 0xd225e5e8368d1427ULL, - 0x65977b70c7af4631ULL, 0x99f889b2de39d74fULL, 0x233f30bf54e1d143ULL, - 0x9a9675d3d9a63c97ULL, 0x5470554ff334f9a8ULL, 0x166acb744a4f5688ULL, - 0x70c74caab2e4aeadULL, 0xf0d091646f294d12ULL, 0x57b82a89684031d1ULL, - 0xefd95a5a61be0b6bULL, 0x2fbd12e969f2f29aULL, 0x9bd37013feff9fe8ULL, - 0x3f9b0404d6085a06ULL, 0x4940c1f3166cfe15ULL, 0x09542c4dcdf3defbULL, - 0xb4c5218385cd5ce3ULL, 0xc935b7dc4462a641ULL, 0x3417f8a68ed3b63fULL, - 0xb80959295b215b40ULL, 0xf99cdaef3b8c8572ULL, 0x018c0614f8fcb95dULL, - 0x1b14accd1a3acdf3ULL, 0x84d471f200bb732dULL, 0xc1a3110e95e8da16ULL, - 0x430a7220bf1a82b8ULL, 0xb77e090d39df210eULL, 0x5ef4bd9f3cd05e9dULL, - 0x9d4ff6da7e57a444ULL, 0xda1d60e183d4a5f8ULL, 0xb287c38417998e47ULL, - 0xfe3edc121bb31886ULL, 0xc7fe3ccc980ccbefULL, 0xe46fb590189bfd03ULL, - 0x3732fd469a4c57dcULL, 0x7ef700a07cf1ad65ULL, 0x59c64468a31d8859ULL, - 0x762fb0b4d45b61f6ULL, 0x155baed099047718ULL, 0x68755e4c3d50baa6ULL, - 0xe9214e7f22d8b4dfULL, 0x2addbf532eac95f4ULL, 0x32ae3909b4bd0109ULL, - 0x834df537b08e3450ULL, 0xfa209da84220728dULL, 0x9e691d9b9efe23f7ULL, - 0x0446d288c4ae8d7fULL, 0x7b4cc524e169785bULL, 0x21d87f0135ca1385ULL, - 0xcebb400f137b8aa5ULL, 0x272e2b66580796beULL, 0x3612264125c2b0deULL, - 0x057702bdad1efbb2ULL, 0xd4babb8eacf84be9ULL, 0x91583139641bc67bULL, - 0x8bdc2de08036e024ULL, 0x603c8156f49f68edULL, 0xf7d236f7dbef5111ULL, - 0x9727c4598ad21e80ULL, 0xa08a0896670a5fd7ULL, 0xcb4a8f4309eba9cbULL, - 0x81af564b0f7036a1ULL, 0xc0b99aa778199abdULL, 0x959f1ec83fc8e952ULL, - 0x8c505077794a81b9ULL, 0x3acaaf8f056338f0ULL, 0x07b43f50627a6778ULL, - 0x4a44ab49f5eccc77ULL, 0x3bc3d6e4b679ee98ULL, 0x9cc0d4d1cf14108cULL, - 0x4406c00b206bc8a0ULL, 0x82a18854c8d72d89ULL, 0x67e366b35c3c432cULL, - 0xb923dd61102b37f2ULL, 0x56ab2779d884271dULL, 0xbe83e1b0ff1525afULL, - 0xfb7c65d4217e49a9ULL, 0x6bdbe0e76d48e7d4ULL, 0x08df828745d9179eULL, - 0x22ea6a9add53bd34ULL, 0xe36e141c5622200aULL, 0x7f805d1b8cb750eeULL, - 0xafe5c7a59f58e837ULL, 0xe27f996a4fb1c23cULL, 0xd3867dfb0775f0d0ULL, - 0xd0e673de6e88891aULL, 0x123aeb9eafb86c25ULL, 0x30f1d5d5c145b895ULL, - 0xbb434a2dee7269e7ULL, 0x78cb67ecf931fa38ULL, 0xf33b0372323bbf9cULL, - 0x52d66336fb279c74ULL, 0x505f33ac0afb4eaaULL, 0xe8a5cd99a2cce187ULL, - 0x534974801e2d30bbULL, 0x8d2d5711d5876d90ULL, 0x1f1a412891bc038eULL, - 0xd6e2e71d82e56648ULL, 0x74036c3a497732b7ULL, 0x89b67ed96361f5abULL, - 0xffed95d8f1ea02a2ULL, 0xe72b3bd61464d43dULL, 0xa6300f170bdc4820ULL, - 0xebc18760ed78a77aULL -}; - -static const u64 sbox2[256] = { - 0xe6a6be5a05a12138ULL, 0xb5a122a5b4f87c98ULL, 0x563c6089140b6990ULL, - 0x4c46cb2e391f5dd5ULL, 0xd932addbc9b79434ULL, 0x08ea70e42015aff5ULL, - 0xd765a6673e478cf1ULL, 0xc4fb757eab278d99ULL, 0xdf11c6862d6e0692ULL, - 0xddeb84f10d7f3b16ULL, 0x6f2ef604a665ea04ULL, 0x4a8e0f0ff0e0dfb3ULL, - 0xa5edeef83dbcba51ULL, 0xfc4f0a2a0ea4371eULL, 0xe83e1da85cb38429ULL, - 0xdc8ff882ba1b1ce2ULL, 0xcd45505e8353e80dULL, 0x18d19a00d4db0717ULL, - 0x34a0cfeda5f38101ULL, 0x0be77e518887caf2ULL, 0x1e341438b3c45136ULL, - 0xe05797f49089ccf9ULL, 0xffd23f9df2591d14ULL, 0x543dda228595c5cdULL, - 0x661f81fd99052a33ULL, 0x8736e641db0f7b76ULL, 0x15227725418e5307ULL, - 0xe25f7f46162eb2faULL, 0x48a8b2126c13d9feULL, 0xafdc541792e76eeaULL, - 0x03d912bfc6d1898fULL, 0x31b1aafa1b83f51bULL, 0xf1ac2796e42ab7d9ULL, - 0x40a3a7d7fcd2ebacULL, 0x1056136d0afbbcc5ULL, 0x7889e1dd9a6d0c85ULL, - 0xd33525782a7974aaULL, 0xa7e25d09078ac09bULL, 0xbd4138b3eac6edd0ULL, - 0x920abfbe71eb9e70ULL, 0xa2a5d0f54fc2625cULL, 0xc054e36b0b1290a3ULL, - 0xf6dd59ff62fe932bULL, 0x3537354511a8ac7dULL, 0xca845e9172fadcd4ULL, - 0x84f82b60329d20dcULL, 0x79c62ce1cd672f18ULL, 0x8b09a2add124642cULL, - 0xd0c1e96a19d9e726ULL, 0x5a786a9b4ba9500cULL, 0x0e020336634c43f3ULL, - 0xc17b474aeb66d822ULL, 0x6a731ae3ec9baac2ULL, 0x8226667ae0840258ULL, - 0x67d4567691caeca5ULL, 0x1d94155c4875adb5ULL, 0x6d00fd985b813fdfULL, - 0x51286efcb774cd06ULL, 0x5e8834471fa744afULL, 0xf72ca0aee761ae2eULL, - 0xbe40e4cdaee8e09aULL, 0xe9970bbb5118f665ULL, 0x726e4beb33df1964ULL, - 0x703b000729199762ULL, 0x4631d816f5ef30a7ULL, 0xb880b5b51504a6beULL, - 0x641793c37ed84b6cULL, 0x7b21ed77f6e97d96ULL, 0x776306312ef96b73ULL, - 0xae528948e86ff3f4ULL, 0x53dbd7f286a3f8f8ULL, 0x16cadce74cfc1063ULL, - 0x005c19bdfa52c6ddULL, 0x68868f5d64d46ad3ULL, 0x3a9d512ccf1e186aULL, - 0x367e62c2385660aeULL, 0xe359e7ea77dcb1d7ULL, 0x526c0773749abe6eULL, - 0x735ae5f9d09f734bULL, 0x493fc7cc8a558ba8ULL, 0xb0b9c1533041ab45ULL, - 0x321958ba470a59bdULL, 0x852db00b5f46c393ULL, 0x91209b2bd336b0e5ULL, - 0x6e604f7d659ef19fULL, 0xb99a8ae2782ccb24ULL, 0xccf52ab6c814c4c7ULL, - 0x4727d9afbe11727bULL, 0x7e950d0c0121b34dULL, 0x756f435670ad471fULL, - 0xf5add442615a6849ULL, 0x4e87e09980b9957aULL, 0x2acfa1df50aee355ULL, - 0xd898263afd2fd556ULL, 0xc8f4924dd80c8fd6ULL, 0xcf99ca3d754a173aULL, - 0xfe477bacaf91bf3cULL, 0xed5371f6d690c12dULL, 0x831a5c285e687094ULL, - 0xc5d3c90a3708a0a4ULL, 0x0f7f903717d06580ULL, 0x19f9bb13b8fdf27fULL, - 0xb1bd6f1b4d502843ULL, 0x1c761ba38fff4012ULL, 0x0d1530c4e2e21f3bULL, - 0x8943ce69a7372c8aULL, 0xe5184e11feb5ce66ULL, 0x618bdb80bd736621ULL, - 0x7d29bad68b574d0bULL, 0x81bb613e25e6fe5bULL, 0x071c9c10bc07913fULL, - 0xc7beeb7909ac2d97ULL, 0xc3e58d353bc5d757ULL, 0xeb017892f38f61e8ULL, - 0xd4effb9c9b1cc21aULL, 0x99727d26f494f7abULL, 0xa3e063a2956b3e03ULL, - 0x9d4a8b9a4aa09c30ULL, 0x3f6ab7d500090fb4ULL, 0x9cc0f2a057268ac0ULL, - 0x3dee9d2dedbf42d1ULL, 0x330f49c87960a972ULL, 0xc6b2720287421b41ULL, - 0x0ac59ec07c00369cULL, 0xef4eac49cb353425ULL, 0xf450244eef0129d8ULL, - 0x8acc46e5caf4deb6ULL, 0x2ffeab63989263f7ULL, 0x8f7cb9fe5d7a4578ULL, - 0x5bd8f7644e634635ULL, 0x427a7315bf2dc900ULL, 0x17d0c4aa2125261cULL, - 0x3992486c93518e50ULL, 0xb4cbfee0a2d7d4c3ULL, 0x7c75d6202c5ddd8dULL, - 0xdbc295d8e35b6c61ULL, 0x60b369d302032b19ULL, 0xce42685fdce44132ULL, - 0x06f3ddb9ddf65610ULL, 0x8ea4d21db5e148f0ULL, 0x20b0fce62fcd496fULL, - 0x2c1b912358b0ee31ULL, 0xb28317b818f5a308ULL, 0xa89c1e189ca6d2cfULL, - 0x0c6b18576aaadbc8ULL, 0xb65deaa91299fae3ULL, 0xfb2b794b7f1027e7ULL, - 0x04e4317f443b5bebULL, 0x4b852d325939d0a6ULL, 0xd5ae6beefb207ffcULL, - 0x309682b281c7d374ULL, 0xbae309a194c3b475ULL, 0x8cc3f97b13b49f05ULL, - 0x98a9422ff8293967ULL, 0x244b16b01076ff7cULL, 0xf8bf571c663d67eeULL, - 0x1f0d6758eee30da1ULL, 0xc9b611d97adeb9b7ULL, 0xb7afd5887b6c57a2ULL, - 0x6290ae846b984fe1ULL, 0x94df4cdeacc1a5fdULL, 0x058a5bd1c5483affULL, - 0x63166cc142ba3c37ULL, 0x8db8526eb2f76f40ULL, 0xe10880036f0d6d4eULL, - 0x9e0523c9971d311dULL, 0x45ec2824cc7cd691ULL, 0x575b8359e62382c9ULL, - 0xfa9e400dc4889995ULL, 0xd1823ecb45721568ULL, 0xdafd983b8206082fULL, - 0xaa7d29082386a8cbULL, 0x269fcd4403b87588ULL, 0x1b91f5f728bdd1e0ULL, - 0xe4669f39040201f6ULL, 0x7a1d7c218cf04adeULL, 0x65623c29d79ce5ceULL, - 0x2368449096c00bb1ULL, 0xab9bf1879da503baULL, 0xbc23ecb1a458058eULL, - 0x9a58df01bb401eccULL, 0xa070e868a85f143dULL, 0x4ff188307df2239eULL, - 0x14d565b41a641183ULL, 0xee13337452701602ULL, 0x950e3dcf3f285e09ULL, - 0x59930254b9c80953ULL, 0x3bf299408930da6dULL, 0xa955943f53691387ULL, - 0xa15edecaa9cb8784ULL, 0x29142127352be9a0ULL, 0x76f0371fff4e7afbULL, - 0x0239f450274f2228ULL, 0xbb073af01d5e868bULL, 0xbfc80571c10e96c1ULL, - 0xd267088568222e23ULL, 0x9671a3d48e80b5b0ULL, 0x55b5d38ae193bb81ULL, - 0x693ae2d0a18b04b8ULL, 0x5c48b4ecadd5335fULL, 0xfd743b194916a1caULL, - 0x2577018134be98c4ULL, 0xe77987e83c54a4adULL, 0x28e11014da33e1b9ULL, - 0x270cc59e226aa213ULL, 0x71495f756d1a5f60ULL, 0x9be853fb60afef77ULL, - 0xadc786a7f7443dbfULL, 0x0904456173b29a82ULL, 0x58bc7a66c232bd5eULL, - 0xf306558c673ac8b2ULL, 0x41f639c6b6c9772aULL, 0x216defe99fda35daULL, - 0x11640cc71c7be615ULL, 0x93c43694565c5527ULL, 0xea038e6246777839ULL, - 0xf9abf3ce5a3e2469ULL, 0x741e768d0fd312d2ULL, 0x0144b883ced652c6ULL, - 0xc20b5a5ba33f8552ULL, 0x1ae69633c3435a9dULL, 0x97a28ca4088cfdecULL, - 0x8824a43c1e96f420ULL, 0x37612fa66eeea746ULL, 0x6b4cb165f9cf0e5aULL, - 0x43aa1c06a0abfb4aULL, 0x7f4dc26ff162796bULL, 0x6cbacc8e54ed9b0fULL, - 0xa6b7ffefd2bb253eULL, 0x2e25bc95b0a29d4fULL, 0x86d6a58bdef1388cULL, - 0xded74ac576b6f054ULL, 0x8030bdbc2b45805dULL, 0x3c81af70e94d9289ULL, - 0x3eff6dda9e3100dbULL, 0xb38dc39fdfcc8847ULL, 0x123885528d17b87eULL, - 0xf2da0ed240b1b642ULL, 0x44cefadcd54bf9a9ULL, 0x1312200e433c7ee6ULL, - 0x9ffcc84f3a78c748ULL, 0xf0cd1f72248576bbULL, 0xec6974053638cfe4ULL, - 0x2ba7b67c0cec4e4cULL, 0xac2f4df3e5ce32edULL, 0xcb33d14326ea4c11ULL, - 0xa4e9044cc77e58bcULL, 0x5f513293d934fcefULL, 0x5dc9645506e55444ULL, - 0x50de418f317de40aULL, 0x388cb31a69dde259ULL, 0x2db4a83455820a86ULL, - 0x9010a91e84711ae9ULL, 0x4df7f0b7b1498371ULL, 0xd62a2eabc0977179ULL, - 0x22fac097aa8d5c0eULL -}; - -static const u64 sbox3[256] = { - 0xf49fcc2ff1daf39bULL, 0x487fd5c66ff29281ULL, 0xe8a30667fcdca83fULL, - 0x2c9b4be3d2fcce63ULL, 0xda3ff74b93fbbbc2ULL, 0x2fa165d2fe70ba66ULL, - 0xa103e279970e93d4ULL, 0xbecdec77b0e45e71ULL, 0xcfb41e723985e497ULL, - 0xb70aaa025ef75017ULL, 0xd42309f03840b8e0ULL, 0x8efc1ad035898579ULL, - 0x96c6920be2b2abc5ULL, 0x66af4163375a9172ULL, 0x2174abdcca7127fbULL, - 0xb33ccea64a72ff41ULL, 0xf04a4933083066a5ULL, 0x8d970acdd7289af5ULL, - 0x8f96e8e031c8c25eULL, 0xf3fec02276875d47ULL, 0xec7bf310056190ddULL, - 0xf5adb0aebb0f1491ULL, 0x9b50f8850fd58892ULL, 0x4975488358b74de8ULL, - 0xa3354ff691531c61ULL, 0x0702bbe481d2c6eeULL, 0x89fb24057deded98ULL, - 0xac3075138596e902ULL, 0x1d2d3580172772edULL, 0xeb738fc28e6bc30dULL, - 0x5854ef8f63044326ULL, 0x9e5c52325add3bbeULL, 0x90aa53cf325c4623ULL, - 0xc1d24d51349dd067ULL, 0x2051cfeea69ea624ULL, 0x13220f0a862e7e4fULL, - 0xce39399404e04864ULL, 0xd9c42ca47086fcb7ULL, 0x685ad2238a03e7ccULL, - 0x066484b2ab2ff1dbULL, 0xfe9d5d70efbf79ecULL, 0x5b13b9dd9c481854ULL, - 0x15f0d475ed1509adULL, 0x0bebcd060ec79851ULL, 0xd58c6791183ab7f8ULL, - 0xd1187c5052f3eee4ULL, 0xc95d1192e54e82ffULL, 0x86eea14cb9ac6ca2ULL, - 0x3485beb153677d5dULL, 0xdd191d781f8c492aULL, 0xf60866baa784ebf9ULL, - 0x518f643ba2d08c74ULL, 0x8852e956e1087c22ULL, 0xa768cb8dc410ae8dULL, - 0x38047726bfec8e1aULL, 0xa67738b4cd3b45aaULL, 0xad16691cec0dde19ULL, - 0xc6d4319380462e07ULL, 0xc5a5876d0ba61938ULL, 0x16b9fa1fa58fd840ULL, - 0x188ab1173ca74f18ULL, 0xabda2f98c99c021fULL, 0x3e0580ab134ae816ULL, - 0x5f3b05b773645abbULL, 0x2501a2be5575f2f6ULL, 0x1b2f74004e7e8ba9ULL, - 0x1cd7580371e8d953ULL, 0x7f6ed89562764e30ULL, 0xb15926ff596f003dULL, - 0x9f65293da8c5d6b9ULL, 0x6ecef04dd690f84cULL, 0x4782275fff33af88ULL, - 0xe41433083f820801ULL, 0xfd0dfe409a1af9b5ULL, 0x4325a3342cdb396bULL, - 0x8ae77e62b301b252ULL, 0xc36f9e9f6655615aULL, 0x85455a2d92d32c09ULL, - 0xf2c7dea949477485ULL, 0x63cfb4c133a39ebaULL, 0x83b040cc6ebc5462ULL, - 0x3b9454c8fdb326b0ULL, 0x56f56a9e87ffd78cULL, 0x2dc2940d99f42bc6ULL, - 0x98f7df096b096e2dULL, 0x19a6e01e3ad852bfULL, 0x42a99ccbdbd4b40bULL, - 0xa59998af45e9c559ULL, 0x366295e807d93186ULL, 0x6b48181bfaa1f773ULL, - 0x1fec57e2157a0a1dULL, 0x4667446af6201ad5ULL, 0xe615ebcacfb0f075ULL, - 0xb8f31f4f68290778ULL, 0x22713ed6ce22d11eULL, 0x3057c1a72ec3c93bULL, - 0xcb46acc37c3f1f2fULL, 0xdbb893fd02aaf50eULL, 0x331fd92e600b9fcfULL, - 0xa498f96148ea3ad6ULL, 0xa8d8426e8b6a83eaULL, 0xa089b274b7735cdcULL, - 0x87f6b3731e524a11ULL, 0x118808e5cbc96749ULL, 0x9906e4c7b19bd394ULL, - 0xafed7f7e9b24a20cULL, 0x6509eadeeb3644a7ULL, 0x6c1ef1d3e8ef0edeULL, - 0xb9c97d43e9798fb4ULL, 0xa2f2d784740c28a3ULL, 0x7b8496476197566fULL, - 0x7a5be3e6b65f069dULL, 0xf96330ed78be6f10ULL, 0xeee60de77a076a15ULL, - 0x2b4bee4aa08b9bd0ULL, 0x6a56a63ec7b8894eULL, 0x02121359ba34fef4ULL, - 0x4cbf99f8283703fcULL, 0x398071350caf30c8ULL, 0xd0a77a89f017687aULL, - 0xf1c1a9eb9e423569ULL, 0x8c7976282dee8199ULL, 0x5d1737a5dd1f7abdULL, - 0x4f53433c09a9fa80ULL, 0xfa8b0c53df7ca1d9ULL, 0x3fd9dcbc886ccb77ULL, - 0xc040917ca91b4720ULL, 0x7dd00142f9d1dcdfULL, 0x8476fc1d4f387b58ULL, - 0x23f8e7c5f3316503ULL, 0x032a2244e7e37339ULL, 0x5c87a5d750f5a74bULL, - 0x082b4cc43698992eULL, 0xdf917becb858f63cULL, 0x3270b8fc5bf86ddaULL, - 0x10ae72bb29b5dd76ULL, 0x576ac94e7700362bULL, 0x1ad112dac61efb8fULL, - 0x691bc30ec5faa427ULL, 0xff246311cc327143ULL, 0x3142368e30e53206ULL, - 0x71380e31e02ca396ULL, 0x958d5c960aad76f1ULL, 0xf8d6f430c16da536ULL, - 0xc8ffd13f1be7e1d2ULL, 0x7578ae66004ddbe1ULL, 0x05833f01067be646ULL, - 0xbb34b5ad3bfe586dULL, 0x095f34c9a12b97f0ULL, 0x247ab64525d60ca8ULL, - 0xdcdbc6f3017477d1ULL, 0x4a2e14d4decad24dULL, 0xbdb5e6d9be0a1eebULL, - 0x2a7e70f7794301abULL, 0xdef42d8a270540fdULL, 0x01078ec0a34c22c1ULL, - 0xe5de511af4c16387ULL, 0x7ebb3a52bd9a330aULL, 0x77697857aa7d6435ULL, - 0x004e831603ae4c32ULL, 0xe7a21020ad78e312ULL, 0x9d41a70c6ab420f2ULL, - 0x28e06c18ea1141e6ULL, 0xd2b28cbd984f6b28ULL, 0x26b75f6c446e9d83ULL, - 0xba47568c4d418d7fULL, 0xd80badbfe6183d8eULL, 0x0e206d7f5f166044ULL, - 0xe258a43911cbca3eULL, 0x723a1746b21dc0bcULL, 0xc7caa854f5d7cdd3ULL, - 0x7cac32883d261d9cULL, 0x7690c26423ba942cULL, 0x17e55524478042b8ULL, - 0xe0be477656a2389fULL, 0x4d289b5e67ab2da0ULL, 0x44862b9c8fbbfd31ULL, - 0xb47cc8049d141365ULL, 0x822c1b362b91c793ULL, 0x4eb14655fb13dfd8ULL, - 0x1ecbba0714e2a97bULL, 0x6143459d5cde5f14ULL, 0x53a8fbf1d5f0ac89ULL, - 0x97ea04d81c5e5b00ULL, 0x622181a8d4fdb3f3ULL, 0xe9bcd341572a1208ULL, - 0x1411258643cce58aULL, 0x9144c5fea4c6e0a4ULL, 0x0d33d06565cf620fULL, - 0x54a48d489f219ca1ULL, 0xc43e5eac6d63c821ULL, 0xa9728b3a72770dafULL, - 0xd7934e7b20df87efULL, 0xe35503b61a3e86e5ULL, 0xcae321fbc819d504ULL, - 0x129a50b3ac60bfa6ULL, 0xcd5e68ea7e9fb6c3ULL, 0xb01c90199483b1c7ULL, - 0x3de93cd5c295376cULL, 0xaed52edf2ab9ad13ULL, 0x2e60f512c0a07884ULL, - 0xbc3d86a3e36210c9ULL, 0x35269d9b163951ceULL, 0x0c7d6e2ad0cdb5faULL, - 0x59e86297d87f5733ULL, 0x298ef221898db0e7ULL, 0x55000029d1a5aa7eULL, - 0x8bc08ae1b5061b45ULL, 0xc2c31c2b6c92703aULL, 0x94cc596baf25ef42ULL, - 0x0a1d73db22540456ULL, 0x04b6a0f9d9c4179aULL, 0xeffdafa2ae3d3c60ULL, - 0xf7c8075bb49496c4ULL, 0x9cc5c7141d1cd4e3ULL, 0x78bd1638218e5534ULL, - 0xb2f11568f850246aULL, 0xedfabcfa9502bc29ULL, 0x796ce5f2da23051bULL, - 0xaae128b0dc93537cULL, 0x3a493da0ee4b29aeULL, 0xb5df6b2c416895d7ULL, - 0xfcabbd25122d7f37ULL, 0x70810b58105dc4b1ULL, 0xe10fdd37f7882a90ULL, - 0x524dcab5518a3f5cULL, 0x3c9e85878451255bULL, 0x4029828119bd34e2ULL, - 0x74a05b6f5d3ceccbULL, 0xb610021542e13ecaULL, 0x0ff979d12f59e2acULL, - 0x6037da27e4f9cc50ULL, 0x5e92975a0df1847dULL, 0xd66de190d3e623feULL, - 0x5032d6b87b568048ULL, 0x9a36b7ce8235216eULL, 0x80272a7a24f64b4aULL, - 0x93efed8b8c6916f7ULL, 0x37ddbff44cce1555ULL, 0x4b95db5d4b99bd25ULL, - 0x92d3fda169812fc0ULL, 0xfb1a4a9a90660bb6ULL, 0x730c196946a4b9b2ULL, - 0x81e289aa7f49da68ULL, 0x64669a0f83b1a05fULL, 0x27b3ff7d9644f48bULL, - 0xcc6b615c8db675b3ULL, 0x674f20b9bcebbe95ULL, 0x6f31238275655982ULL, - 0x5ae488713e45cf05ULL, 0xbf619f9954c21157ULL, 0xeabac46040a8eae9ULL, - 0x454c6fe9f2c0c1cdULL, 0x419cf6496412691cULL, 0xd3dc3bef265b0f70ULL, - 0x6d0e60f5c3578a9eULL -}; - -static const u64 sbox4[256] = { - 0x5b0e608526323c55ULL, 0x1a46c1a9fa1b59f5ULL, 0xa9e245a17c4c8ffaULL, - 0x65ca5159db2955d7ULL, 0x05db0a76ce35afc2ULL, 0x81eac77ea9113d45ULL, - 0x528ef88ab6ac0a0dULL, 0xa09ea253597be3ffULL, 0x430ddfb3ac48cd56ULL, - 0xc4b3a67af45ce46fULL, 0x4ececfd8fbe2d05eULL, 0x3ef56f10b39935f0ULL, - 0x0b22d6829cd619c6ULL, 0x17fd460a74df2069ULL, 0x6cf8cc8e8510ed40ULL, - 0xd6c824bf3a6ecaa7ULL, 0x61243d581a817049ULL, 0x048bacb6bbc163a2ULL, - 0xd9a38ac27d44cc32ULL, 0x7fddff5baaf410abULL, 0xad6d495aa804824bULL, - 0xe1a6a74f2d8c9f94ULL, 0xd4f7851235dee8e3ULL, 0xfd4b7f886540d893ULL, - 0x247c20042aa4bfdaULL, 0x096ea1c517d1327cULL, 0xd56966b4361a6685ULL, - 0x277da5c31221057dULL, 0x94d59893a43acff7ULL, 0x64f0c51ccdc02281ULL, - 0x3d33bcc4ff6189dbULL, 0xe005cb184ce66af1ULL, 0xff5ccd1d1db99beaULL, - 0xb0b854a7fe42980fULL, 0x7bd46a6a718d4b9fULL, 0xd10fa8cc22a5fd8cULL, - 0xd31484952be4bd31ULL, 0xc7fa975fcb243847ULL, 0x4886ed1e5846c407ULL, - 0x28cddb791eb70b04ULL, 0xc2b00be2f573417fULL, 0x5c9590452180f877ULL, - 0x7a6bddfff370eb00ULL, 0xce509e38d6d9d6a4ULL, 0xebeb0f00647fa702ULL, - 0x1dcc06cf76606f06ULL, 0xe4d9f28ba286ff0aULL, 0xd85a305dc918c262ULL, - 0x475b1d8732225f54ULL, 0x2d4fb51668ccb5feULL, 0xa679b9d9d72bba20ULL, - 0x53841c0d912d43a5ULL, 0x3b7eaa48bf12a4e8ULL, 0x781e0e47f22f1ddfULL, - 0xeff20ce60ab50973ULL, 0x20d261d19dffb742ULL, 0x16a12b03062a2e39ULL, - 0x1960eb2239650495ULL, 0x251c16fed50eb8b8ULL, 0x9ac0c330f826016eULL, - 0xed152665953e7671ULL, 0x02d63194a6369570ULL, 0x5074f08394b1c987ULL, - 0x70ba598c90b25ce1ULL, 0x794a15810b9742f6ULL, 0x0d5925e9fcaf8c6cULL, - 0x3067716cd868744eULL, 0x910ab077e8d7731bULL, 0x6a61bbdb5ac42f61ULL, - 0x93513efbf0851567ULL, 0xf494724b9e83e9d5ULL, 0xe887e1985c09648dULL, - 0x34b1d3c675370cfdULL, 0xdc35e433bc0d255dULL, 0xd0aab84234131be0ULL, - 0x08042a50b48b7eafULL, 0x9997c4ee44a3ab35ULL, 0x829a7b49201799d0ULL, - 0x263b8307b7c54441ULL, 0x752f95f4fd6a6ca6ULL, 0x927217402c08c6e5ULL, - 0x2a8ab754a795d9eeULL, 0xa442f7552f72943dULL, 0x2c31334e19781208ULL, - 0x4fa98d7ceaee6291ULL, 0x55c3862f665db309ULL, 0xbd0610175d53b1f3ULL, - 0x46fe6cb840413f27ULL, 0x3fe03792df0cfa59ULL, 0xcfe700372eb85e8fULL, - 0xa7be29e7adbce118ULL, 0xe544ee5cde8431ddULL, 0x8a781b1b41f1873eULL, - 0xa5c94c78a0d2f0e7ULL, 0x39412e2877b60728ULL, 0xa1265ef3afc9a62cULL, - 0xbcc2770c6a2506c5ULL, 0x3ab66dd5dce1ce12ULL, 0xe65499d04a675b37ULL, - 0x7d8f523481bfd216ULL, 0x0f6f64fcec15f389ULL, 0x74efbe618b5b13c8ULL, - 0xacdc82b714273e1dULL, 0xdd40bfe003199d17ULL, 0x37e99257e7e061f8ULL, - 0xfa52626904775aaaULL, 0x8bbbf63a463d56f9ULL, 0xf0013f1543a26e64ULL, - 0xa8307e9f879ec898ULL, 0xcc4c27a4150177ccULL, 0x1b432f2cca1d3348ULL, - 0xde1d1f8f9f6fa013ULL, 0x606602a047a7ddd6ULL, 0xd237ab64cc1cb2c7ULL, - 0x9b938e7225fcd1d3ULL, 0xec4e03708e0ff476ULL, 0xfeb2fbda3d03c12dULL, - 0xae0bced2ee43889aULL, 0x22cb8923ebfb4f43ULL, 0x69360d013cf7396dULL, - 0x855e3602d2d4e022ULL, 0x073805bad01f784cULL, 0x33e17a133852f546ULL, - 0xdf4874058ac7b638ULL, 0xba92b29c678aa14aULL, 0x0ce89fc76cfaadcdULL, - 0x5f9d4e0908339e34ULL, 0xf1afe9291f5923b9ULL, 0x6e3480f60f4a265fULL, - 0xeebf3a2ab29b841cULL, 0xe21938a88f91b4adULL, 0x57dfeff845c6d3c3ULL, - 0x2f006b0bf62caaf2ULL, 0x62f479ef6f75ee78ULL, 0x11a55ad41c8916a9ULL, - 0xf229d29084fed453ULL, 0x42f1c27b16b000e6ULL, 0x2b1f76749823c074ULL, - 0x4b76eca3c2745360ULL, 0x8c98f463b91691bdULL, 0x14bcc93cf1ade66aULL, - 0x8885213e6d458397ULL, 0x8e177df0274d4711ULL, 0xb49b73b5503f2951ULL, - 0x10168168c3f96b6bULL, 0x0e3d963b63cab0aeULL, 0x8dfc4b5655a1db14ULL, - 0xf789f1356e14de5cULL, 0x683e68af4e51dac1ULL, 0xc9a84f9d8d4b0fd9ULL, - 0x3691e03f52a0f9d1ULL, 0x5ed86e46e1878e80ULL, 0x3c711a0e99d07150ULL, - 0x5a0865b20c4e9310ULL, 0x56fbfc1fe4f0682eULL, 0xea8d5de3105edf9bULL, - 0x71abfdb12379187aULL, 0x2eb99de1bee77b9cULL, 0x21ecc0ea33cf4523ULL, - 0x59a4d7521805c7a1ULL, 0x3896f5eb56ae7c72ULL, 0xaa638f3db18f75dcULL, - 0x9f39358dabe9808eULL, 0xb7defa91c00b72acULL, 0x6b5541fd62492d92ULL, - 0x6dc6dee8f92e4d5bULL, 0x353f57abc4beea7eULL, 0x735769d6da5690ceULL, - 0x0a234aa642391484ULL, 0xf6f9508028f80d9dULL, 0xb8e319a27ab3f215ULL, - 0x31ad9c1151341a4dULL, 0x773c22a57bef5805ULL, 0x45c7561a07968633ULL, - 0xf913da9e249dbe36ULL, 0xda652d9b78a64c68ULL, 0x4c27a97f3bc334efULL, - 0x76621220e66b17f4ULL, 0x967743899acd7d0bULL, 0xf3ee5bcae0ed6782ULL, - 0x409f753600c879fcULL, 0x06d09a39b5926db6ULL, 0x6f83aeb0317ac588ULL, - 0x01e6ca4a86381f21ULL, 0x66ff3462d19f3025ULL, 0x72207c24ddfd3bfbULL, - 0x4af6b6d3e2ece2ebULL, 0x9c994dbec7ea08deULL, 0x49ace597b09a8bc4ULL, - 0xb38c4766cf0797baULL, 0x131b9373c57c2a75ULL, 0xb1822cce61931e58ULL, - 0x9d7555b909ba1c0cULL, 0x127fafdd937d11d2ULL, 0x29da3badc66d92e4ULL, - 0xa2c1d57154c2ecbcULL, 0x58c5134d82f6fe24ULL, 0x1c3ae3515b62274fULL, - 0xe907c82e01cb8126ULL, 0xf8ed091913e37fcbULL, 0x3249d8f9c80046c9ULL, - 0x80cf9bede388fb63ULL, 0x1881539a116cf19eULL, 0x5103f3f76bd52457ULL, - 0x15b7e6f5ae47f7a8ULL, 0xdbd7c6ded47e9ccfULL, 0x44e55c410228bb1aULL, - 0xb647d4255edb4e99ULL, 0x5d11882bb8aafc30ULL, 0xf5098bbb29d3212aULL, - 0x8fb5ea14e90296b3ULL, 0x677b942157dd025aULL, 0xfb58e7c0a390acb5ULL, - 0x89d3674c83bd4a01ULL, 0x9e2da4df4bf3b93bULL, 0xfcc41e328cab4829ULL, - 0x03f38c96ba582c52ULL, 0xcad1bdbd7fd85db2ULL, 0xbbb442c16082ae83ULL, - 0xb95fe86ba5da9ab0ULL, 0xb22e04673771a93fULL, 0x845358c9493152d8ULL, - 0xbe2a488697b4541eULL, 0x95a2dc2dd38e6966ULL, 0xc02c11ac923c852bULL, - 0x2388b1990df2a87bULL, 0x7c8008fa1b4f37beULL, 0x1f70d0c84d54e503ULL, - 0x5490adec7ece57d4ULL, 0x002b3c27d9063a3aULL, 0x7eaea3848030a2bfULL, - 0xc602326ded2003c0ULL, 0x83a7287d69a94086ULL, 0xc57a5fcb30f57a8aULL, - 0xb56844e479ebe779ULL, 0xa373b40f05dcbce9ULL, 0xd71a786e88570ee2ULL, - 0x879cbacdbde8f6a0ULL, 0x976ad1bcc164a32fULL, 0xab21e25e9666d78bULL, - 0x901063aae5e5c33cULL, 0x9818b34448698d90ULL, 0xe36487ae3e1e8abbULL, - 0xafbdf931893bdcb4ULL, 0x6345a0dc5fbbd519ULL, 0x8628fe269b9465caULL, - 0x1e5d01603f9c51ecULL, 0x4de44006a15049b7ULL, 0xbf6c70e5f776cbb1ULL, - 0x411218f2ef552bedULL, 0xcb0c0708705a36a3ULL, 0xe74d14754f986044ULL, - 0xcd56d9430ea8280eULL, 0xc12591d7535f5065ULL, 0xc83223f1720aef96ULL, - 0xc3a0396f7363a51fULL -}; - - -static void tgr192_round(u64 * ra, u64 * rb, u64 * rc, u64 x, int mul) -{ - u64 a = *ra; - u64 b = *rb; - u64 c = *rc; - - c ^= x; - a -= sbox1[c & 0xff] ^ sbox2[(c >> 16) & 0xff] - ^ sbox3[(c >> 32) & 0xff] ^ sbox4[(c >> 48) & 0xff]; - b += sbox4[(c >> 8) & 0xff] ^ sbox3[(c >> 24) & 0xff] - ^ sbox2[(c >> 40) & 0xff] ^ sbox1[(c >> 56) & 0xff]; - b *= mul; - - *ra = a; - *rb = b; - *rc = c; -} - - -static void tgr192_pass(u64 * ra, u64 * rb, u64 * rc, u64 * x, int mul) -{ - u64 a = *ra; - u64 b = *rb; - u64 c = *rc; - - tgr192_round(&a, &b, &c, x[0], mul); - tgr192_round(&b, &c, &a, x[1], mul); - tgr192_round(&c, &a, &b, x[2], mul); - tgr192_round(&a, &b, &c, x[3], mul); - tgr192_round(&b, &c, &a, x[4], mul); - tgr192_round(&c, &a, &b, x[5], mul); - tgr192_round(&a, &b, &c, x[6], mul); - tgr192_round(&b, &c, &a, x[7], mul); - - *ra = a; - *rb = b; - *rc = c; -} - - -static void tgr192_key_schedule(u64 * x) -{ - x[0] -= x[7] ^ 0xa5a5a5a5a5a5a5a5ULL; - x[1] ^= x[0]; - x[2] += x[1]; - x[3] -= x[2] ^ ((~x[1]) << 19); - x[4] ^= x[3]; - x[5] += x[4]; - x[6] -= x[5] ^ ((~x[4]) >> 23); - x[7] ^= x[6]; - x[0] += x[7]; - x[1] -= x[0] ^ ((~x[7]) << 19); - x[2] ^= x[1]; - x[3] += x[2]; - x[4] -= x[3] ^ ((~x[2]) >> 23); - x[5] ^= x[4]; - x[6] += x[5]; - x[7] -= x[6] ^ 0x0123456789abcdefULL; -} - - -/**************** - * Transform the message DATA which consists of 512 bytes (8 words) - */ - -static void tgr192_transform(struct tgr192_ctx *tctx, const u8 * data) -{ - u64 a, b, c, aa, bb, cc; - u64 x[8]; - int i; - - for (i = 0; i < 8; i++) - x[i] = get_unaligned_le64(data + i * sizeof(__le64)); - - /* save */ - a = aa = tctx->a; - b = bb = tctx->b; - c = cc = tctx->c; - - tgr192_pass(&a, &b, &c, x, 5); - tgr192_key_schedule(x); - tgr192_pass(&c, &a, &b, x, 7); - tgr192_key_schedule(x); - tgr192_pass(&b, &c, &a, x, 9); - - - /* feedforward */ - a ^= aa; - b -= bb; - c += cc; - /* store */ - tctx->a = a; - tctx->b = b; - tctx->c = c; -} - -static int tgr192_init(struct shash_desc *desc) -{ - struct tgr192_ctx *tctx = shash_desc_ctx(desc); - - tctx->a = 0x0123456789abcdefULL; - tctx->b = 0xfedcba9876543210ULL; - tctx->c = 0xf096a5b4c3b2e187ULL; - tctx->nblocks = 0; - tctx->count = 0; - - return 0; -} - - -/* Update the message digest with the contents - * of INBUF with length INLEN. */ -static int tgr192_update(struct shash_desc *desc, const u8 *inbuf, - unsigned int len) -{ - struct tgr192_ctx *tctx = shash_desc_ctx(desc); - - if (tctx->count == 64) { /* flush the buffer */ - tgr192_transform(tctx, tctx->hash); - tctx->count = 0; - tctx->nblocks++; - } - if (!inbuf) { - return 0; - } - if (tctx->count) { - for (; len && tctx->count < 64; len--) { - tctx->hash[tctx->count++] = *inbuf++; - } - tgr192_update(desc, NULL, 0); - if (!len) { - return 0; - } - - } - - while (len >= 64) { - tgr192_transform(tctx, inbuf); - tctx->count = 0; - tctx->nblocks++; - len -= 64; - inbuf += 64; - } - for (; len && tctx->count < 64; len--) { - tctx->hash[tctx->count++] = *inbuf++; - } - - return 0; -} - - - -/* The routine terminates the computation */ -static int tgr192_final(struct shash_desc *desc, u8 * out) -{ - struct tgr192_ctx *tctx = shash_desc_ctx(desc); - __be64 *dst = (__be64 *)out; - __be64 *be64p; - __le32 *le32p; - u32 t, msb, lsb; - - tgr192_update(desc, NULL, 0); /* flush */ - - msb = 0; - t = tctx->nblocks; - if ((lsb = t << 6) < t) { /* multiply by 64 to make a byte count */ - msb++; - } - msb += t >> 26; - t = lsb; - if ((lsb = t + tctx->count) < t) { /* add the count */ - msb++; - } - t = lsb; - if ((lsb = t << 3) < t) { /* multiply by 8 to make a bit count */ - msb++; - } - msb += t >> 29; - - if (tctx->count < 56) { /* enough room */ - tctx->hash[tctx->count++] = 0x01; /* pad */ - while (tctx->count < 56) { - tctx->hash[tctx->count++] = 0; /* pad */ - } - } else { /* need one extra block */ - tctx->hash[tctx->count++] = 0x01; /* pad character */ - while (tctx->count < 64) { - tctx->hash[tctx->count++] = 0; - } - tgr192_update(desc, NULL, 0); /* flush */ - memset(tctx->hash, 0, 56); /* fill next block with zeroes */ - } - /* append the 64 bit count */ - le32p = (__le32 *)&tctx->hash[56]; - le32p[0] = cpu_to_le32(lsb); - le32p[1] = cpu_to_le32(msb); - - tgr192_transform(tctx, tctx->hash); - - be64p = (__be64 *)tctx->hash; - dst[0] = be64p[0] = cpu_to_be64(tctx->a); - dst[1] = be64p[1] = cpu_to_be64(tctx->b); - dst[2] = be64p[2] = cpu_to_be64(tctx->c); - - return 0; -} - -static int tgr160_final(struct shash_desc *desc, u8 * out) -{ - u8 D[64]; - - tgr192_final(desc, D); - memcpy(out, D, TGR160_DIGEST_SIZE); - memzero_explicit(D, TGR192_DIGEST_SIZE); - - return 0; -} - -static int tgr128_final(struct shash_desc *desc, u8 * out) -{ - u8 D[64]; - - tgr192_final(desc, D); - memcpy(out, D, TGR128_DIGEST_SIZE); - memzero_explicit(D, TGR192_DIGEST_SIZE); - - return 0; -} - -static struct shash_alg tgr_algs[3] = { { - .digestsize = TGR192_DIGEST_SIZE, - .init = tgr192_init, - .update = tgr192_update, - .final = tgr192_final, - .descsize = sizeof(struct tgr192_ctx), - .base = { - .cra_name = "tgr192", - .cra_driver_name = "tgr192-generic", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = TGR160_DIGEST_SIZE, - .init = tgr192_init, - .update = tgr192_update, - .final = tgr160_final, - .descsize = sizeof(struct tgr192_ctx), - .base = { - .cra_name = "tgr160", - .cra_driver_name = "tgr160-generic", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = TGR128_DIGEST_SIZE, - .init = tgr192_init, - .update = tgr192_update, - .final = tgr128_final, - .descsize = sizeof(struct tgr192_ctx), - .base = { - .cra_name = "tgr128", - .cra_driver_name = "tgr128-generic", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init tgr192_mod_init(void) -{ - return crypto_register_shashes(tgr_algs, ARRAY_SIZE(tgr_algs)); -} - -static void __exit tgr192_mod_fini(void) -{ - crypto_unregister_shashes(tgr_algs, ARRAY_SIZE(tgr_algs)); -} - -MODULE_ALIAS_CRYPTO("tgr192"); -MODULE_ALIAS_CRYPTO("tgr160"); -MODULE_ALIAS_CRYPTO("tgr128"); - -subsys_initcall(tgr192_mod_init); -module_exit(tgr192_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Tiger Message Digest Algorithm"); diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c index 4f7c033224f9..86b2f067a416 100644 --- a/crypto/twofish_generic.c +++ b/crypto/twofish_generic.c @@ -24,7 +24,7 @@ * Third Edition. */ -#include +#include #include #include #include @@ -83,11 +83,11 @@ * whitening subkey number m. */ #define INPACK(n, x, m) \ - x = le32_to_cpu(src[n]) ^ ctx->w[m] + x = get_unaligned_le32(in + (n) * 4) ^ ctx->w[m] #define OUTUNPACK(n, x, m) \ x ^= ctx->w[m]; \ - dst[n] = cpu_to_le32(x) + put_unaligned_le32(x, out + (n) * 4) @@ -95,8 +95,6 @@ static void twofish_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct twofish_ctx *ctx = crypto_tfm_ctx(tfm); - const __le32 *src = (const __le32 *)in; - __le32 *dst = (__le32 *)out; /* The four 32-bit chunks of the text. */ u32 a, b, c, d; @@ -132,8 +130,6 @@ static void twofish_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) static void twofish_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct twofish_ctx *ctx = crypto_tfm_ctx(tfm); - const __le32 *src = (const __le32 *)in; - __le32 *dst = (__le32 *)out; /* The four 32-bit chunks of the text. */ u32 a, b, c, d; @@ -172,7 +168,6 @@ static struct crypto_alg alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = TF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct twofish_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { .cia_min_keysize = TF_MIN_KEY_SIZE, diff --git a/crypto/vmac.c b/crypto/vmac.c index 9b565d1040d6..4633b2dda1e0 100644 --- a/crypto/vmac.c +++ b/crypto/vmac.c @@ -36,6 +36,7 @@ #include #include #include +#include #include /* @@ -693,3 +694,4 @@ module_exit(vmac_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("VMAC hash algorithm"); MODULE_ALIAS_CRYPTO("vmac64"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/xcbc.c b/crypto/xcbc.c index af3b7eb5d7c7..6074c5c1da49 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -6,6 +6,7 @@ * Kazunori Miyazawa */ +#include #include #include #include @@ -272,3 +273,4 @@ module_exit(crypto_xcbc_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("XCBC keyed hash algorithm"); MODULE_ALIAS_CRYPTO("xcbc"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/xor.c b/crypto/xor.c index 8f899f898ec9..8e72e5d5db0d 100644 --- a/crypto/xor.c +++ b/crypto/xor.c @@ -95,7 +95,7 @@ do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2) for (i = 0; i < 3; i++) { start = ktime_get(); for (j = 0; j < REPS; j++) { - mb(); /* prevent loop optimzation */ + mb(); /* prevent loop optimization */ tmpl->do_2(BENCH_SIZE, b1, b2); mb(); } diff --git a/crypto/xts.c b/crypto/xts.c index ad45b009774b..6c12f30dbdd6 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -7,6 +7,7 @@ * Based on ecb.c * Copyright (c) 2006 Herbert Xu */ +#include #include #include #include @@ -464,3 +465,4 @@ module_exit(xts_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("XTS block cipher mode"); MODULE_ALIAS_CRYPTO("xts"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/Kconfig b/drivers/Kconfig index dcecc9f6e33f..62c753a73651 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -6,6 +6,7 @@ menu "Device Drivers" source "drivers/amba/Kconfig" source "drivers/eisa/Kconfig" source "drivers/pci/Kconfig" +source "drivers/cxl/Kconfig" source "drivers/pcmcia/Kconfig" source "drivers/rapidio/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index fd11b9ac4cc3..6fba7daba591 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -27,7 +27,7 @@ obj-y += idle/ obj-y += char/ipmi/ obj-$(CONFIG_ACPI) += acpi/ -obj-$(CONFIG_SFI) += sfi/ + # PnP must come after ACPI since it will eventually need to check if acpi # was used and do nothing if so obj-$(CONFIG_PNP) += pnp/ @@ -73,6 +73,7 @@ obj-$(CONFIG_NVM) += lightnvm/ obj-y += base/ block/ misc/ mfd/ nfc/ obj-$(CONFIG_LIBNVDIMM) += nvdimm/ obj-$(CONFIG_DAX) += dax/ +obj-$(CONFIG_CXL_BUS) += cxl/ obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf/ obj-$(CONFIG_NUBUS) += nubus/ obj-y += macintosh/ diff --git a/drivers/accessibility/speakup/serialio.c b/drivers/accessibility/speakup/serialio.c index 403b01d66367..53580bdc5baa 100644 --- a/drivers/accessibility/speakup/serialio.c +++ b/drivers/accessibility/speakup/serialio.c @@ -27,11 +27,11 @@ static const struct old_serial_port *serstate; static int timeouts; static int spk_serial_out(struct spk_synth *in_synth, const char ch); -static void spk_serial_send_xchar(char ch); -static void spk_serial_tiocmset(unsigned int set, unsigned int clear); -static unsigned char spk_serial_in(void); -static unsigned char spk_serial_in_nowait(void); -static void spk_serial_flush_buffer(void); +static void spk_serial_send_xchar(struct spk_synth *in_synth, char ch); +static void spk_serial_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear); +static unsigned char spk_serial_in(struct spk_synth *in_synth); +static unsigned char spk_serial_in_nowait(struct spk_synth *in_synth); +static void spk_serial_flush_buffer(struct spk_synth *in_synth); static int spk_serial_wait_for_xmitr(struct spk_synth *in_synth); struct spk_io_ops spk_serial_io_ops = { @@ -150,7 +150,7 @@ static void start_serial_interrupt(int irq) outb(1, speakup_info.port_tts + UART_FCR); /* Turn FIFO On */ } -static void spk_serial_send_xchar(char ch) +static void spk_serial_send_xchar(struct spk_synth *synth, char ch) { int timeout = SPK_XMITR_TIMEOUT; @@ -162,7 +162,7 @@ static void spk_serial_send_xchar(char ch) outb(ch, speakup_info.port_tts); } -static void spk_serial_tiocmset(unsigned int set, unsigned int clear) +static void spk_serial_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear) { int old = inb(speakup_info.port_tts + UART_MCR); @@ -251,7 +251,7 @@ static int spk_serial_wait_for_xmitr(struct spk_synth *in_synth) return 1; } -static unsigned char spk_serial_in(void) +static unsigned char spk_serial_in(struct spk_synth *in_synth) { int tmout = SPK_SERIAL_TIMEOUT; @@ -265,7 +265,7 @@ static unsigned char spk_serial_in(void) return inb_p(speakup_info.port_tts + UART_RX); } -static unsigned char spk_serial_in_nowait(void) +static unsigned char spk_serial_in_nowait(struct spk_synth *in_synth) { unsigned char lsr; @@ -275,7 +275,7 @@ static unsigned char spk_serial_in_nowait(void) return inb_p(speakup_info.port_tts + UART_RX); } -static void spk_serial_flush_buffer(void) +static void spk_serial_flush_buffer(struct spk_synth *in_synth) { /* TODO: flush the UART 16550 buffer */ } @@ -307,7 +307,7 @@ const char *spk_serial_synth_immediate(struct spk_synth *synth, } EXPORT_SYMBOL_GPL(spk_serial_synth_immediate); -void spk_serial_release(void) +void spk_serial_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts == 0) diff --git a/drivers/accessibility/speakup/speakup_acntpc.c b/drivers/accessibility/speakup/speakup_acntpc.c index c94328a5bd4a..c1ec087dca13 100644 --- a/drivers/accessibility/speakup/speakup_acntpc.c +++ b/drivers/accessibility/speakup/speakup_acntpc.c @@ -25,7 +25,7 @@ #define PROCSPEECH '\r' static int synth_probe(struct spk_synth *synth); -static void accent_release(void); +static void accent_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -294,7 +294,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void accent_release(void) +static void accent_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_apollo.c b/drivers/accessibility/speakup/speakup_apollo.c index 0877b4044c28..cd63581b2e99 100644 --- a/drivers/accessibility/speakup/speakup_apollo.c +++ b/drivers/accessibility/speakup/speakup_apollo.c @@ -163,8 +163,8 @@ static void do_catch_up(struct spk_synth *synth) full_time_val = full_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); if (!synth->io_ops->synth_out(synth, ch)) { - synth->io_ops->tiocmset(0, UART_MCR_RTS); - synth->io_ops->tiocmset(UART_MCR_RTS, 0); + synth->io_ops->tiocmset(synth, 0, UART_MCR_RTS); + synth->io_ops->tiocmset(synth, UART_MCR_RTS, 0); schedule_timeout(msecs_to_jiffies(full_time_val)); continue; } diff --git a/drivers/accessibility/speakup/speakup_audptr.c b/drivers/accessibility/speakup/speakup_audptr.c index e6a6a9665d8f..e89fd72579e6 100644 --- a/drivers/accessibility/speakup/speakup_audptr.c +++ b/drivers/accessibility/speakup/speakup_audptr.c @@ -119,8 +119,8 @@ static struct spk_synth synth_audptr = { static void synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); - synth->io_ops->send_xchar(SYNTH_CLEAR); + synth->io_ops->flush_buffer(synth); + synth->io_ops->send_xchar(synth, SYNTH_CLEAR); synth->io_ops->synth_out(synth, PROCSPEECH); } @@ -130,11 +130,11 @@ static void synth_version(struct spk_synth *synth) char synth_id[40] = ""; synth->synth_immediate(synth, "\x05[Q]"); - synth_id[test] = synth->io_ops->synth_in(); + synth_id[test] = synth->io_ops->synth_in(synth); if (synth_id[test] == 'A') { do { /* read version string from synth */ - synth_id[++test] = synth->io_ops->synth_in(); + synth_id[++test] = synth->io_ops->synth_in(synth); } while (synth_id[test] != '\n' && test < 32); synth_id[++test] = 0x00; } diff --git a/drivers/accessibility/speakup/speakup_decext.c b/drivers/accessibility/speakup/speakup_decext.c index 7408eb29cf38..092cfd08a9e1 100644 --- a/drivers/accessibility/speakup/speakup_decext.c +++ b/drivers/accessibility/speakup/speakup_decext.c @@ -218,7 +218,7 @@ static void do_catch_up(struct spk_synth *synth) static void synth_flush(struct spk_synth *synth) { in_escape = 0; - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->synth_immediate(synth, "\033P;10z\033\\"); } diff --git a/drivers/accessibility/speakup/speakup_decpc.c b/drivers/accessibility/speakup/speakup_decpc.c index 96f24c848cc5..dec314dee214 100644 --- a/drivers/accessibility/speakup/speakup_decpc.c +++ b/drivers/accessibility/speakup/speakup_decpc.c @@ -125,7 +125,7 @@ enum { PRIMARY_DIC = 0, USER_DIC, COMMAND_DIC, ABBREV_DIC }; #define SYNTH_IO_EXTENT 8 static int synth_probe(struct spk_synth *synth); -static void dtpc_release(void); +static void dtpc_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -474,7 +474,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void dtpc_release(void) +static void dtpc_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_dectlk.c b/drivers/accessibility/speakup/speakup_dectlk.c index ab6d61e80b1c..580ec796816b 100644 --- a/drivers/accessibility/speakup/speakup_dectlk.c +++ b/drivers/accessibility/speakup/speakup_dectlk.c @@ -78,6 +78,8 @@ static struct kobj_attribute direct_attribute = __ATTR(direct, 0644, spk_var_show, spk_var_store); static struct kobj_attribute full_time_attribute = __ATTR(full_time, 0644, spk_var_show, spk_var_store); +static struct kobj_attribute flush_time_attribute = + __ATTR(flush_time, 0644, spk_var_show, spk_var_store); static struct kobj_attribute jiffy_delta_attribute = __ATTR(jiffy_delta, 0644, spk_var_show, spk_var_store); static struct kobj_attribute trigger_time_attribute = @@ -99,6 +101,7 @@ static struct attribute *synth_attrs[] = { &delay_time_attribute.attr, &direct_attribute.attr, &full_time_attribute.attr, + &flush_time_attribute.attr, &jiffy_delta_attribute.attr, &trigger_time_attribute.attr, NULL, /* need to NULL terminate the list of attributes */ @@ -118,6 +121,7 @@ static struct spk_synth synth_dectlk = { .trigger = 50, .jiffies = 50, .full = 40000, + .flush_time = 4000, .dev_name = SYNTH_DEFAULT_DEV, .startup = SYNTH_START, .checkval = SYNTH_CHECK, @@ -200,18 +204,23 @@ static void do_catch_up(struct spk_synth *synth) static u_char last = '\0'; unsigned long flags; unsigned long jiff_max; - unsigned long timeout = msecs_to_jiffies(4000); + unsigned long timeout; DEFINE_WAIT(wait); struct var_t *jiffy_delta; struct var_t *delay_time; + struct var_t *flush_time; int jiffy_delta_val; int delay_time_val; + int timeout_val; jiffy_delta = spk_get_var(JIFFY); delay_time = spk_get_var(DELAY); + flush_time = spk_get_var(FLUSH); spin_lock_irqsave(&speakup_info.spinlock, flags); jiffy_delta_val = jiffy_delta->u.n.value; + timeout_val = flush_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); + timeout = msecs_to_jiffies(timeout_val); jiff_max = jiffies + jiffy_delta_val; while (!kthread_should_stop()) { @@ -289,7 +298,7 @@ static void synth_flush(struct spk_synth *synth) synth->io_ops->synth_out(synth, ']'); in_escape = 0; is_flushing = 1; - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->io_ops->synth_out(synth, SYNTH_CLEAR); } diff --git a/drivers/accessibility/speakup/speakup_dtlk.c b/drivers/accessibility/speakup/speakup_dtlk.c index dbebed0eeeec..92838d3ae9eb 100644 --- a/drivers/accessibility/speakup/speakup_dtlk.c +++ b/drivers/accessibility/speakup/speakup_dtlk.c @@ -24,7 +24,7 @@ #define PROCSPEECH 0x00 static int synth_probe(struct spk_synth *synth); -static void dtlk_release(void); +static void dtlk_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -365,7 +365,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void dtlk_release(void) +static void dtlk_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_keypc.c b/drivers/accessibility/speakup/speakup_keypc.c index 414827e888fc..311f4aa0be22 100644 --- a/drivers/accessibility/speakup/speakup_keypc.c +++ b/drivers/accessibility/speakup/speakup_keypc.c @@ -24,7 +24,7 @@ #define SYNTH_CLEAR 0x03 static int synth_probe(struct spk_synth *synth); -static void keynote_release(void); +static void keynote_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -295,7 +295,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void keynote_release(void) +static void keynote_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (synth_port) diff --git a/drivers/accessibility/speakup/speakup_ltlk.c b/drivers/accessibility/speakup/speakup_ltlk.c index 3c59519a871f..3e59b387d0c4 100644 --- a/drivers/accessibility/speakup/speakup_ltlk.c +++ b/drivers/accessibility/speakup/speakup_ltlk.c @@ -132,7 +132,7 @@ static void synth_interrogate(struct spk_synth *synth) synth->synth_immediate(synth, "\x18\x01?"); for (i = 0; i < 50; i++) { - buf[i] = synth->io_ops->synth_in(); + buf[i] = synth->io_ops->synth_in(synth); if (i > 2 && buf[i] == 0x7f) break; } diff --git a/drivers/accessibility/speakup/speakup_soft.c b/drivers/accessibility/speakup/speakup_soft.c index 9a7029539f35..c3f97c572fb6 100644 --- a/drivers/accessibility/speakup/speakup_soft.c +++ b/drivers/accessibility/speakup/speakup_soft.c @@ -24,7 +24,7 @@ #define CLEAR_SYNTH 0x18 static int softsynth_probe(struct spk_synth *synth); -static void softsynth_release(void); +static void softsynth_release(struct spk_synth *synth); static int softsynth_is_alive(struct spk_synth *synth); static unsigned char get_index(struct spk_synth *synth); @@ -402,7 +402,7 @@ static int softsynth_probe(struct spk_synth *synth) return 0; } -static void softsynth_release(void) +static void softsynth_release(struct spk_synth *synth) { misc_deregister(&synth_device); misc_deregister(&synthu_device); diff --git a/drivers/accessibility/speakup/speakup_spkout.c b/drivers/accessibility/speakup/speakup_spkout.c index 6e933bf1de2e..bd3d8dc300ff 100644 --- a/drivers/accessibility/speakup/speakup_spkout.c +++ b/drivers/accessibility/speakup/speakup_spkout.c @@ -117,8 +117,8 @@ static struct spk_synth synth_spkout = { static void synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); - synth->io_ops->send_xchar(SYNTH_CLEAR); + synth->io_ops->flush_buffer(synth); + synth->io_ops->send_xchar(synth, SYNTH_CLEAR); } module_param_named(ser, synth_spkout.ser, int, 0444); diff --git a/drivers/accessibility/speakup/spk_priv.h b/drivers/accessibility/speakup/spk_priv.h index 0f4bcbe5ddb9..9da57ead17cb 100644 --- a/drivers/accessibility/speakup/spk_priv.h +++ b/drivers/accessibility/speakup/spk_priv.h @@ -34,8 +34,8 @@ const struct old_serial_port *spk_serial_init(int index); void spk_stop_serial_interrupt(void); -void spk_serial_release(void); -void spk_ttyio_release(void); +void spk_serial_release(struct spk_synth *synth); +void spk_ttyio_release(struct spk_synth *synth); void spk_ttyio_register_ldisc(void); void spk_ttyio_unregister_ldisc(void); diff --git a/drivers/accessibility/speakup/spk_ttyio.c b/drivers/accessibility/speakup/spk_ttyio.c index 835d17455fcd..9af1d4c124d3 100644 --- a/drivers/accessibility/speakup/spk_ttyio.c +++ b/drivers/accessibility/speakup/spk_ttyio.c @@ -12,14 +12,15 @@ struct spk_ldisc_data { char buf; struct completion completion; bool buf_free; + struct spk_synth *synth; }; -static struct spk_synth *spk_ttyio_synth; -static struct tty_struct *speakup_tty; -/* mutex to protect against speakup_tty disappearing from underneath us while - * we are using it. this can happen when the device physically unplugged, - * while in use. it also serialises access to speakup_tty. +/* + * This allows to catch within spk_ttyio_ldisc_open whether it is getting set + * on for a speakup-driven device. */ +static struct tty_struct *speakup_tty; +/* This mutex serializes the use of such global speakup_tty variable */ static DEFINE_MUTEX(speakup_tty_mutex); static int ser_to_dev(int ser, dev_t *dev_no) @@ -67,22 +68,20 @@ static int spk_ttyio_ldisc_open(struct tty_struct *tty) static void spk_ttyio_ldisc_close(struct tty_struct *tty) { - mutex_lock(&speakup_tty_mutex); - kfree(speakup_tty->disc_data); - speakup_tty = NULL; - mutex_unlock(&speakup_tty_mutex); + kfree(tty->disc_data); } static int spk_ttyio_receive_buf2(struct tty_struct *tty, const unsigned char *cp, char *fp, int count) { struct spk_ldisc_data *ldisc_data = tty->disc_data; + struct spk_synth *synth = ldisc_data->synth; - if (spk_ttyio_synth->read_buff_add) { + if (synth->read_buff_add) { int i; for (i = 0; i < count; i++) - spk_ttyio_synth->read_buff_add(cp[i]); + synth->read_buff_add(cp[i]); return count; } @@ -114,11 +113,11 @@ static struct tty_ldisc_ops spk_ttyio_ldisc_ops = { static int spk_ttyio_out(struct spk_synth *in_synth, const char ch); static int spk_ttyio_out_unicode(struct spk_synth *in_synth, u16 ch); -static void spk_ttyio_send_xchar(char ch); -static void spk_ttyio_tiocmset(unsigned int set, unsigned int clear); -static unsigned char spk_ttyio_in(void); -static unsigned char spk_ttyio_in_nowait(void); -static void spk_ttyio_flush_buffer(void); +static void spk_ttyio_send_xchar(struct spk_synth *in_synth, char ch); +static void spk_ttyio_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear); +static unsigned char spk_ttyio_in(struct spk_synth *in_synth); +static unsigned char spk_ttyio_in_nowait(struct spk_synth *in_synth); +static void spk_ttyio_flush_buffer(struct spk_synth *in_synth); static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth); struct spk_io_ops spk_ttyio_ops = { @@ -187,13 +186,17 @@ static int spk_ttyio_initialise_ldisc(struct spk_synth *synth) mutex_lock(&speakup_tty_mutex); speakup_tty = tty; ret = tty_set_ldisc(tty, N_SPEAKUP); - if (ret) - speakup_tty = NULL; + speakup_tty = NULL; mutex_unlock(&speakup_tty_mutex); - if (!ret) + if (!ret) { /* Success */ + struct spk_ldisc_data *ldisc_data = tty->disc_data; + + ldisc_data->synth = synth; + synth->dev = tty; return 0; + } pr_err("speakup: Failed to set N_SPEAKUP on tty\n"); @@ -221,29 +224,30 @@ void spk_ttyio_unregister_ldisc(void) static int spk_ttyio_out(struct spk_synth *in_synth, const char ch) { - mutex_lock(&speakup_tty_mutex); - if (in_synth->alive && speakup_tty && speakup_tty->ops->write) { - int ret = speakup_tty->ops->write(speakup_tty, &ch, 1); + struct tty_struct *tty = in_synth->dev; + int ret; - mutex_unlock(&speakup_tty_mutex); - if (ret == 0) - /* No room */ - return 0; - if (ret < 0) { - pr_warn("%s: I/O error, deactivating speakup\n", - in_synth->long_name); - /* No synth any more, so nobody will restart TTYs, - * and we thus need to do it ourselves. Now that there - * is no synth we can let application flood anyway - */ - in_synth->alive = 0; - speakup_start_ttys(); - return 0; - } + if (!in_synth->alive || !tty->ops->write) + return 0; + + ret = tty->ops->write(tty, &ch, 1); + + if (ret == 0) + /* No room */ + return 0; + + if (ret > 0) + /* Success */ return 1; - } - mutex_unlock(&speakup_tty_mutex); + pr_warn("%s: I/O error, deactivating speakup\n", + in_synth->long_name); + /* No synth any more, so nobody will restart TTYs, + * and we thus need to do it ourselves. Now that there + * is no synth we can let application flood anyway + */ + in_synth->alive = 0; + speakup_start_ttys(); return 0; } @@ -264,47 +268,20 @@ static int spk_ttyio_out_unicode(struct spk_synth *in_synth, u16 ch) return ret; } -static int check_tty(struct tty_struct *tty) +static void spk_ttyio_send_xchar(struct spk_synth *in_synth, char ch) { - if (!tty) { - pr_warn("%s: I/O error, deactivating speakup\n", - spk_ttyio_synth->long_name); - /* No synth any more, so nobody will restart TTYs, and we thus - * need to do it ourselves. Now that there is no synth we can - * let application flood anyway - */ - spk_ttyio_synth->alive = 0; - speakup_start_ttys(); - return 1; - } + struct tty_struct *tty = in_synth->dev; - return 0; + if (tty->ops->send_xchar) + tty->ops->send_xchar(tty, ch); } -static void spk_ttyio_send_xchar(char ch) +static void spk_ttyio_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->send_xchar) - speakup_tty->ops->send_xchar(speakup_tty, ch); - mutex_unlock(&speakup_tty_mutex); -} - -static void spk_ttyio_tiocmset(unsigned int set, unsigned int clear) -{ - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } - - if (speakup_tty->ops->tiocmset) - speakup_tty->ops->tiocmset(speakup_tty, set, clear); - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->tiocmset) + tty->ops->tiocmset(tty, set, clear); } static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth) @@ -312,9 +289,10 @@ static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth) return 1; } -static unsigned char ttyio_in(int timeout) +static unsigned char ttyio_in(struct spk_synth *in_synth, int timeout) { - struct spk_ldisc_data *ldisc_data = speakup_tty->disc_data; + struct tty_struct *tty = in_synth->dev; + struct spk_ldisc_data *ldisc_data = tty->disc_data; char rv; if (!timeout) { @@ -334,35 +312,29 @@ static unsigned char ttyio_in(int timeout) mb(); ldisc_data->buf_free = true; /* Let TTY push more characters */ - tty_schedule_flip(speakup_tty->port); + tty_schedule_flip(tty->port); return rv; } -static unsigned char spk_ttyio_in(void) +static unsigned char spk_ttyio_in(struct spk_synth *in_synth) { - return ttyio_in(SPK_SYNTH_TIMEOUT); + return ttyio_in(in_synth, SPK_SYNTH_TIMEOUT); } -static unsigned char spk_ttyio_in_nowait(void) +static unsigned char spk_ttyio_in_nowait(struct spk_synth *in_synth) { - u8 rv = ttyio_in(0); + u8 rv = ttyio_in(in_synth, 0); return (rv == 0xff) ? 0 : rv; } -static void spk_ttyio_flush_buffer(void) +static void spk_ttyio_flush_buffer(struct spk_synth *in_synth) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->flush_buffer) - speakup_tty->ops->flush_buffer(speakup_tty); - - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->flush_buffer) + tty->ops->flush_buffer(tty); } int spk_ttyio_synth_probe(struct spk_synth *synth) @@ -373,37 +345,38 @@ int spk_ttyio_synth_probe(struct spk_synth *synth) return rv; synth->alive = 1; - spk_ttyio_synth = synth; return 0; } EXPORT_SYMBOL_GPL(spk_ttyio_synth_probe); -void spk_ttyio_release(void) +void spk_ttyio_release(struct spk_synth *in_synth) { - if (!speakup_tty) - return; + struct tty_struct *tty = in_synth->dev; - tty_lock(speakup_tty); + tty_lock(tty); - if (speakup_tty->ops->close) - speakup_tty->ops->close(speakup_tty, NULL); + if (tty->ops->close) + tty->ops->close(tty, NULL); - tty_ldisc_flush(speakup_tty); - tty_unlock(speakup_tty); - tty_kclose(speakup_tty); + tty_ldisc_flush(tty); + tty_unlock(tty); + tty_kclose(tty); + + in_synth->dev = NULL; } EXPORT_SYMBOL_GPL(spk_ttyio_release); -const char *spk_ttyio_synth_immediate(struct spk_synth *synth, const char *buff) +const char *spk_ttyio_synth_immediate(struct spk_synth *in_synth, const char *buff) { + struct tty_struct *tty = in_synth->dev; u_char ch; while ((ch = *buff)) { if (ch == '\n') - ch = synth->procspeech; - if (tty_write_room(speakup_tty) < 1 || - !synth->io_ops->synth_out(synth, ch)) + ch = in_synth->procspeech; + if (tty_write_room(tty) < 1 || + !in_synth->io_ops->synth_out(in_synth, ch)) return buff; buff++; } diff --git a/drivers/accessibility/speakup/spk_types.h b/drivers/accessibility/speakup/spk_types.h index 91fca3033a45..6a96ad94bc3f 100644 --- a/drivers/accessibility/speakup/spk_types.h +++ b/drivers/accessibility/speakup/spk_types.h @@ -48,7 +48,7 @@ enum var_id_t { ATTRIB_BLEEP, BLEEPS, RATE, PITCH, VOL, TONE, PUNCT, VOICE, FREQUENCY, LANG, DIRECT, PAUSE, - CAPS_START, CAPS_STOP, CHARTAB, INFLECTION, + CAPS_START, CAPS_STOP, CHARTAB, INFLECTION, FLUSH, MAXVARS }; @@ -157,11 +157,11 @@ struct spk_synth; struct spk_io_ops { int (*synth_out)(struct spk_synth *synth, const char ch); int (*synth_out_unicode)(struct spk_synth *synth, u16 ch); - void (*send_xchar)(char ch); - void (*tiocmset)(unsigned int set, unsigned int clear); - unsigned char (*synth_in)(void); - unsigned char (*synth_in_nowait)(void); - void (*flush_buffer)(void); + void (*send_xchar)(struct spk_synth *synth, char ch); + void (*tiocmset)(struct spk_synth *synth, unsigned int set, unsigned int clear); + unsigned char (*synth_in)(struct spk_synth *synth); + unsigned char (*synth_in_nowait)(struct spk_synth *synth); + void (*flush_buffer)(struct spk_synth *synth); int (*wait_for_xmitr)(struct spk_synth *synth); }; @@ -178,6 +178,7 @@ struct spk_synth { int trigger; int jiffies; int full; + int flush_time; int ser; char *dev_name; short flags; @@ -188,7 +189,7 @@ struct spk_synth { int *default_vol; struct spk_io_ops *io_ops; int (*probe)(struct spk_synth *synth); - void (*release)(void); + void (*release)(struct spk_synth *synth); const char *(*synth_immediate)(struct spk_synth *synth, const char *buff); void (*catch_up)(struct spk_synth *synth); @@ -200,6 +201,8 @@ struct spk_synth { struct synth_indexing indexing; int alive; struct attribute_group attributes; + + void *dev; }; /* diff --git a/drivers/accessibility/speakup/synth.c b/drivers/accessibility/speakup/synth.c index ac47dbac7207..2b8699673bac 100644 --- a/drivers/accessibility/speakup/synth.c +++ b/drivers/accessibility/speakup/synth.c @@ -137,14 +137,14 @@ EXPORT_SYMBOL_GPL(spk_do_catch_up_unicode); void spk_synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->io_ops->synth_out(synth, synth->clear); } EXPORT_SYMBOL_GPL(spk_synth_flush); unsigned char spk_synth_get_index(struct spk_synth *synth) { - return synth->io_ops->synth_in_nowait(); + return synth->io_ops->synth_in_nowait(synth); } EXPORT_SYMBOL_GPL(spk_synth_get_index); @@ -348,6 +348,7 @@ struct var_t synth_time_vars[] = { { TRIGGER, .u.n = {NULL, 20, 10, 2000, 0, 0, NULL } }, { JIFFY, .u.n = {NULL, 50, 20, 200, 0, 0, NULL } }, { FULL, .u.n = {NULL, 400, 200, 60000, 0, 0, NULL } }, + { FLUSH, .u.n = {NULL, 4000, 100, 4000, 0, 0, NULL } }, V_LAST_VAR }; @@ -408,6 +409,8 @@ static int do_synth_init(struct spk_synth *in_synth) synth_time_vars[2].u.n.default_val = synth->jiffies; synth_time_vars[3].u.n.value = synth_time_vars[3].u.n.default_val = synth->full; + synth_time_vars[4].u.n.value = + synth_time_vars[4].u.n.default_val = synth->flush_time; synth_printf("%s", synth->init); for (var = synth->vars; (var->var_id >= 0) && (var->var_id < MAXVARS); var++) @@ -440,7 +443,7 @@ void synth_release(void) sysfs_remove_group(speakup_kobj, &synth->attributes); for (var = synth->vars; var->var_id != MAXVARS; var++) speakup_unregister_var(var->var_id); - synth->release(); + synth->release(synth); synth = NULL; } diff --git a/drivers/accessibility/speakup/varhandlers.c b/drivers/accessibility/speakup/varhandlers.c index d7f6bec7ff06..067c0da97dcb 100644 --- a/drivers/accessibility/speakup/varhandlers.c +++ b/drivers/accessibility/speakup/varhandlers.c @@ -23,6 +23,7 @@ static struct st_var_header var_headers[] = { { "trigger_time", TRIGGER, VAR_TIME, NULL, NULL }, { "jiffy_delta", JIFFY, VAR_TIME, NULL, NULL }, { "full_time", FULL, VAR_TIME, NULL, NULL }, + { "flush_time", FLUSH, VAR_TIME, NULL, NULL }, { "spell_delay", SPELL_DELAY, VAR_NUM, &spk_spell_delay, NULL }, { "bleeps", BLEEPS, VAR_NUM, &spk_bleeps, NULL }, { "attrib_bleep", ATTRIB_BLEEP, VAR_NUM, &spk_attrib_bleep, NULL }, diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index fc06945d3f99..eedec61e3476 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -87,6 +87,14 @@ config ACPI_SPCR_TABLE This table provides information about the configuration of the earlycon console. +config ACPI_FPDT + bool "ACPI Firmware Performance Data Table (FPDT) support" + depends on X86_64 + help + Enable support for the Firmware Performance Data Table (FPDT). + This table provides information on the timing of the system + boot, S3 suspend and S3 resume firmware code paths. + config ACPI_LPIT bool depends on X86_64 @@ -327,21 +335,7 @@ config ACPI_THERMAL the module will be called thermal. config ACPI_PLATFORM_PROFILE - tristate "ACPI Platform Profile Driver" - default m - help - This driver adds support for platform-profiles on platforms that - support it. - - Platform-profiles can be used to control the platform behaviour. For - example whether to operate in a lower power mode, in a higher - power performance mode or between the two. - - This driver provides the sysfs interface and is used as the registration - point for platform specific drivers. - - Which profiles are supported is determined on a per-platform basis and - should be obtained from the platform specific driver. + tristate config ACPI_CUSTOM_DSDT_FILE string "Custom DSDT Table file to include" diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 52b627c7f977..700b41adf2db 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -57,6 +57,7 @@ acpi-$(CONFIG_X86) += x86/utils.o acpi-$(CONFIG_X86) += x86/s2idle.o acpi-$(CONFIG_DEBUG_FS) += debugfs.o acpi-y += acpi_lpat.o +acpi-$(CONFIG_ACPI_FPDT) += acpi_fpdt.o acpi-$(CONFIG_ACPI_LPIT) += acpi_lpit.o acpi-$(CONFIG_ACPI_GENERIC_GSI) += irq.o acpi-$(CONFIG_ACPI_WATCHDOG) += acpi_watchdog.o diff --git a/drivers/acpi/acpi_fpdt.c b/drivers/acpi/acpi_fpdt.c new file mode 100644 index 000000000000..a89a806a7a2a --- /dev/null +++ b/drivers/acpi/acpi_fpdt.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * FPDT support for exporting boot and suspend/resume performance data + * + * Copyright (C) 2021 Intel Corporation. All rights reserved. + */ + +#define pr_fmt(fmt) "ACPI FPDT: " fmt + +#include + +/* + * FPDT contains ACPI table header and a number of fpdt_subtable_entries. + * Each fpdt_subtable_entry points to a subtable: FBPT or S3PT. + * Each FPDT subtable (FBPT/S3PT) is composed of a fpdt_subtable_header + * and a number of fpdt performance records. + * Each FPDT performance record is composed of a fpdt_record_header and + * performance data fields, for boot or suspend or resume phase. + */ +enum fpdt_subtable_type { + SUBTABLE_FBPT, + SUBTABLE_S3PT, +}; + +struct fpdt_subtable_entry { + u16 type; /* refer to enum fpdt_subtable_type */ + u8 length; + u8 revision; + u32 reserved; + u64 address; /* physical address of the S3PT/FBPT table */ +}; + +struct fpdt_subtable_header { + u32 signature; + u32 length; +}; + +enum fpdt_record_type { + RECORD_S3_RESUME, + RECORD_S3_SUSPEND, + RECORD_BOOT, +}; + +struct fpdt_record_header { + u16 type; /* refer to enum fpdt_record_type */ + u8 length; + u8 revision; +}; + +struct resume_performance_record { + struct fpdt_record_header header; + u32 resume_count; + u64 resume_prev; + u64 resume_avg; +} __attribute__((packed)); + +struct boot_performance_record { + struct fpdt_record_header header; + u32 reserved; + u64 firmware_start; + u64 bootloader_load; + u64 bootloader_launch; + u64 exitbootservice_start; + u64 exitbootservice_end; +} __attribute__((packed)); + +struct suspend_performance_record { + struct fpdt_record_header header; + u64 suspend_start; + u64 suspend_end; +} __attribute__((packed)); + + +static struct resume_performance_record *record_resume; +static struct suspend_performance_record *record_suspend; +static struct boot_performance_record *record_boot; + +#define FPDT_ATTR(phase, name) \ +static ssize_t name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return sprintf(buf, "%llu\n", record_##phase->name); \ +} \ +static struct kobj_attribute name##_attr = \ +__ATTR(name##_ns, 0444, name##_show, NULL) + +FPDT_ATTR(resume, resume_prev); +FPDT_ATTR(resume, resume_avg); +FPDT_ATTR(suspend, suspend_start); +FPDT_ATTR(suspend, suspend_end); +FPDT_ATTR(boot, firmware_start); +FPDT_ATTR(boot, bootloader_load); +FPDT_ATTR(boot, bootloader_launch); +FPDT_ATTR(boot, exitbootservice_start); +FPDT_ATTR(boot, exitbootservice_end); + +static ssize_t resume_count_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%u\n", record_resume->resume_count); +} + +static struct kobj_attribute resume_count_attr = +__ATTR_RO(resume_count); + +static struct attribute *resume_attrs[] = { + &resume_count_attr.attr, + &resume_prev_attr.attr, + &resume_avg_attr.attr, + NULL +}; + +static const struct attribute_group resume_attr_group = { + .attrs = resume_attrs, + .name = "resume", +}; + +static struct attribute *suspend_attrs[] = { + &suspend_start_attr.attr, + &suspend_end_attr.attr, + NULL +}; + +static const struct attribute_group suspend_attr_group = { + .attrs = suspend_attrs, + .name = "suspend", +}; + +static struct attribute *boot_attrs[] = { + &firmware_start_attr.attr, + &bootloader_load_attr.attr, + &bootloader_launch_attr.attr, + &exitbootservice_start_attr.attr, + &exitbootservice_end_attr.attr, + NULL +}; + +static const struct attribute_group boot_attr_group = { + .attrs = boot_attrs, + .name = "boot", +}; + +static struct kobject *fpdt_kobj; + +static int fpdt_process_subtable(u64 address, u32 subtable_type) +{ + struct fpdt_subtable_header *subtable_header; + struct fpdt_record_header *record_header; + char *signature = (subtable_type == SUBTABLE_FBPT ? "FBPT" : "S3PT"); + u32 length, offset; + int result; + + subtable_header = acpi_os_map_memory(address, sizeof(*subtable_header)); + if (!subtable_header) + return -ENOMEM; + + if (strncmp((char *)&subtable_header->signature, signature, 4)) { + pr_info(FW_BUG "subtable signature and type mismatch!\n"); + return -EINVAL; + } + + length = subtable_header->length; + acpi_os_unmap_memory(subtable_header, sizeof(*subtable_header)); + + subtable_header = acpi_os_map_memory(address, length); + if (!subtable_header) + return -ENOMEM; + + offset = sizeof(*subtable_header); + while (offset < length) { + record_header = (void *)subtable_header + offset; + offset += record_header->length; + + switch (record_header->type) { + case RECORD_S3_RESUME: + if (subtable_type != SUBTABLE_S3PT) { + pr_err(FW_BUG "Invalid record %d for subtable %s\n", + record_header->type, signature); + return -EINVAL; + } + if (record_resume) { + pr_err("Duplicate resume performance record found.\n"); + continue; + } + record_resume = (struct resume_performance_record *)record_header; + result = sysfs_create_group(fpdt_kobj, &resume_attr_group); + if (result) + return result; + break; + case RECORD_S3_SUSPEND: + if (subtable_type != SUBTABLE_S3PT) { + pr_err(FW_BUG "Invalid %d for subtable %s\n", + record_header->type, signature); + continue; + } + if (record_suspend) { + pr_err("Duplicate suspend performance record found.\n"); + continue; + } + record_suspend = (struct suspend_performance_record *)record_header; + result = sysfs_create_group(fpdt_kobj, &suspend_attr_group); + if (result) + return result; + break; + case RECORD_BOOT: + if (subtable_type != SUBTABLE_FBPT) { + pr_err(FW_BUG "Invalid %d for subtable %s\n", + record_header->type, signature); + return -EINVAL; + } + if (record_boot) { + pr_err("Duplicate boot performance record found.\n"); + continue; + } + record_boot = (struct boot_performance_record *)record_header; + result = sysfs_create_group(fpdt_kobj, &boot_attr_group); + if (result) + return result; + break; + + default: + pr_err(FW_BUG "Invalid record %d found.\n", record_header->type); + return -EINVAL; + } + } + return 0; +} + +static int __init acpi_init_fpdt(void) +{ + acpi_status status; + struct acpi_table_header *header; + struct fpdt_subtable_entry *subtable; + u32 offset = sizeof(*header); + + status = acpi_get_table(ACPI_SIG_FPDT, 0, &header); + + if (ACPI_FAILURE(status)) + return 0; + + fpdt_kobj = kobject_create_and_add("fpdt", acpi_kobj); + if (!fpdt_kobj) + return -ENOMEM; + + while (offset < header->length) { + subtable = (void *)header + offset; + switch (subtable->type) { + case SUBTABLE_FBPT: + case SUBTABLE_S3PT: + fpdt_process_subtable(subtable->address, + subtable->type); + break; + default: + pr_info(FW_BUG "Invalid subtable type %d found.\n", + subtable->type); + break; + } + offset += sizeof(*subtable); + } + return 0; +} + +fs_initcall(acpi_init_fpdt); diff --git a/drivers/acpi/acpica/acobject.h b/drivers/acpi/acpica/acobject.h index af47a3ffd2a4..9db5ae0f79ea 100644 --- a/drivers/acpi/acpica/acobject.h +++ b/drivers/acpi/acpica/acobject.h @@ -284,6 +284,7 @@ struct acpi_object_addr_handler { acpi_adr_space_handler handler; struct acpi_namespace_node *node; /* Parent device */ void *context; + acpi_mutex context_mutex; acpi_adr_space_setup setup; union acpi_operand_object *region_list; /* Regions using this handler */ union acpi_operand_object *next; diff --git a/drivers/acpi/acpica/evhandler.c b/drivers/acpi/acpica/evhandler.c index ea9485e6a475..c0cd7147a5a3 100644 --- a/drivers/acpi/acpica/evhandler.c +++ b/drivers/acpi/acpica/evhandler.c @@ -489,6 +489,13 @@ acpi_ev_install_space_handler(struct acpi_namespace_node *node, /* Init handler obj */ + status = + acpi_os_create_mutex(&handler_obj->address_space.context_mutex); + if (ACPI_FAILURE(status)) { + acpi_ut_remove_reference(handler_obj); + goto unlock_and_exit; + } + handler_obj->address_space.space_id = (u8)space_id; handler_obj->address_space.handler_flags = flags; handler_obj->address_space.region_list = NULL; diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c index 3ed7d9ae95cf..4ef43c8ef5e7 100644 --- a/drivers/acpi/acpica/evregion.c +++ b/drivers/acpi/acpica/evregion.c @@ -112,6 +112,8 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, union acpi_operand_object *region_obj2; void *region_context = NULL; struct acpi_connection_info *context; + acpi_mutex context_mutex; + u8 context_locked; acpi_physical_address address; ACPI_FUNCTION_TRACE(ev_address_space_dispatch); @@ -136,6 +138,8 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, } context = handler_desc->address_space.context; + context_mutex = handler_desc->address_space.context_mutex; + context_locked = FALSE; /* * It may be the case that the region has never been initialized. @@ -204,41 +208,6 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, handler = handler_desc->address_space.handler; address = (region_obj->region.address + region_offset); - /* - * Special handling for generic_serial_bus and general_purpose_io: - * There are three extra parameters that must be passed to the - * handler via the context: - * 1) Connection buffer, a resource template from Connection() op - * 2) Length of the above buffer - * 3) Actual access length from the access_as() op - * - * In addition, for general_purpose_io, the Address and bit_width fields - * are defined as follows: - * 1) Address is the pin number index of the field (bit offset from - * the previous Connection) - * 2) bit_width is the actual bit length of the field (number of pins) - */ - if ((region_obj->region.space_id == ACPI_ADR_SPACE_GSBUS) && - context && field_obj) { - - /* Get the Connection (resource_template) buffer */ - - context->connection = field_obj->field.resource_buffer; - context->length = field_obj->field.resource_length; - context->access_length = field_obj->field.access_length; - } - if ((region_obj->region.space_id == ACPI_ADR_SPACE_GPIO) && - context && field_obj) { - - /* Get the Connection (resource_template) buffer */ - - context->connection = field_obj->field.resource_buffer; - context->length = field_obj->field.resource_length; - context->access_length = field_obj->field.access_length; - address = field_obj->field.pin_number_index; - bit_width = field_obj->field.bit_length; - } - ACPI_DEBUG_PRINT((ACPI_DB_OPREGION, "Handler %p (@%p) Address %8.8X%8.8X [%s]\n", ®ion_obj->region.handler->address_space, handler, @@ -256,11 +225,58 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, acpi_ex_exit_interpreter(); } + /* + * Special handling for generic_serial_bus and general_purpose_io: + * There are three extra parameters that must be passed to the + * handler via the context: + * 1) Connection buffer, a resource template from Connection() op + * 2) Length of the above buffer + * 3) Actual access length from the access_as() op + * + * Since we pass these extra parameters via the context, which is + * shared between threads, we must lock the context to avoid these + * parameters being changed from another thread before the handler + * has completed running. + * + * In addition, for general_purpose_io, the Address and bit_width fields + * are defined as follows: + * 1) Address is the pin number index of the field (bit offset from + * the previous Connection) + * 2) bit_width is the actual bit length of the field (number of pins) + */ + if ((region_obj->region.space_id == ACPI_ADR_SPACE_GSBUS || + region_obj->region.space_id == ACPI_ADR_SPACE_GPIO) && + context && field_obj) { + + status = + acpi_os_acquire_mutex(context_mutex, ACPI_WAIT_FOREVER); + if (ACPI_FAILURE(status)) { + goto re_enter_interpreter; + } + + context_locked = TRUE; + + /* Get the Connection (resource_template) buffer */ + + context->connection = field_obj->field.resource_buffer; + context->length = field_obj->field.resource_length; + context->access_length = field_obj->field.access_length; + + if (region_obj->region.space_id == ACPI_ADR_SPACE_GPIO) { + address = field_obj->field.pin_number_index; + bit_width = field_obj->field.bit_length; + } + } + /* Call the handler */ status = handler(function, address, bit_width, value, context, region_obj2->extra.region_context); + if (context_locked) { + acpi_os_release_mutex(context_mutex); + } + if (ACPI_FAILURE(status)) { ACPI_EXCEPTION((AE_INFO, status, "Returned by Handler for [%s]", acpi_ut_get_region_name(region_obj->region. @@ -277,6 +293,7 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, } } +re_enter_interpreter: if (!(handler_desc->address_space.handler_flags & ACPI_ADDR_HANDLER_DEFAULT_INSTALLED)) { /* diff --git a/drivers/acpi/acpica/evxfregn.c b/drivers/acpi/acpica/evxfregn.c index 7672d70da850..b1ff0a8f9c14 100644 --- a/drivers/acpi/acpica/evxfregn.c +++ b/drivers/acpi/acpica/evxfregn.c @@ -201,6 +201,8 @@ acpi_remove_address_space_handler(acpi_handle device, /* Now we can delete the handler object */ + acpi_os_release_mutex(handler_obj->address_space. + context_mutex); acpi_ut_remove_reference(handler_obj); goto unlock_and_exit; } diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index 0bf072cef6cf..dcd593766a64 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -56,8 +56,6 @@ static struct acpi_scan_handler pci_root_handler = { }, }; -static DEFINE_MUTEX(osc_lock); - /** * acpi_is_root_bridge - determine whether an ACPI CA node is a PCI root bridge * @handle: the ACPI CA node in question. @@ -223,12 +221,7 @@ static acpi_status acpi_pci_query_osc(struct acpi_pci_root *root, static acpi_status acpi_pci_osc_support(struct acpi_pci_root *root, u32 flags) { - acpi_status status; - - mutex_lock(&osc_lock); - status = acpi_pci_query_osc(root, flags, NULL); - mutex_unlock(&osc_lock); - return status; + return acpi_pci_query_osc(root, flags, NULL); } struct acpi_pci_root *acpi_pci_find_root(acpi_handle handle) @@ -353,10 +346,10 @@ EXPORT_SYMBOL_GPL(acpi_get_pci_dev); * _OSC bits the BIOS has granted control of, but its contents are meaningless * on failure. **/ -acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) +static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) { struct acpi_pci_root *root; - acpi_status status = AE_OK; + acpi_status status; u32 ctrl, capbuf[3]; if (!mask) @@ -370,18 +363,16 @@ acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) if (!root) return AE_NOT_EXIST; - mutex_lock(&osc_lock); - *mask = ctrl | root->osc_control_set; /* No need to evaluate _OSC if the control was already granted. */ if ((root->osc_control_set & ctrl) == ctrl) - goto out; + return AE_OK; /* Need to check the available controls bits before requesting them. */ while (*mask) { status = acpi_pci_query_osc(root, root->osc_support_set, mask); if (ACPI_FAILURE(status)) - goto out; + return status; if (ctrl == *mask) break; decode_osc_control(root, "platform does not support", @@ -392,21 +383,19 @@ acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) if ((ctrl & req) != req) { decode_osc_control(root, "not requesting control; platform does not support", req & ~(ctrl)); - status = AE_SUPPORT; - goto out; + return AE_SUPPORT; } capbuf[OSC_QUERY_DWORD] = 0; capbuf[OSC_SUPPORT_DWORD] = root->osc_support_set; capbuf[OSC_CONTROL_DWORD] = ctrl; status = acpi_pci_run_osc(handle, capbuf, mask); - if (ACPI_SUCCESS(status)) - root->osc_control_set = *mask; -out: - mutex_unlock(&osc_lock); - return status; + if (ACPI_FAILURE(status)) + return status; + + root->osc_control_set = *mask; + return AE_OK; } -EXPORT_SYMBOL(acpi_pci_osc_control_set); static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, bool is_pcie) @@ -452,9 +441,8 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, if ((status == AE_NOT_FOUND) && !is_pcie) return; - dev_info(&device->dev, "_OSC failed (%s)%s\n", - acpi_format_exception(status), - pcie_aspm_support_enabled() ? "; disabling ASPM" : ""); + dev_info(&device->dev, "_OSC: platform retains control of PCIe features (%s)\n", + acpi_format_exception(status)); return; } @@ -510,7 +498,7 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, } else { decode_osc_control(root, "OS requested", requested); decode_osc_control(root, "platform willing to grant", control); - dev_info(&device->dev, "_OSC failed (%s); disabling ASPM\n", + dev_info(&device->dev, "_OSC: platform retains control of PCIe features (%s)\n", acpi_format_exception(status)); /* diff --git a/drivers/acpi/platform_profile.c b/drivers/acpi/platform_profile.c index 4a59c5993bde..dd2fbf38e414 100644 --- a/drivers/acpi/platform_profile.c +++ b/drivers/acpi/platform_profile.c @@ -17,6 +17,7 @@ static const char * const profile_names[] = { [PLATFORM_PROFILE_COOL] = "cool", [PLATFORM_PROFILE_QUIET] = "quiet", [PLATFORM_PROFILE_BALANCED] = "balanced", + [PLATFORM_PROFILE_BALANCED_PERFORMANCE] = "balanced-performance", [PLATFORM_PROFILE_PERFORMANCE] = "performance", }; static_assert(ARRAY_SIZE(profile_names) == PLATFORM_PROFILE_LAST); diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 4f906380b031..95105db642b9 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -670,27 +670,24 @@ static int thermal_get_trend(struct thermal_zone_device *thermal, return 0; } - -static int thermal_notify(struct thermal_zone_device *thermal, int trip, - enum thermal_trip_type trip_type) +static void acpi_thermal_zone_device_hot(struct thermal_zone_device *thermal) { - u8 type = 0; struct acpi_thermal *tz = thermal->devdata; - if (trip_type == THERMAL_TRIP_CRITICAL) - type = ACPI_THERMAL_NOTIFY_CRITICAL; - else if (trip_type == THERMAL_TRIP_HOT) - type = ACPI_THERMAL_NOTIFY_HOT; - else - return 0; + acpi_bus_generate_netlink_event(tz->device->pnp.device_class, + dev_name(&tz->device->dev), + ACPI_THERMAL_NOTIFY_HOT, 1); +} + +static void acpi_thermal_zone_device_critical(struct thermal_zone_device *thermal) +{ + struct acpi_thermal *tz = thermal->devdata; acpi_bus_generate_netlink_event(tz->device->pnp.device_class, - dev_name(&tz->device->dev), type, 1); + dev_name(&tz->device->dev), + ACPI_THERMAL_NOTIFY_CRITICAL, 1); - if (trip_type == THERMAL_TRIP_CRITICAL && nocrt) - return 1; - - return 0; + thermal_zone_device_critical(thermal); } static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal, @@ -760,25 +757,6 @@ static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal, } } - for (i = 0; i < tz->devices.count; i++) { - handle = tz->devices.handles[i]; - status = acpi_bus_get_device(handle, &dev); - if (ACPI_SUCCESS(status) && (dev == device)) { - if (bind) - result = thermal_zone_bind_cooling_device - (thermal, THERMAL_TRIPS_NONE, - cdev, THERMAL_NO_LIMIT, - THERMAL_NO_LIMIT, - THERMAL_WEIGHT_DEFAULT); - else - result = thermal_zone_unbind_cooling_device - (thermal, THERMAL_TRIPS_NONE, - cdev); - if (result) - goto failed; - } - } - failed: return result; } @@ -805,7 +783,8 @@ static struct thermal_zone_device_ops acpi_thermal_zone_ops = { .get_trip_temp = thermal_get_trip_temp, .get_crit_temp = thermal_get_crit_temp, .get_trend = thermal_get_trend, - .notify = thermal_notify, + .hot = acpi_thermal_zone_device_hot, + .critical = acpi_thermal_zone_device_critical, }; static int acpi_thermal_register_thermal_zone(struct acpi_thermal *tz) diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index ecc304149067..939ca220bf78 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -56,31 +56,28 @@ amba_lookup(const struct amba_id *table, struct amba_device *dev) return NULL; } -static int amba_match(struct device *dev, struct device_driver *drv) +static int amba_get_enable_pclk(struct amba_device *pcdev) { - struct amba_device *pcdev = to_amba_device(dev); - struct amba_driver *pcdrv = to_amba_driver(drv); + int ret; - /* When driver_override is set, only bind to the matching driver */ - if (pcdev->driver_override) - return !strcmp(pcdev->driver_override, drv->name); + pcdev->pclk = clk_get(&pcdev->dev, "apb_pclk"); + if (IS_ERR(pcdev->pclk)) + return PTR_ERR(pcdev->pclk); - return amba_lookup(pcdrv->id_table, pcdev) != NULL; + ret = clk_prepare_enable(pcdev->pclk); + if (ret) + clk_put(pcdev->pclk); + + return ret; } -static int amba_uevent(struct device *dev, struct kobj_uevent_env *env) +static void amba_put_disable_pclk(struct amba_device *pcdev) { - struct amba_device *pcdev = to_amba_device(dev); - int retval = 0; - - retval = add_uevent_var(env, "AMBA_ID=%08x", pcdev->periphid); - if (retval) - return retval; - - retval = add_uevent_var(env, "MODALIAS=amba:d%08X", pcdev->periphid); - return retval; + clk_disable_unprepare(pcdev->pclk); + clk_put(pcdev->pclk); } + static ssize_t driver_override_show(struct device *_dev, struct device_attribute *attr, char *buf) { @@ -152,6 +149,109 @@ static struct attribute *amba_dev_attrs[] = { }; ATTRIBUTE_GROUPS(amba_dev); +static int amba_match(struct device *dev, struct device_driver *drv) +{ + struct amba_device *pcdev = to_amba_device(dev); + struct amba_driver *pcdrv = to_amba_driver(drv); + + /* When driver_override is set, only bind to the matching driver */ + if (pcdev->driver_override) + return !strcmp(pcdev->driver_override, drv->name); + + return amba_lookup(pcdrv->id_table, pcdev) != NULL; +} + +static int amba_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + struct amba_device *pcdev = to_amba_device(dev); + int retval = 0; + + retval = add_uevent_var(env, "AMBA_ID=%08x", pcdev->periphid); + if (retval) + return retval; + + retval = add_uevent_var(env, "MODALIAS=amba:d%08X", pcdev->periphid); + return retval; +} + +/* + * These are the device model conversion veneers; they convert the + * device model structures to our more specific structures. + */ +static int amba_probe(struct device *dev) +{ + struct amba_device *pcdev = to_amba_device(dev); + struct amba_driver *pcdrv = to_amba_driver(dev->driver); + const struct amba_id *id = amba_lookup(pcdrv->id_table, pcdev); + int ret; + + do { + ret = of_clk_set_defaults(dev->of_node, false); + if (ret < 0) + break; + + ret = dev_pm_domain_attach(dev, true); + if (ret) + break; + + ret = amba_get_enable_pclk(pcdev); + if (ret) { + dev_pm_domain_detach(dev, true); + break; + } + + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + + ret = pcdrv->probe(pcdev, id); + if (ret == 0) + break; + + pm_runtime_disable(dev); + pm_runtime_set_suspended(dev); + pm_runtime_put_noidle(dev); + + amba_put_disable_pclk(pcdev); + dev_pm_domain_detach(dev, true); + } while (0); + + return ret; +} + +static int amba_remove(struct device *dev) +{ + struct amba_device *pcdev = to_amba_device(dev); + struct amba_driver *drv = to_amba_driver(dev->driver); + + pm_runtime_get_sync(dev); + if (drv->remove) + drv->remove(pcdev); + pm_runtime_put_noidle(dev); + + /* Undo the runtime PM settings in amba_probe() */ + pm_runtime_disable(dev); + pm_runtime_set_suspended(dev); + pm_runtime_put_noidle(dev); + + amba_put_disable_pclk(pcdev); + dev_pm_domain_detach(dev, true); + + return 0; +} + +static void amba_shutdown(struct device *dev) +{ + struct amba_driver *drv; + + if (!dev->driver) + return; + + drv = to_amba_driver(dev->driver); + if (drv->shutdown) + drv->shutdown(to_amba_device(dev)); +} + #ifdef CONFIG_PM /* * Hooks to provide runtime PM of the pclk (bus clock). It is safe to @@ -217,6 +317,9 @@ struct bus_type amba_bustype = { .dev_groups = amba_dev_groups, .match = amba_match, .uevent = amba_uevent, + .probe = amba_probe, + .remove = amba_remove, + .shutdown = amba_shutdown, .dma_configure = platform_dma_configure, .pm = &amba_pm, }; @@ -229,99 +332,6 @@ static int __init amba_init(void) postcore_initcall(amba_init); -static int amba_get_enable_pclk(struct amba_device *pcdev) -{ - int ret; - - pcdev->pclk = clk_get(&pcdev->dev, "apb_pclk"); - if (IS_ERR(pcdev->pclk)) - return PTR_ERR(pcdev->pclk); - - ret = clk_prepare_enable(pcdev->pclk); - if (ret) - clk_put(pcdev->pclk); - - return ret; -} - -static void amba_put_disable_pclk(struct amba_device *pcdev) -{ - clk_disable_unprepare(pcdev->pclk); - clk_put(pcdev->pclk); -} - -/* - * These are the device model conversion veneers; they convert the - * device model structures to our more specific structures. - */ -static int amba_probe(struct device *dev) -{ - struct amba_device *pcdev = to_amba_device(dev); - struct amba_driver *pcdrv = to_amba_driver(dev->driver); - const struct amba_id *id = amba_lookup(pcdrv->id_table, pcdev); - int ret; - - do { - ret = of_clk_set_defaults(dev->of_node, false); - if (ret < 0) - break; - - ret = dev_pm_domain_attach(dev, true); - if (ret) - break; - - ret = amba_get_enable_pclk(pcdev); - if (ret) { - dev_pm_domain_detach(dev, true); - break; - } - - pm_runtime_get_noresume(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); - - ret = pcdrv->probe(pcdev, id); - if (ret == 0) - break; - - pm_runtime_disable(dev); - pm_runtime_set_suspended(dev); - pm_runtime_put_noidle(dev); - - amba_put_disable_pclk(pcdev); - dev_pm_domain_detach(dev, true); - } while (0); - - return ret; -} - -static int amba_remove(struct device *dev) -{ - struct amba_device *pcdev = to_amba_device(dev); - struct amba_driver *drv = to_amba_driver(dev->driver); - int ret; - - pm_runtime_get_sync(dev); - ret = drv->remove(pcdev); - pm_runtime_put_noidle(dev); - - /* Undo the runtime PM settings in amba_probe() */ - pm_runtime_disable(dev); - pm_runtime_set_suspended(dev); - pm_runtime_put_noidle(dev); - - amba_put_disable_pclk(pcdev); - dev_pm_domain_detach(dev, true); - - return ret; -} - -static void amba_shutdown(struct device *dev) -{ - struct amba_driver *drv = to_amba_driver(dev->driver); - drv->shutdown(to_amba_device(dev)); -} - /** * amba_driver_register - register an AMBA device driver * @drv: amba device driver structure @@ -332,12 +342,10 @@ static void amba_shutdown(struct device *dev) */ int amba_driver_register(struct amba_driver *drv) { - drv->drv.bus = &amba_bustype; + if (!drv->probe) + return -EINVAL; -#define SETFN(fn) if (drv->fn) drv->drv.fn = amba_##fn - SETFN(probe); - SETFN(remove); - SETFN(shutdown); + drv->drv.bus = &amba_bustype; return driver_register(&drv->drv); } diff --git a/drivers/android/binderfs.c b/drivers/android/binderfs.c index 7b4f154f07e6..e80ba93c62a9 100644 --- a/drivers/android/binderfs.c +++ b/drivers/android/binderfs.c @@ -355,7 +355,8 @@ static inline bool is_binderfs_control_device(const struct dentry *dentry) return info->control_dentry == dentry; } -static int binderfs_rename(struct inode *old_dir, struct dentry *old_dentry, +static int binderfs_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { @@ -363,7 +364,8 @@ static int binderfs_rename(struct inode *old_dir, struct dentry *old_dentry, is_binderfs_control_device(new_dentry)) return -EPERM; - return simple_rename(old_dir, old_dentry, new_dir, new_dentry, flags); + return simple_rename(&init_user_ns, old_dir, old_dentry, new_dir, + new_dentry, flags); } static int binderfs_unlink(struct inode *dir, struct dentry *dentry) diff --git a/drivers/auxdisplay/Kconfig b/drivers/auxdisplay/Kconfig index a2b59b84bb88..1509cb74705a 100644 --- a/drivers/auxdisplay/Kconfig +++ b/drivers/auxdisplay/Kconfig @@ -507,6 +507,3 @@ config PANEL depends on PARPORT select AUXDISPLAY select PARPORT_PANEL - -config CHARLCD - tristate "Character LCD core support" if COMPILE_TEST diff --git a/drivers/auxdisplay/cfag12864b.c b/drivers/auxdisplay/cfag12864b.c index 7eebae7e322c..fd430e6866a1 100644 --- a/drivers/auxdisplay/cfag12864b.c +++ b/drivers/auxdisplay/cfag12864b.c @@ -5,7 +5,7 @@ * Description: cfag12864b LCD driver * Depends: ks0108 * - * Author: Copyright (C) Miguel Ojeda Sandonis + * Author: Copyright (C) Miguel Ojeda * Date: 2006-10-31 */ @@ -376,5 +376,5 @@ module_init(cfag12864b_init); module_exit(cfag12864b_exit); MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Miguel Ojeda Sandonis "); +MODULE_AUTHOR("Miguel Ojeda "); MODULE_DESCRIPTION("cfag12864b LCD driver"); diff --git a/drivers/auxdisplay/cfag12864bfb.c b/drivers/auxdisplay/cfag12864bfb.c index 2002291ab338..d66821adf453 100644 --- a/drivers/auxdisplay/cfag12864bfb.c +++ b/drivers/auxdisplay/cfag12864bfb.c @@ -5,7 +5,7 @@ * Description: cfag12864b LCD framebuffer driver * Depends: cfag12864b * - * Author: Copyright (C) Miguel Ojeda Sandonis + * Author: Copyright (C) Miguel Ojeda * Date: 2006-10-31 */ @@ -171,5 +171,5 @@ module_init(cfag12864bfb_init); module_exit(cfag12864bfb_exit); MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Miguel Ojeda Sandonis "); +MODULE_AUTHOR("Miguel Ojeda "); MODULE_DESCRIPTION("cfag12864b LCD framebuffer driver"); diff --git a/drivers/auxdisplay/ht16k33.c b/drivers/auxdisplay/ht16k33.c index d951d54b26f5..1e69cc6d21a0 100644 --- a/drivers/auxdisplay/ht16k33.c +++ b/drivers/auxdisplay/ht16k33.c @@ -117,8 +117,7 @@ static void ht16k33_fb_queue(struct ht16k33_priv *priv) { struct ht16k33_fbdev *fbdev = &priv->fbdev; - schedule_delayed_work(&fbdev->work, - msecs_to_jiffies(HZ / fbdev->refresh_rate)); + schedule_delayed_work(&fbdev->work, HZ / fbdev->refresh_rate); } /* @@ -402,11 +401,6 @@ static int ht16k33_probe(struct i2c_client *client, return -EIO; } - if (client->irq <= 0) { - dev_err(&client->dev, "No IRQ specified\n"); - return -EINVAL; - } - priv = devm_kzalloc(&client->dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -459,9 +453,12 @@ static int ht16k33_probe(struct i2c_client *client, if (err) goto err_fbdev_info; - err = ht16k33_keypad_probe(client, &priv->keypad); - if (err) - goto err_fbdev_unregister; + /* Keypad */ + if (client->irq > 0) { + err = ht16k33_keypad_probe(client, &priv->keypad); + if (err) + goto err_fbdev_unregister; + } /* Backlight */ memset(&bl_props, 0, sizeof(struct backlight_properties)); diff --git a/drivers/auxdisplay/ks0108.c b/drivers/auxdisplay/ks0108.c index abfe3fa9e6f4..03c95ad4216c 100644 --- a/drivers/auxdisplay/ks0108.c +++ b/drivers/auxdisplay/ks0108.c @@ -5,7 +5,7 @@ * Description: ks0108 LCD Controller driver * Depends: parport * - * Author: Copyright (C) Miguel Ojeda Sandonis + * Author: Copyright (C) Miguel Ojeda * Date: 2006-10-31 */ @@ -182,6 +182,6 @@ module_init(ks0108_init); module_exit(ks0108_exit); MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Miguel Ojeda Sandonis "); +MODULE_AUTHOR("Miguel Ojeda "); MODULE_DESCRIPTION("ks0108 LCD Controller driver"); diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index 040be48ce046..ffcbe2bc460e 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig @@ -161,7 +161,7 @@ config HMEM_REPORTING default n depends on NUMA help - Enable reporting for heterogenous memory access attributes under + Enable reporting for heterogeneous memory access attributes under their non-uniform memory nodes. source "drivers/base/test/Kconfig" @@ -213,4 +213,10 @@ config GENERIC_ARCH_TOPOLOGY appropriate scaling, sysfs interface for reading capacity values at runtime. +config GENERIC_ARCH_NUMA + bool + help + Enable support for generic NUMA implementation. Currently, RISC-V + and ARM64 use it. + endmenu diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 5e7bf9669a81..8b93a7f291ec 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o obj-y += test/ diff --git a/arch/arm64/mm/numa.c b/drivers/base/arch_numa.c similarity index 93% rename from arch/arm64/mm/numa.c rename to drivers/base/arch_numa.c index a8303bc6b62a..4cc4e117727d 100644 --- a/arch/arm64/mm/numa.c +++ b/drivers/base/arch_numa.c @@ -13,7 +13,6 @@ #include #include -#include #include struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; @@ -356,11 +355,12 @@ static int __init numa_register_nodes(void) /* Check that valid nid is set to memblks */ for_each_mem_region(mblk) { int mblk_nid = memblock_get_region_node(mblk); + phys_addr_t start = mblk->base; + phys_addr_t end = mblk->base + mblk->size - 1; if (mblk_nid == NUMA_NO_NODE || mblk_nid >= MAX_NUMNODES) { - pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", - mblk_nid, mblk->base, - mblk->base + mblk->size - 1); + pr_warn("Warning: invalid memblk node %d [mem %pap-%pap]\n", + mblk_nid, &start, &end); return -EINVAL; } } @@ -428,14 +428,14 @@ out_free_distance: static int __init dummy_numa_init(void) { phys_addr_t start = memblock_start_of_DRAM(); - phys_addr_t end = memblock_end_of_DRAM(); + phys_addr_t end = memblock_end_of_DRAM() - 1; int ret; if (numa_off) pr_info("NUMA disabled\n"); /* Forced off on command line. */ - pr_info("Faking a node at [mem %#018Lx-%#018Lx]\n", start, end - 1); + pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end); - ret = numa_add_memblk(0, start, end); + ret = numa_add_memblk(0, start, end + 1); if (ret) { pr_err("NUMA init failed\n"); return ret; @@ -445,16 +445,36 @@ static int __init dummy_numa_init(void) return 0; } +#ifdef CONFIG_ACPI_NUMA +static int __init arch_acpi_numa_init(void) +{ + int ret; + + ret = acpi_numa_init(); + if (ret) { + pr_info("Failed to initialise from firmware\n"); + return ret; + } + + return srat_disabled() ? -EINVAL : 0; +} +#else +static int __init arch_acpi_numa_init(void) +{ + return -EOPNOTSUPP; +} +#endif + /** - * arm64_numa_init() - Initialize NUMA + * arch_numa_init() - Initialize NUMA * * Try each configured NUMA initialization method until one succeeds. The * last fallback is dummy single node config encompassing whole memory. */ -void __init arm64_numa_init(void) +void __init arch_numa_init(void) { if (!numa_off) { - if (!acpi_disabled && !numa_init(arm64_acpi_numa_init)) + if (!acpi_disabled && !numa_init(arch_acpi_numa_init)) return; if (acpi_disabled && !numa_init(of_numa_init)) return; diff --git a/drivers/base/auxiliary.c b/drivers/base/auxiliary.c index 8336535f1e11..d8b314e7d0fd 100644 --- a/drivers/base/auxiliary.c +++ b/drivers/base/auxiliary.c @@ -15,6 +15,7 @@ #include #include #include +#include "base.h" static const struct auxiliary_device_id *auxiliary_match_id(const struct auxiliary_device_id *id, const struct auxiliary_device *auxdev) @@ -260,19 +261,11 @@ void auxiliary_driver_unregister(struct auxiliary_driver *auxdrv) } EXPORT_SYMBOL_GPL(auxiliary_driver_unregister); -static int __init auxiliary_bus_init(void) +void __init auxiliary_bus_init(void) { - return bus_register(&auxiliary_bus_type); + WARN_ON(bus_register(&auxiliary_bus_type)); } -static void __exit auxiliary_bus_exit(void) -{ - bus_unregister(&auxiliary_bus_type); -} - -module_init(auxiliary_bus_init); -module_exit(auxiliary_bus_exit); - MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Auxiliary Bus"); MODULE_AUTHOR("David Ertman "); diff --git a/drivers/base/base.h b/drivers/base/base.h index f5600a83124f..52b3d7b75c27 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -119,6 +119,11 @@ static inline int hypervisor_init(void) { return 0; } extern int platform_bus_init(void); extern void cpu_dev_init(void); extern void container_dev_init(void); +#ifdef CONFIG_AUXILIARY_BUS +extern void auxiliary_bus_init(void); +#else +static inline void auxiliary_bus_init(void) { } +#endif struct kobject *virtual_device_parent(struct device *dev); diff --git a/drivers/base/bus.c b/drivers/base/bus.c index a9c23ecebc7c..36d0c654ea61 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -633,7 +633,7 @@ int bus_add_driver(struct device_driver *drv) error = driver_add_groups(drv, bus->drv_groups); if (error) { /* How the hell do we get out of this pickle? Give up */ - printk(KERN_ERR "%s: driver_create_groups(%s) failed\n", + printk(KERN_ERR "%s: driver_add_groups(%s) failed\n", __func__, drv->name); } @@ -729,23 +729,6 @@ int device_reprobe(struct device *dev) } EXPORT_SYMBOL_GPL(device_reprobe); -/** - * find_bus - locate bus by name. - * @name: name of bus. - * - * Call kset_find_obj() to iterate over list of buses to - * find a bus by name. Return bus if found. - * - * Note that kset_find_obj increments bus' reference count. - */ -#if 0 -struct bus_type *find_bus(char *name) -{ - struct kobject *k = kset_find_obj(bus_kset, name); - return k ? to_bus(k) : NULL; -} -#endif /* 0 */ - static int bus_add_groups(struct bus_type *bus, const struct attribute_group **groups) { diff --git a/drivers/base/core.c b/drivers/base/core.c index 7c0406e675e9..f29839382f81 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -149,6 +149,21 @@ void fwnode_links_purge(struct fwnode_handle *fwnode) fwnode_links_purge_consumers(fwnode); } +static void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode) +{ + struct fwnode_handle *child; + + /* Don't purge consumer links of an added child */ + if (fwnode->dev) + return; + + fwnode->flags |= FWNODE_FLAG_NOT_DEVICE; + fwnode_links_purge_consumers(fwnode); + + fwnode_for_each_available_child_node(fwnode, child) + fw_devlink_purge_absent_suppliers(child); +} + #ifdef CONFIG_SRCU static DEFINE_MUTEX(device_links_lock); DEFINE_STATIC_SRCU(device_links_srcu); @@ -245,7 +260,8 @@ int device_is_dependent(struct device *dev, void *target) return ret; list_for_each_entry(link, &dev->links.consumers, s_node) { - if (link->flags == (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + if ((link->flags & ~DL_FLAG_INFERRED) == + (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) continue; if (link->consumer == target) @@ -318,7 +334,8 @@ static int device_reorder_to_tail(struct device *dev, void *not_used) device_for_each_child(dev, NULL, device_reorder_to_tail); list_for_each_entry(link, &dev->links.consumers, s_node) { - if (link->flags == (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + if ((link->flags & ~DL_FLAG_INFERRED) == + (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) continue; device_reorder_to_tail(link->consumer, NULL); } @@ -566,7 +583,8 @@ postcore_initcall(devlink_class_init); #define DL_MANAGED_LINK_FLAGS (DL_FLAG_AUTOREMOVE_CONSUMER | \ DL_FLAG_AUTOREMOVE_SUPPLIER | \ DL_FLAG_AUTOPROBE_CONSUMER | \ - DL_FLAG_SYNC_STATE_ONLY) + DL_FLAG_SYNC_STATE_ONLY | \ + DL_FLAG_INFERRED) #define DL_ADD_VALID_FLAGS (DL_MANAGED_LINK_FLAGS | DL_FLAG_STATELESS | \ DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE) @@ -635,7 +653,7 @@ struct device_link *device_link_add(struct device *consumer, if (!consumer || !supplier || flags & ~DL_ADD_VALID_FLAGS || (flags & DL_FLAG_STATELESS && flags & DL_MANAGED_LINK_FLAGS) || (flags & DL_FLAG_SYNC_STATE_ONLY && - flags != DL_FLAG_SYNC_STATE_ONLY) || + (flags & ~DL_FLAG_INFERRED) != DL_FLAG_SYNC_STATE_ONLY) || (flags & DL_FLAG_AUTOPROBE_CONSUMER && flags & (DL_FLAG_AUTOREMOVE_CONSUMER | DL_FLAG_AUTOREMOVE_SUPPLIER))) @@ -691,6 +709,10 @@ struct device_link *device_link_add(struct device *consumer, if (link->consumer != consumer) continue; + if (link->flags & DL_FLAG_INFERRED && + !(flags & DL_FLAG_INFERRED)) + link->flags &= ~DL_FLAG_INFERRED; + if (flags & DL_FLAG_PM_RUNTIME) { if (!(link->flags & DL_FLAG_PM_RUNTIME)) { pm_runtime_new_link(consumer); @@ -950,6 +972,10 @@ int device_links_check_suppliers(struct device *dev) mutex_lock(&fwnode_link_lock); if (dev->fwnode && !list_empty(&dev->fwnode->suppliers) && !fw_devlink_is_permissive()) { + dev_dbg(dev, "probe deferral - wait for supplier %pfwP\n", + list_first_entry(&dev->fwnode->suppliers, + struct fwnode_link, + c_hook)->supplier); mutex_unlock(&fwnode_link_lock); return -EPROBE_DEFER; } @@ -964,6 +990,8 @@ int device_links_check_suppliers(struct device *dev) if (link->status != DL_STATE_AVAILABLE && !(link->flags & DL_FLAG_SYNC_STATE_ONLY)) { device_links_missing_supplier(dev); + dev_dbg(dev, "probe deferral - supplier %s not ready\n", + dev_name(link->supplier)); ret = -EPROBE_DEFER; break; } @@ -1142,12 +1170,22 @@ void device_links_driver_bound(struct device *dev) LIST_HEAD(sync_list); /* - * If a device probes successfully, it's expected to have created all + * If a device binds successfully, it's expected to have created all * the device links it needs to or make new device links as it needs - * them. So, it no longer needs to wait on any suppliers. + * them. So, fw_devlink no longer needs to create device links to any + * of the device's suppliers. + * + * Also, if a child firmware node of this bound device is not added as + * a device by now, assume it is never going to be added and make sure + * other devices don't defer probe indefinitely by waiting for such a + * child device. */ - if (dev->fwnode && dev->fwnode->dev == dev) + if (dev->fwnode && dev->fwnode->dev == dev) { + struct fwnode_handle *child; fwnode_links_purge_suppliers(dev->fwnode); + fwnode_for_each_available_child_node(dev->fwnode, child) + fw_devlink_purge_absent_suppliers(child); + } device_remove_file(dev, &dev_attr_waiting_for_supplier); device_links_write_lock(); @@ -1458,7 +1496,14 @@ static void device_links_purge(struct device *dev) device_links_write_unlock(); } -static u32 fw_devlink_flags = DL_FLAG_SYNC_STATE_ONLY; +#define FW_DEVLINK_FLAGS_PERMISSIVE (DL_FLAG_INFERRED | \ + DL_FLAG_SYNC_STATE_ONLY) +#define FW_DEVLINK_FLAGS_ON (DL_FLAG_INFERRED | \ + DL_FLAG_AUTOPROBE_CONSUMER) +#define FW_DEVLINK_FLAGS_RPM (FW_DEVLINK_FLAGS_ON | \ + DL_FLAG_PM_RUNTIME) + +static u32 fw_devlink_flags = FW_DEVLINK_FLAGS_PERMISSIVE; static int __init fw_devlink_setup(char *arg) { if (!arg) @@ -1467,17 +1512,23 @@ static int __init fw_devlink_setup(char *arg) if (strcmp(arg, "off") == 0) { fw_devlink_flags = 0; } else if (strcmp(arg, "permissive") == 0) { - fw_devlink_flags = DL_FLAG_SYNC_STATE_ONLY; + fw_devlink_flags = FW_DEVLINK_FLAGS_PERMISSIVE; } else if (strcmp(arg, "on") == 0) { - fw_devlink_flags = DL_FLAG_AUTOPROBE_CONSUMER; + fw_devlink_flags = FW_DEVLINK_FLAGS_ON; } else if (strcmp(arg, "rpm") == 0) { - fw_devlink_flags = DL_FLAG_AUTOPROBE_CONSUMER | - DL_FLAG_PM_RUNTIME; + fw_devlink_flags = FW_DEVLINK_FLAGS_RPM; } return 0; } early_param("fw_devlink", fw_devlink_setup); +static bool fw_devlink_strict; +static int __init fw_devlink_strict_setup(char *arg) +{ + return strtobool(arg, &fw_devlink_strict); +} +early_param("fw_devlink.strict", fw_devlink_strict_setup); + u32 fw_devlink_get_flags(void) { return fw_devlink_flags; @@ -1485,7 +1536,12 @@ u32 fw_devlink_get_flags(void) static bool fw_devlink_is_permissive(void) { - return fw_devlink_flags == DL_FLAG_SYNC_STATE_ONLY; + return fw_devlink_flags == FW_DEVLINK_FLAGS_PERMISSIVE; +} + +bool fw_devlink_is_strict(void) +{ + return fw_devlink_strict && !fw_devlink_is_permissive(); } static void fw_devlink_parse_fwnode(struct fwnode_handle *fwnode) @@ -1507,6 +1563,53 @@ static void fw_devlink_parse_fwtree(struct fwnode_handle *fwnode) fw_devlink_parse_fwtree(child); } +/** + * fw_devlink_relax_cycle - Convert cyclic links to SYNC_STATE_ONLY links + * @con: Device to check dependencies for. + * @sup: Device to check against. + * + * Check if @sup depends on @con or any device dependent on it (its child or + * its consumer etc). When such a cyclic dependency is found, convert all + * device links created solely by fw_devlink into SYNC_STATE_ONLY device links. + * This is the equivalent of doing fw_devlink=permissive just between the + * devices in the cycle. We need to do this because, at this point, fw_devlink + * can't tell which of these dependencies is not a real dependency. + * + * Return 1 if a cycle is found. Otherwise, return 0. + */ +static int fw_devlink_relax_cycle(struct device *con, void *sup) +{ + struct device_link *link; + int ret; + + if (con == sup) + return 1; + + ret = device_for_each_child(con, sup, fw_devlink_relax_cycle); + if (ret) + return ret; + + list_for_each_entry(link, &con->links.consumers, s_node) { + if ((link->flags & ~DL_FLAG_INFERRED) == + (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + continue; + + if (!fw_devlink_relax_cycle(link->consumer, sup)) + continue; + + ret = 1; + + if (!(link->flags & DL_FLAG_INFERRED)) + continue; + + pm_runtime_drop_link(link); + link->flags = DL_FLAG_MANAGED | FW_DEVLINK_FLAGS_PERMISSIVE; + dev_dbg(link->consumer, "Relaxing link with %s\n", + dev_name(link->supplier)); + } + return ret; +} + /** * fw_devlink_create_devlink - Create a device link from a consumer to fwnode * @con - Consumer device for the device link @@ -1534,16 +1637,40 @@ static int fw_devlink_create_devlink(struct device *con, sup_dev = get_dev_from_fwnode(sup_handle); if (sup_dev) { + /* + * If it's one of those drivers that don't actually bind to + * their device using driver core, then don't wait on this + * supplier device indefinitely. + */ + if (sup_dev->links.status == DL_DEV_NO_DRIVER && + sup_handle->flags & FWNODE_FLAG_INITIALIZED) { + ret = -EINVAL; + goto out; + } + /* * If this fails, it is due to cycles in device links. Just * give up on this link and treat it as invalid. */ - if (!device_link_add(con, sup_dev, flags)) + if (!device_link_add(con, sup_dev, flags) && + !(flags & DL_FLAG_SYNC_STATE_ONLY)) { + dev_info(con, "Fixing up cyclic dependency with %s\n", + dev_name(sup_dev)); + device_links_write_lock(); + fw_devlink_relax_cycle(con, sup_dev); + device_links_write_unlock(); + device_link_add(con, sup_dev, + FW_DEVLINK_FLAGS_PERMISSIVE); ret = -EINVAL; + } goto out; } + /* Supplier that's already initialized without a struct device. */ + if (sup_handle->flags & FWNODE_FLAG_INITIALIZED) + return -EINVAL; + /* * DL_FLAG_SYNC_STATE_ONLY doesn't block probing and supports * cycles. So cycle detection isn't necessary and shouldn't be @@ -1632,7 +1759,7 @@ static void __fw_devlink_link_to_consumers(struct device *dev) con_dev = NULL; } else { own_link = false; - dl_flags = DL_FLAG_SYNC_STATE_ONLY; + dl_flags = FW_DEVLINK_FLAGS_PERMISSIVE; } } @@ -1687,7 +1814,7 @@ static void __fw_devlink_link_to_suppliers(struct device *dev, if (own_link) dl_flags = fw_devlink_get_flags(); else - dl_flags = DL_FLAG_SYNC_STATE_ONLY; + dl_flags = FW_DEVLINK_FLAGS_PERMISSIVE; list_for_each_entry_safe(link, tmp, &fwnode->suppliers, c_hook) { int ret; diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c index eac184e6d657..653c8c6ac7a7 100644 --- a/drivers/base/devtmpfs.c +++ b/drivers/base/devtmpfs.c @@ -162,7 +162,7 @@ static int dev_mkdir(const char *name, umode_t mode) if (IS_ERR(dentry)) return PTR_ERR(dentry); - err = vfs_mkdir(d_inode(path.dentry), dentry, mode); + err = vfs_mkdir(&init_user_ns, d_inode(path.dentry), dentry, mode); if (!err) /* mark as kernel-created inode */ d_inode(dentry)->i_private = &thread; @@ -212,7 +212,8 @@ static int handle_create(const char *nodename, umode_t mode, kuid_t uid, if (IS_ERR(dentry)) return PTR_ERR(dentry); - err = vfs_mknod(d_inode(path.dentry), dentry, mode, dev->devt); + err = vfs_mknod(&init_user_ns, d_inode(path.dentry), dentry, mode, + dev->devt); if (!err) { struct iattr newattrs; @@ -221,7 +222,7 @@ static int handle_create(const char *nodename, umode_t mode, kuid_t uid, newattrs.ia_gid = gid; newattrs.ia_valid = ATTR_MODE|ATTR_UID|ATTR_GID; inode_lock(d_inode(dentry)); - notify_change(dentry, &newattrs, NULL); + notify_change(&init_user_ns, dentry, &newattrs, NULL); inode_unlock(d_inode(dentry)); /* mark as kernel-created inode */ @@ -242,7 +243,8 @@ static int dev_rmdir(const char *name) return PTR_ERR(dentry); if (d_really_is_positive(dentry)) { if (d_inode(dentry)->i_private == &thread) - err = vfs_rmdir(d_inode(parent.dentry), dentry); + err = vfs_rmdir(&init_user_ns, d_inode(parent.dentry), + dentry); else err = -EPERM; } else { @@ -328,9 +330,10 @@ static int handle_remove(const char *nodename, struct device *dev) newattrs.ia_valid = ATTR_UID|ATTR_GID|ATTR_MODE; inode_lock(d_inode(dentry)); - notify_change(dentry, &newattrs, NULL); + notify_change(&init_user_ns, dentry, &newattrs, NULL); inode_unlock(d_inode(dentry)); - err = vfs_unlink(d_inode(parent.dentry), dentry, NULL); + err = vfs_unlink(&init_user_ns, d_inode(parent.dentry), + dentry, NULL); if (!err || err == -ENOENT) deleted = 1; } diff --git a/drivers/base/init.c b/drivers/base/init.c index 908e6520e804..a9f57c22fb9e 100644 --- a/drivers/base/init.c +++ b/drivers/base/init.c @@ -32,6 +32,7 @@ void __init driver_init(void) */ of_core_init(); platform_bus_init(); + auxiliary_bus_init(); cpu_dev_init(); memory_dev_init(); container_dev_init(); diff --git a/drivers/base/memory.c b/drivers/base/memory.c index eef4ffb6122c..f35298425575 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -35,7 +35,7 @@ static const char *const online_type_to_str[] = { [MMOP_ONLINE_MOVABLE] = "online_movable", }; -int memhp_online_type_from_str(const char *str) +int mhp_online_type_from_str(const char *str) { int i; @@ -253,7 +253,7 @@ static int memory_subsys_offline(struct device *dev) static ssize_t state_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - const int online_type = memhp_online_type_from_str(buf); + const int online_type = mhp_online_type_from_str(buf); struct memory_block *mem = to_memory_block(dev); int ret; @@ -290,20 +290,20 @@ static ssize_t state_store(struct device *dev, struct device_attribute *attr, } /* - * phys_device is a bad name for this. What I really want - * is a way to differentiate between memory ranges that - * are part of physical devices that constitute - * a complete removable unit or fru. - * i.e. do these ranges belong to the same physical device, - * s.t. if I offline all of these sections I can then - * remove the physical device? + * Legacy interface that we cannot remove: s390x exposes the storage increment + * covered by a memory block, allowing for identifying which memory blocks + * comprise a storage increment. Since a memory block spans complete + * storage increments nowadays, this interface is basically unused. Other + * archs never exposed != 0. */ static ssize_t phys_device_show(struct device *dev, struct device_attribute *attr, char *buf) { struct memory_block *mem = to_memory_block(dev); + unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); - return sysfs_emit(buf, "%d\n", mem->phys_device); + return sysfs_emit(buf, "%d\n", + arch_get_memory_phys_device(start_pfn)); } #ifdef CONFIG_MEMORY_HOTREMOVE @@ -387,19 +387,19 @@ static ssize_t auto_online_blocks_show(struct device *dev, struct device_attribute *attr, char *buf) { return sysfs_emit(buf, "%s\n", - online_type_to_str[memhp_default_online_type]); + online_type_to_str[mhp_default_online_type]); } static ssize_t auto_online_blocks_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - const int online_type = memhp_online_type_from_str(buf); + const int online_type = mhp_online_type_from_str(buf); if (online_type < 0) return -EINVAL; - memhp_default_online_type = online_type; + mhp_default_online_type = online_type; return count; } @@ -488,11 +488,7 @@ static DEVICE_ATTR_WO(soft_offline_page); static DEVICE_ATTR_WO(hard_offline_page); #endif -/* - * Note that phys_device is optional. It is here to allow for - * differentiation between which *physical* devices each - * section belongs to... - */ +/* See phys_device_show(). */ int __weak arch_get_memory_phys_device(unsigned long start_pfn) { return 0; @@ -574,7 +570,6 @@ int register_memory(struct memory_block *memory) static int init_memory_block(unsigned long block_id, unsigned long state) { struct memory_block *mem; - unsigned long start_pfn; int ret = 0; mem = find_memory_block_by_id(block_id); @@ -588,8 +583,6 @@ static int init_memory_block(unsigned long block_id, unsigned long state) mem->start_section_nr = block_id * sections_per_block; mem->state = state; - start_pfn = section_nr_to_pfn(mem->start_section_nr); - mem->phys_device = arch_get_memory_phys_device(start_pfn); mem->nid = NUMA_NO_NODE; ret = register_memory(mem); diff --git a/drivers/base/node.c b/drivers/base/node.c index 04f71c7bc3f8..f449dbb2c746 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -372,14 +372,19 @@ static ssize_t node_read_meminfo(struct device *dev, struct pglist_data *pgdat = NODE_DATA(nid); struct sysinfo i; unsigned long sreclaimable, sunreclaimable; + unsigned long swapcached = 0; si_meminfo_node(&i, nid); sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B); sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B); +#ifdef CONFIG_SWAP + swapcached = node_page_state_pages(pgdat, NR_SWAPCACHE); +#endif len = sysfs_emit_at(buf, len, "Node %d MemTotal: %8lu kB\n" "Node %d MemFree: %8lu kB\n" "Node %d MemUsed: %8lu kB\n" + "Node %d SwapCached: %8lu kB\n" "Node %d Active: %8lu kB\n" "Node %d Inactive: %8lu kB\n" "Node %d Active(anon): %8lu kB\n" @@ -391,6 +396,7 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(i.totalram), nid, K(i.freeram), nid, K(i.totalram - i.freeram), + nid, K(swapcached), nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) + node_page_state(pgdat, NR_ACTIVE_FILE)), nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) + @@ -461,16 +467,11 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(sunreclaimable) #ifdef CONFIG_TRANSPARENT_HUGEPAGE , - nid, K(node_page_state(pgdat, NR_ANON_THPS) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_SHMEM_THPS) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_FILE_THPS) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) * - HPAGE_PMD_NR) + nid, K(node_page_state(pgdat, NR_ANON_THPS)), + nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), + nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), + nid, K(node_page_state(pgdat, NR_FILE_THPS)), + nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED)) #endif ); len += hugetlb_report_node_meminfo(buf, len, nid); @@ -519,10 +520,14 @@ static ssize_t node_read_vmstat(struct device *dev, sum_zone_numa_state(nid, i)); #endif - for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) - len += sysfs_emit_at(buf, len, "%s %lu\n", - node_stat_name(i), - node_page_state_pages(pgdat, i)); + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { + unsigned long pages = node_page_state_pages(pgdat, i); + + if (vmstat_item_print_in_thp(i)) + pages /= HPAGE_PMD_NR; + len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i), + pages); + } return len; } diff --git a/drivers/base/platform.c b/drivers/base/platform.c index 8456d8384ac8..6e1f8e0b661c 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -573,7 +573,7 @@ static void platform_device_release(struct device *dev) struct platform_object *pa = container_of(dev, struct platform_object, pdev.dev); - of_device_node_put(&pa->pdev.dev); + of_node_put(pa->pdev.dev.of_node); kfree(pa->pdev.dev.platform_data); kfree(pa->pdev.mfd_cell); kfree(pa->pdev.resource); @@ -1463,13 +1463,16 @@ static int platform_remove(struct device *_dev) { struct platform_driver *drv = to_platform_driver(_dev->driver); struct platform_device *dev = to_platform_device(_dev); - int ret = 0; - if (drv->remove) - ret = drv->remove(dev); + if (drv->remove) { + int ret = drv->remove(dev); + + if (ret) + dev_warn(_dev, "remove callback returned a non-zero value. This will be ignored.\n"); + } dev_pm_domain_detach(_dev, true); - return ret; + return 0; } static void platform_shutdown(struct device *_dev) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index aaf6c83b5cf6..78c310d3179d 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -2196,6 +2196,7 @@ static int genpd_add_provider(struct device_node *np, genpd_xlate_t xlate, cp->node = of_node_get(np); cp->data = data; cp->xlate = xlate; + fwnode_dev_initialized(&np->fwnode, true); mutex_lock(&of_genpd_mutex); list_add(&cp->link, &of_genpd_providers); @@ -2385,6 +2386,7 @@ void of_genpd_del_provider(struct device_node *np) } } + fwnode_dev_initialized(&cp->node->fwnode, false); list_del(&cp->link); of_node_put(cp->node); kfree(cp); diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c index 7f4b3b62492c..f2469d3435ca 100644 --- a/drivers/base/regmap/regcache.c +++ b/drivers/base/regmap/regcache.c @@ -68,7 +68,7 @@ static int regcache_hw_init(struct regmap *map) map->cache_bypass = cache_bypass; if (ret == 0) { map->reg_defaults_raw = tmp_buf; - map->cache_free = 1; + map->cache_free = true; } else { kfree(tmp_buf); } diff --git a/drivers/base/regmap/regmap-sdw-mbq.c b/drivers/base/regmap/regmap-sdw-mbq.c index 8ce30650b97c..fe3ac26b66ad 100644 --- a/drivers/base/regmap/regmap-sdw-mbq.c +++ b/drivers/base/regmap/regmap-sdw-mbq.c @@ -15,11 +15,11 @@ static int regmap_sdw_mbq_write(void *context, unsigned int reg, unsigned int va struct sdw_slave *slave = dev_to_sdw_dev(dev); int ret; - ret = sdw_write(slave, SDW_SDCA_MBQ_CTL(reg), (val >> 8) & 0xff); + ret = sdw_write_no_pm(slave, SDW_SDCA_MBQ_CTL(reg), (val >> 8) & 0xff); if (ret < 0) return ret; - return sdw_write(slave, reg, val & 0xff); + return sdw_write_no_pm(slave, reg, val & 0xff); } static int regmap_sdw_mbq_read(void *context, unsigned int reg, unsigned int *val) @@ -29,11 +29,11 @@ static int regmap_sdw_mbq_read(void *context, unsigned int reg, unsigned int *va int read0; int read1; - read0 = sdw_read(slave, reg); + read0 = sdw_read_no_pm(slave, reg); if (read0 < 0) return read0; - read1 = sdw_read(slave, SDW_SDCA_MBQ_CTL(reg)); + read1 = sdw_read_no_pm(slave, SDW_SDCA_MBQ_CTL(reg)); if (read1 < 0) return read1; @@ -98,4 +98,4 @@ struct regmap *__devm_regmap_init_sdw_mbq(struct sdw_slave *sdw, EXPORT_SYMBOL_GPL(__devm_regmap_init_sdw_mbq); MODULE_DESCRIPTION("Regmap SoundWire MBQ Module"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap-sdw.c b/drivers/base/regmap/regmap-sdw.c index c83be26434e7..966de8a136d9 100644 --- a/drivers/base/regmap/regmap-sdw.c +++ b/drivers/base/regmap/regmap-sdw.c @@ -13,7 +13,7 @@ static int regmap_sdw_write(void *context, unsigned int reg, unsigned int val) struct device *dev = context; struct sdw_slave *slave = dev_to_sdw_dev(dev); - return sdw_write(slave, reg, val); + return sdw_write_no_pm(slave, reg, val); } static int regmap_sdw_read(void *context, unsigned int reg, unsigned int *val) @@ -22,7 +22,7 @@ static int regmap_sdw_read(void *context, unsigned int reg, unsigned int *val) struct sdw_slave *slave = dev_to_sdw_dev(dev); int read; - read = sdw_read(slave, reg); + read = sdw_read_no_pm(slave, reg); if (read < 0) return read; diff --git a/drivers/base/test/Makefile b/drivers/base/test/Makefile index 3ca56367c84b..2f15fae8625f 100644 --- a/drivers/base/test/Makefile +++ b/drivers/base/test/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_TEST_ASYNC_DRIVER_PROBE) += test_async_driver_probe.o obj-$(CONFIG_KUNIT_DRIVER_PE_TEST) += property-entry-test.o +CFLAGS_REMOVE_property-entry-test.o += -fplugin-arg-structleak_plugin-byref -fplugin-arg-structleak_plugin-byref-all diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index 2779e85795a7..fd236158f32d 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -66,6 +66,12 @@ config AMIGA_Z2RAM To compile this driver as a module, choose M here: the module will be called z2ram. +config N64CART + bool "N64 cart support" + depends on MACH_NINTENDO64 + help + Support for the N64 cart. + config CDROM tristate select BLK_SCSI_REQUEST diff --git a/drivers/block/Makefile b/drivers/block/Makefile index b501b8728fb9..e3e3f1c79a82 100644 --- a/drivers/block/Makefile +++ b/drivers/block/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_PS3_DISK) += ps3disk.o obj-$(CONFIG_PS3_VRAM) += ps3vram.o obj-$(CONFIG_ATARI_FLOPPY) += ataflop.o obj-$(CONFIG_AMIGA_Z2RAM) += z2ram.o +obj-$(CONFIG_N64CART) += n64cart.o obj-$(CONFIG_BLK_DEV_RAM) += brd.o obj-$(CONFIG_BLK_DEV_LOOP) += loop.o obj-$(CONFIG_XILINX_SYSACE) += xsysace.o diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 578fc034db3f..a370cde3ddd4 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -663,7 +663,7 @@ static inline int is_loop_device(struct file *file) { struct inode *i = file->f_mapping->host; - return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR; + return i && S_ISBLK(i->i_mode) && imajor(i) == LOOP_MAJOR; } static int loop_validate_file(struct file *file, struct block_device *bdev) @@ -1212,6 +1212,9 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) goto out_unlock; } + if (test_bit(QUEUE_FLAG_WC, &lo->lo_queue->queue_flags)) + blk_queue_write_cache(lo->lo_queue, false, false); + /* freeze request queue during the transition */ blk_mq_freeze_queue(lo->lo_queue); diff --git a/drivers/block/n64cart.c b/drivers/block/n64cart.c new file mode 100644 index 000000000000..47bdf324e962 --- /dev/null +++ b/drivers/block/n64cart.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for the N64 cart. + * + * Copyright (c) 2021 Lauri Kasanen + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include +#include +#include +#include +#include +#include + +enum { + PI_DRAM_REG = 0, + PI_CART_REG, + PI_READ_REG, + PI_WRITE_REG, + PI_STATUS_REG, +}; + +#define PI_STATUS_DMA_BUSY (1 << 0) +#define PI_STATUS_IO_BUSY (1 << 1) + +#define CART_DOMAIN 0x10000000 +#define CART_MAX 0x1FFFFFFF + +#define MIN_ALIGNMENT 8 + +static u32 __iomem *reg_base; + +static unsigned int start; +module_param(start, uint, 0); +MODULE_PARM_DESC(start, "Start address of the cart block data"); + +static unsigned int size; +module_param(size, uint, 0); +MODULE_PARM_DESC(size, "Size of the cart block data, in bytes"); + +static void n64cart_write_reg(const u8 reg, const u32 value) +{ + writel(value, reg_base + reg); +} + +static u32 n64cart_read_reg(const u8 reg) +{ + return readl(reg_base + reg); +} + +static void n64cart_wait_dma(void) +{ + while (n64cart_read_reg(PI_STATUS_REG) & + (PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY)) + cpu_relax(); +} + +/* + * Process a single bvec of a bio. + */ +static bool n64cart_do_bvec(struct device *dev, struct bio_vec *bv, u32 pos) +{ + dma_addr_t dma_addr; + const u32 bstart = pos + start; + + /* Alignment check */ + WARN_ON_ONCE((bv->bv_offset & (MIN_ALIGNMENT - 1)) || + (bv->bv_len & (MIN_ALIGNMENT - 1))); + + dma_addr = dma_map_bvec(dev, bv, DMA_FROM_DEVICE, 0); + if (dma_mapping_error(dev, dma_addr)) + return false; + + n64cart_wait_dma(); + + n64cart_write_reg(PI_DRAM_REG, dma_addr + bv->bv_offset); + n64cart_write_reg(PI_CART_REG, (bstart | CART_DOMAIN) & CART_MAX); + n64cart_write_reg(PI_WRITE_REG, bv->bv_len - 1); + + n64cart_wait_dma(); + + dma_unmap_page(dev, dma_addr, bv->bv_len, DMA_FROM_DEVICE); + return true; +} + +static blk_qc_t n64cart_submit_bio(struct bio *bio) +{ + struct bio_vec bvec; + struct bvec_iter iter; + struct device *dev = bio->bi_disk->private_data; + u32 pos = bio->bi_iter.bi_sector << SECTOR_SHIFT; + + bio_for_each_segment(bvec, bio, iter) { + if (!n64cart_do_bvec(dev, &bvec, pos)) + goto io_error; + pos += bvec.bv_len; + } + + bio_endio(bio); + return BLK_QC_T_NONE; +io_error: + bio_io_error(bio); + return BLK_QC_T_NONE; +} + +static const struct block_device_operations n64cart_fops = { + .owner = THIS_MODULE, + .submit_bio = n64cart_submit_bio, +}; + +/* + * The target device is embedded and RAM-constrained. We save RAM + * by initializing in __init code that gets dropped late in boot. + * For the same reason there is no module or unloading support. + */ +static int __init n64cart_probe(struct platform_device *pdev) +{ + struct gendisk *disk; + + if (!start || !size) { + pr_err("start or size not specified\n"); + return -ENODEV; + } + + if (size & 4095) { + pr_err("size must be a multiple of 4K\n"); + return -ENODEV; + } + + reg_base = devm_platform_ioremap_resource(pdev, 0); + if (!reg_base) + return -EINVAL; + + disk = alloc_disk(0); + if (!disk) + return -ENOMEM; + + disk->queue = blk_alloc_queue(NUMA_NO_NODE); + if (!disk->queue) + return -ENOMEM; + + disk->first_minor = 0; + disk->flags = GENHD_FL_NO_PART_SCAN | GENHD_FL_EXT_DEVT; + disk->fops = &n64cart_fops; + disk->private_data = &pdev->dev; + strcpy(disk->disk_name, "n64cart"); + + set_capacity(disk, size >> SECTOR_SHIFT); + set_disk_ro(disk, 1); + + blk_queue_flag_set(QUEUE_FLAG_NONROT, disk->queue); + blk_queue_physical_block_size(disk->queue, 4096); + blk_queue_logical_block_size(disk->queue, 4096); + + add_disk(disk); + + pr_info("n64cart: %u kb disk\n", size / 1024); + + return 0; +} + +static struct platform_driver n64cart_driver = { + .driver = { + .name = "n64cart", + }, +}; + +static int __init n64cart_init(void) +{ + return platform_driver_probe(&n64cart_driver, n64cart_probe); +} + +module_init(n64cart_init); + +MODULE_AUTHOR("Lauri Kasanen "); +MODULE_DESCRIPTION("Driver for the N64 cart"); +MODULE_LICENSE("GPL"); diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 8b9622eb0a21..4ff71b579cfc 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -78,8 +78,7 @@ struct link_dead_args { #define NBD_RT_HAS_PID_FILE 3 #define NBD_RT_HAS_CONFIG_REF 4 #define NBD_RT_BOUND 5 -#define NBD_RT_DESTROY_ON_DISCONNECT 6 -#define NBD_RT_DISCONNECT_ON_CLOSE 7 +#define NBD_RT_DISCONNECT_ON_CLOSE 6 #define NBD_DESTROY_ON_DISCONNECT 0 #define NBD_DISCONNECT_REQUESTED 1 @@ -1904,12 +1903,21 @@ again: if (info->attrs[NBD_ATTR_CLIENT_FLAGS]) { u64 flags = nla_get_u64(info->attrs[NBD_ATTR_CLIENT_FLAGS]); if (flags & NBD_CFLAG_DESTROY_ON_DISCONNECT) { - set_bit(NBD_RT_DESTROY_ON_DISCONNECT, - &config->runtime_flags); - set_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); - put_dev = true; + /* + * We have 1 ref to keep the device around, and then 1 + * ref for our current operation here, which will be + * inherited by the config. If we already have + * DESTROY_ON_DISCONNECT set then we know we don't have + * that extra ref already held so we don't need the + * put_dev. + */ + if (!test_and_set_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) + put_dev = true; } else { - clear_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); + if (test_and_clear_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) + refcount_inc(&nbd->refs); } if (flags & NBD_CFLAG_DISCONNECT_ON_CLOSE) { set_bit(NBD_RT_DISCONNECT_ON_CLOSE, @@ -2080,15 +2088,13 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info) if (info->attrs[NBD_ATTR_CLIENT_FLAGS]) { u64 flags = nla_get_u64(info->attrs[NBD_ATTR_CLIENT_FLAGS]); if (flags & NBD_CFLAG_DESTROY_ON_DISCONNECT) { - if (!test_and_set_bit(NBD_RT_DESTROY_ON_DISCONNECT, - &config->runtime_flags)) + if (!test_and_set_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) put_dev = true; - set_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); } else { - if (test_and_clear_bit(NBD_RT_DESTROY_ON_DISCONNECT, - &config->runtime_flags)) + if (test_and_clear_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) refcount_inc(&nbd->refs); - clear_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); } if (flags & NBD_CFLAG_DISCONNECT_ON_CLOSE) { diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index b0285db7cf4f..b9fa3ef5b57c 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -705,6 +705,7 @@ static int virtblk_probe(struct virtio_device *vdev) u32 v, blk_size, max_size, sg_elems, opt_io_size; u16 min_io_size; u8 physical_block_exp, alignment_offset; + unsigned int queue_depth; if (!vdev->config->get) { dev_err(&vdev->dev, "%s failure: config access disabled\n", @@ -756,16 +757,18 @@ static int virtblk_probe(struct virtio_device *vdev) } /* Default queue sizing is to fill the ring. */ - if (!virtblk_queue_depth) { - virtblk_queue_depth = vblk->vqs[0].vq->num_free; + if (likely(!virtblk_queue_depth)) { + queue_depth = vblk->vqs[0].vq->num_free; /* ... but without indirect descs, we use 2 descs per req */ if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) - virtblk_queue_depth /= 2; + queue_depth /= 2; + } else { + queue_depth = virtblk_queue_depth; } memset(&vblk->tag_set, 0, sizeof(vblk->tag_set)); vblk->tag_set.ops = &virtio_mq_ops; - vblk->tag_set.queue_depth = virtblk_queue_depth; + vblk->tag_set.queue_depth = queue_depth; vblk->tag_set.numa_node = NUMA_NO_NODE; vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; vblk->tag_set.cmd_size = diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index da16121140ca..1cdf09ff67b6 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c @@ -1326,9 +1326,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, pages[i]->page, seg[i].nsec << 9, seg[i].offset) == 0)) { - - int nr_iovecs = min_t(int, (nseg-i), BIO_MAX_PAGES); - bio = bio_alloc(GFP_KERNEL, nr_iovecs); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(nseg - i)); if (unlikely(bio == NULL)) goto fail_put_bio; diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index d7018543842e..a711a2e2a794 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1081,7 +1081,7 @@ static ssize_t mm_stat_show(struct device *dev, zram->limit_pages << PAGE_SHIFT, max_used << PAGE_SHIFT, (u64)atomic64_read(&zram->stats.same_pages), - pool_stats.pages_compacted, + atomic_long_read(&pool_stats.pages_compacted), (u64)atomic64_read(&zram->stats.huge_pages), (u64)atomic64_read(&zram->stats.huge_pages_since)); up_read(&zram->init_lock); diff --git a/drivers/bus/fsl-mc/Kconfig b/drivers/bus/fsl-mc/Kconfig index c23c77c9b705..b1fd55901c50 100644 --- a/drivers/bus/fsl-mc/Kconfig +++ b/drivers/bus/fsl-mc/Kconfig @@ -14,3 +14,10 @@ config FSL_MC_BUS architecture. The fsl-mc bus driver handles discovery of DPAA2 objects (which are represented as Linux devices) and binding objects to drivers. + +config FSL_MC_UAPI_SUPPORT + bool "Management Complex (MC) userspace support" + depends on FSL_MC_BUS + help + Provides userspace support for interrogating, creating, destroying or + configuring DPAA2 objects exported by the Management Complex. diff --git a/drivers/bus/fsl-mc/Makefile b/drivers/bus/fsl-mc/Makefile index 3c518c7e8374..4ae292a30e53 100644 --- a/drivers/bus/fsl-mc/Makefile +++ b/drivers/bus/fsl-mc/Makefile @@ -16,3 +16,6 @@ mc-bus-driver-objs := fsl-mc-bus.o \ fsl-mc-allocator.o \ fsl-mc-msi.o \ dpmcp.o + +# MC userspace support +obj-$(CONFIG_FSL_MC_UAPI_SUPPORT) += fsl-mc-uapi.o diff --git a/drivers/bus/fsl-mc/dprc-driver.c b/drivers/bus/fsl-mc/dprc-driver.c index 68488a7ad0d6..e3e2ae41c22b 100644 --- a/drivers/bus/fsl-mc/dprc-driver.c +++ b/drivers/bus/fsl-mc/dprc-driver.c @@ -237,8 +237,8 @@ static void dprc_add_new_devices(struct fsl_mc_device *mc_bus_dev, * populated before they can get allocation requests from probe callbacks * of the device drivers for the non-allocatable devices. */ -static int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, - bool alloc_interrupts) +int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, + bool alloc_interrupts) { int num_child_objects; int dprc_get_obj_failures; @@ -458,8 +458,9 @@ out: /* * Disable and clear interrupt for a given DPRC object */ -static int disable_dprc_irq(struct fsl_mc_device *mc_dev) +int disable_dprc_irq(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; struct fsl_mc_io *mc_io = mc_dev->mc_io; @@ -496,9 +497,18 @@ static int disable_dprc_irq(struct fsl_mc_device *mc_dev) return error; } + mc_bus->irq_enabled = 0; + return 0; } +int get_dprc_irq_state(struct fsl_mc_device *mc_dev) +{ + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); + + return mc_bus->irq_enabled; +} + static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev) { int error; @@ -525,8 +535,9 @@ static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev) return 0; } -static int enable_dprc_irq(struct fsl_mc_device *mc_dev) +int enable_dprc_irq(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; /* @@ -554,6 +565,8 @@ static int enable_dprc_irq(struct fsl_mc_device *mc_dev) return error; } + mc_bus->irq_enabled = 1; + return 0; } @@ -603,6 +616,7 @@ int dprc_setup(struct fsl_mc_device *mc_dev) struct irq_domain *mc_msi_domain; bool mc_io_created = false; bool msi_domain_set = false; + bool uapi_created = false; u16 major_ver, minor_ver; size_t region_size; int error; @@ -635,6 +649,11 @@ int dprc_setup(struct fsl_mc_device *mc_dev) return error; mc_io_created = true; + } else { + error = fsl_mc_uapi_create_device_file(mc_bus); + if (error < 0) + return -EPROBE_DEFER; + uapi_created = true; } mc_msi_domain = fsl_mc_find_msi_domain(&mc_dev->dev); @@ -692,6 +711,9 @@ error_cleanup_msi_domain: mc_dev->mc_io = NULL; } + if (uapi_created) + fsl_mc_uapi_remove_device_file(mc_bus); + return error; } EXPORT_SYMBOL_GPL(dprc_setup); @@ -763,6 +785,7 @@ static void dprc_teardown_irq(struct fsl_mc_device *mc_dev) int dprc_cleanup(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; /* this function should be called only for DPRCs, it @@ -793,6 +816,8 @@ int dprc_cleanup(struct fsl_mc_device *mc_dev) if (!fsl_mc_is_root_dprc(&mc_dev->dev)) { fsl_destroy_mc_io(mc_dev->mc_io); mc_dev->mc_io = NULL; + } else { + fsl_mc_uapi_remove_device_file(mc_bus); } return 0; diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 8af978bd0000..380ad1fdb745 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -41,7 +41,7 @@ struct fsl_mc { struct fsl_mc_device *root_mc_bus_dev; u8 num_translation_ranges; struct fsl_mc_addr_translation_range *translation_ranges; - void *fsl_mc_regs; + void __iomem *fsl_mc_regs; }; /** @@ -208,12 +208,108 @@ static struct attribute *fsl_mc_dev_attrs[] = { ATTRIBUTE_GROUPS(fsl_mc_dev); +static int scan_fsl_mc_bus(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + struct fsl_mc_bus *root_mc_bus; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + root_mc_bus = to_fsl_mc_bus(root_mc_dev); + mutex_lock(&root_mc_bus->scan_mutex); + dprc_scan_objects(root_mc_dev, NULL); + mutex_unlock(&root_mc_bus->scan_mutex); + +exit: + return 0; +} + +static ssize_t rescan_store(struct bus_type *bus, + const char *buf, size_t count) +{ + unsigned long val; + + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + + if (val) + bus_for_each_dev(bus, NULL, NULL, scan_fsl_mc_bus); + + return count; +} +static BUS_ATTR_WO(rescan); + +static int fsl_mc_bus_set_autorescan(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + unsigned long val; + char *buf = data; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + + if (val) + enable_dprc_irq(root_mc_dev); + else + disable_dprc_irq(root_mc_dev); + +exit: + return 0; +} + +static int fsl_mc_bus_get_autorescan(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + char *buf = data; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + + sprintf(buf, "%d\n", get_dprc_irq_state(root_mc_dev)); +exit: + return 0; +} + +static ssize_t autorescan_store(struct bus_type *bus, + const char *buf, size_t count) +{ + bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_set_autorescan); + + return count; +} + +static ssize_t autorescan_show(struct bus_type *bus, char *buf) +{ + bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_get_autorescan); + return strlen(buf); +} + +static BUS_ATTR_RW(autorescan); + +static struct attribute *fsl_mc_bus_attrs[] = { + &bus_attr_rescan.attr, + &bus_attr_autorescan.attr, + NULL, +}; + +ATTRIBUTE_GROUPS(fsl_mc_bus); + struct bus_type fsl_mc_bus_type = { .name = "fsl-mc", .match = fsl_mc_bus_match, .uevent = fsl_mc_bus_uevent, .dma_configure = fsl_mc_dma_configure, .dev_groups = fsl_mc_dev_groups, + .bus_groups = fsl_mc_bus_groups, }; EXPORT_SYMBOL_GPL(fsl_mc_bus_type); @@ -292,6 +388,11 @@ struct device_type fsl_mc_bus_dpdmai_type = { }; EXPORT_SYMBOL_GPL(fsl_mc_bus_dpdmai_type); +struct device_type fsl_mc_bus_dpdbg_type = { + .name = "fsl_mc_bus_dpdbg" +}; +EXPORT_SYMBOL_GPL(fsl_mc_bus_dpdbg_type); + static struct device_type *fsl_mc_get_device_type(const char *type) { static const struct { @@ -313,6 +414,7 @@ static struct device_type *fsl_mc_get_device_type(const char *type) { &fsl_mc_bus_dpaiop_type, "dpaiop" }, { &fsl_mc_bus_dpci_type, "dpci" }, { &fsl_mc_bus_dpdmai_type, "dpdmai" }, + { &fsl_mc_bus_dpdbg_type, "dpdbg" }, { NULL, NULL } }; int i; diff --git a/drivers/bus/fsl-mc/fsl-mc-private.h b/drivers/bus/fsl-mc/fsl-mc-private.h index c932387641fa..1958fa065360 100644 --- a/drivers/bus/fsl-mc/fsl-mc-private.h +++ b/drivers/bus/fsl-mc/fsl-mc-private.h @@ -10,6 +10,8 @@ #include #include +#include +#include /* * Data Path Management Complex (DPMNG) General API @@ -542,6 +544,22 @@ struct fsl_mc_resource_pool { struct fsl_mc_bus *mc_bus; }; +/** + * struct fsl_mc_uapi - information associated with a device file + * @misc: struct miscdevice linked to the root dprc + * @device: newly created device in /dev + * @mutex: mutex lock to serialize the open/release operations + * @local_instance_in_use: local MC I/O instance in use or not + * @static_mc_io: pointer to the static MC I/O object + */ +struct fsl_mc_uapi { + struct miscdevice misc; + struct device *device; + struct mutex mutex; /* serialize open/release operations */ + u32 local_instance_in_use; + struct fsl_mc_io *static_mc_io; +}; + /** * struct fsl_mc_bus - logical bus that corresponds to a physical DPRC * @mc_dev: fsl-mc device for the bus device itself. @@ -551,6 +569,7 @@ struct fsl_mc_resource_pool { * @irq_resources: Pointer to array of IRQ objects for the IRQ pool * @scan_mutex: Serializes bus scanning * @dprc_attr: DPRC attributes + * @uapi_misc: struct that abstracts the interaction with userspace */ struct fsl_mc_bus { struct fsl_mc_device mc_dev; @@ -558,6 +577,8 @@ struct fsl_mc_bus { struct fsl_mc_device_irq *irq_resources; struct mutex scan_mutex; /* serializes bus scanning */ struct dprc_attributes dprc_attr; + struct fsl_mc_uapi uapi_misc; + int irq_enabled; }; #define to_fsl_mc_bus(_mc_dev) \ @@ -574,6 +595,9 @@ int __init dprc_driver_init(void); void dprc_driver_exit(void); +int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, + bool alloc_interrupts); + int __init fsl_mc_allocator_driver_init(void); void fsl_mc_allocator_driver_exit(void); @@ -612,4 +636,29 @@ void fsl_mc_get_root_dprc(struct device *dev, struct fsl_mc_device *fsl_mc_device_lookup(struct fsl_mc_obj_desc *obj_desc, struct fsl_mc_device *mc_bus_dev); +u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd); + +#ifdef CONFIG_FSL_MC_UAPI_SUPPORT + +int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus); + +void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus); + +#else + +static inline int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus) +{ + return 0; +} + +static inline void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus) +{ +} + +#endif + +int disable_dprc_irq(struct fsl_mc_device *mc_dev); +int enable_dprc_irq(struct fsl_mc_device *mc_dev); +int get_dprc_irq_state(struct fsl_mc_device *mc_dev); + #endif /* _FSL_MC_PRIVATE_H_ */ diff --git a/drivers/bus/fsl-mc/fsl-mc-uapi.c b/drivers/bus/fsl-mc/fsl-mc-uapi.c new file mode 100644 index 000000000000..9c4c1395fcdb --- /dev/null +++ b/drivers/bus/fsl-mc/fsl-mc-uapi.c @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Management Complex (MC) userspace support + * + * Copyright 2021 NXP + * + */ + +#include +#include +#include +#include + +#include "fsl-mc-private.h" + +struct uapi_priv_data { + struct fsl_mc_uapi *uapi; + struct fsl_mc_io *mc_io; +}; + +struct fsl_mc_cmd_desc { + u16 cmdid_value; + u16 cmdid_mask; + int size; + bool token; + int flags; +}; + +#define FSL_MC_CHECK_MODULE_ID BIT(0) +#define FSL_MC_CAP_NET_ADMIN_NEEDED BIT(1) + +enum fsl_mc_cmd_index { + DPDBG_DUMP = 0, + DPDBG_SET, + DPRC_GET_CONTAINER_ID, + DPRC_CREATE_CONT, + DPRC_DESTROY_CONT, + DPRC_ASSIGN, + DPRC_UNASSIGN, + DPRC_GET_OBJ_COUNT, + DPRC_GET_OBJ, + DPRC_GET_RES_COUNT, + DPRC_GET_RES_IDS, + DPRC_SET_OBJ_LABEL, + DPRC_SET_LOCKED, + DPRC_CONNECT, + DPRC_DISCONNECT, + DPRC_GET_POOL, + DPRC_GET_POOL_COUNT, + DPRC_GET_CONNECTION, + DPCI_GET_LINK_STATE, + DPCI_GET_PEER_ATTR, + DPAIOP_GET_SL_VERSION, + DPAIOP_GET_STATE, + DPMNG_GET_VERSION, + DPSECI_GET_TX_QUEUE, + DPMAC_GET_COUNTER, + DPMAC_GET_MAC_ADDR, + DPNI_SET_PRIM_MAC, + DPNI_GET_PRIM_MAC, + DPNI_GET_STATISTICS, + DPNI_GET_LINK_STATE, + DPNI_GET_MAX_FRAME_LENGTH, + DPSW_GET_TAILDROP, + DPSW_SET_TAILDROP, + DPSW_IF_GET_COUNTER, + DPSW_IF_GET_MAX_FRAME_LENGTH, + DPDMUX_GET_COUNTER, + DPDMUX_IF_GET_MAX_FRAME_LENGTH, + GET_ATTR, + GET_IRQ_MASK, + GET_IRQ_STATUS, + CLOSE, + OPEN, + GET_API_VERSION, + DESTROY, + CREATE, +}; + +static struct fsl_mc_cmd_desc fsl_mc_accepted_cmds[] = { + [DPDBG_DUMP] = { + .cmdid_value = 0x1300, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 28, + }, + [DPDBG_SET] = { + .cmdid_value = 0x1400, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 28, + }, + [DPRC_GET_CONTAINER_ID] = { + .cmdid_value = 0x8300, + .cmdid_mask = 0xFFF0, + .token = false, + .size = 8, + }, + [DPRC_CREATE_CONT] = { + .cmdid_value = 0x1510, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_DESTROY_CONT] = { + .cmdid_value = 0x1520, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_ASSIGN] = { + .cmdid_value = 0x1570, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_UNASSIGN] = { + .cmdid_value = 0x1580, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_GET_OBJ_COUNT] = { + .cmdid_value = 0x1590, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + }, + [DPRC_GET_OBJ] = { + .cmdid_value = 0x15A0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + }, + [DPRC_GET_RES_COUNT] = { + .cmdid_value = 0x15B0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + }, + [DPRC_GET_RES_IDS] = { + .cmdid_value = 0x15C0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + }, + [DPRC_SET_OBJ_LABEL] = { + .cmdid_value = 0x1610, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 48, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_SET_LOCKED] = { + .cmdid_value = 0x16B0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_CONNECT] = { + .cmdid_value = 0x1670, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 56, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_DISCONNECT] = { + .cmdid_value = 0x1680, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_GET_POOL] = { + .cmdid_value = 0x1690, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + }, + [DPRC_GET_POOL_COUNT] = { + .cmdid_value = 0x16A0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPRC_GET_CONNECTION] = { + .cmdid_value = 0x16C0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + }, + + [DPCI_GET_LINK_STATE] = { + .cmdid_value = 0x0E10, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPCI_GET_PEER_ATTR] = { + .cmdid_value = 0x0E20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPAIOP_GET_SL_VERSION] = { + .cmdid_value = 0x2820, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPAIOP_GET_STATE] = { + .cmdid_value = 0x2830, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPMNG_GET_VERSION] = { + .cmdid_value = 0x8310, + .cmdid_mask = 0xFFF0, + .token = false, + .size = 8, + }, + [DPSECI_GET_TX_QUEUE] = { + .cmdid_value = 0x1970, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 14, + }, + [DPMAC_GET_COUNTER] = { + .cmdid_value = 0x0c40, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 9, + }, + [DPMAC_GET_MAC_ADDR] = { + .cmdid_value = 0x0c50, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_SET_PRIM_MAC] = { + .cmdid_value = 0x2240, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPNI_GET_PRIM_MAC] = { + .cmdid_value = 0x2250, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_GET_STATISTICS] = { + .cmdid_value = 0x25D0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [DPNI_GET_LINK_STATE] = { + .cmdid_value = 0x2150, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x2170, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPSW_GET_TAILDROP] = { + .cmdid_value = 0x0A80, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 14, + }, + [DPSW_SET_TAILDROP] = { + .cmdid_value = 0x0A90, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 24, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPSW_IF_GET_COUNTER] = { + .cmdid_value = 0x0340, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 11, + }, + [DPSW_IF_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x0450, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [DPDMUX_GET_COUNTER] = { + .cmdid_value = 0x0b20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 11, + }, + [DPDMUX_IF_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x0a20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [GET_ATTR] = { + .cmdid_value = 0x0040, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [GET_IRQ_MASK] = { + .cmdid_value = 0x0150, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 13, + }, + [GET_IRQ_STATUS] = { + .cmdid_value = 0x0160, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 13, + }, + [CLOSE] = { + .cmdid_value = 0x8000, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + + /* Common commands amongst all types of objects. Must be checked last. */ + [OPEN] = { + .cmdid_value = 0x8000, + .cmdid_mask = 0xFC00, + .token = false, + .size = 12, + .flags = FSL_MC_CHECK_MODULE_ID, + }, + [GET_API_VERSION] = { + .cmdid_value = 0xA000, + .cmdid_mask = 0xFC00, + .token = false, + .size = 8, + .flags = FSL_MC_CHECK_MODULE_ID, + }, + [DESTROY] = { + .cmdid_value = 0x9800, + .cmdid_mask = 0xFC00, + .token = true, + .size = 12, + .flags = FSL_MC_CHECK_MODULE_ID | FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [CREATE] = { + .cmdid_value = 0x9000, + .cmdid_mask = 0xFC00, + .token = true, + .size = 64, + .flags = FSL_MC_CHECK_MODULE_ID | FSL_MC_CAP_NET_ADMIN_NEEDED, + }, +}; + +#define FSL_MC_NUM_ACCEPTED_CMDS ARRAY_SIZE(fsl_mc_accepted_cmds) + +#define FSL_MC_MAX_MODULE_ID 0x10 + +static int fsl_mc_command_check(struct fsl_mc_device *mc_dev, + struct fsl_mc_command *mc_cmd) +{ + struct fsl_mc_cmd_desc *desc = NULL; + int mc_cmd_max_size, i; + bool token_provided; + u16 cmdid, module_id; + char *mc_cmd_end; + char sum = 0; + + /* Check if this is an accepted MC command */ + cmdid = mc_cmd_hdr_read_cmdid(mc_cmd); + for (i = 0; i < FSL_MC_NUM_ACCEPTED_CMDS; i++) { + desc = &fsl_mc_accepted_cmds[i]; + if ((cmdid & desc->cmdid_mask) == desc->cmdid_value) + break; + } + if (i == FSL_MC_NUM_ACCEPTED_CMDS) { + dev_err(&mc_dev->dev, "MC command 0x%04x: cmdid not accepted\n", cmdid); + return -EACCES; + } + + /* Check if the size of the command is honored. Anything beyond the + * last valid byte of the command should be zeroed. + */ + mc_cmd_max_size = sizeof(*mc_cmd); + mc_cmd_end = ((char *)mc_cmd) + desc->size; + for (i = desc->size; i < mc_cmd_max_size; i++) + sum |= *mc_cmd_end++; + if (sum) { + dev_err(&mc_dev->dev, "MC command 0x%04x: garbage beyond max size of %d bytes!\n", + cmdid, desc->size); + return -EACCES; + } + + /* Some MC commands request a token to be passed so that object + * identification is possible. Check if the token passed in the command + * is as expected. + */ + token_provided = mc_cmd_hdr_read_token(mc_cmd) ? true : false; + if (token_provided != desc->token) { + dev_err(&mc_dev->dev, "MC command 0x%04x: token 0x%04x is invalid!\n", + cmdid, mc_cmd_hdr_read_token(mc_cmd)); + return -EACCES; + } + + /* If needed, check if the module ID passed is valid */ + if (desc->flags & FSL_MC_CHECK_MODULE_ID) { + /* The module ID is represented by bits [4:9] from the cmdid */ + module_id = (cmdid & GENMASK(9, 4)) >> 4; + if (module_id == 0 || module_id > FSL_MC_MAX_MODULE_ID) { + dev_err(&mc_dev->dev, "MC command 0x%04x: unknown module ID 0x%x\n", + cmdid, module_id); + return -EACCES; + } + } + + /* Some commands alter how hardware resources are managed. For these + * commands, check for CAP_NET_ADMIN. + */ + if (desc->flags & FSL_MC_CAP_NET_ADMIN_NEEDED) { + if (!capable(CAP_NET_ADMIN)) { + dev_err(&mc_dev->dev, "MC command 0x%04x: needs CAP_NET_ADMIN!\n", + cmdid); + return -EPERM; + } + } + + return 0; +} + +static int fsl_mc_uapi_send_command(struct fsl_mc_device *mc_dev, unsigned long arg, + struct fsl_mc_io *mc_io) +{ + struct fsl_mc_command mc_cmd; + int error; + + error = copy_from_user(&mc_cmd, (void __user *)arg, sizeof(mc_cmd)); + if (error) + return -EFAULT; + + error = fsl_mc_command_check(mc_dev, &mc_cmd); + if (error) + return error; + + error = mc_send_command(mc_io, &mc_cmd); + if (error) + return error; + + error = copy_to_user((void __user *)arg, &mc_cmd, sizeof(mc_cmd)); + if (error) + return -EFAULT; + + return 0; +} + +static int fsl_mc_uapi_dev_open(struct inode *inode, struct file *filep) +{ + struct fsl_mc_device *root_mc_device; + struct uapi_priv_data *priv_data; + struct fsl_mc_io *dynamic_mc_io; + struct fsl_mc_uapi *mc_uapi; + struct fsl_mc_bus *mc_bus; + int error; + + priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL); + if (!priv_data) + return -ENOMEM; + + mc_uapi = container_of(filep->private_data, struct fsl_mc_uapi, misc); + mc_bus = container_of(mc_uapi, struct fsl_mc_bus, uapi_misc); + root_mc_device = &mc_bus->mc_dev; + + mutex_lock(&mc_uapi->mutex); + + if (!mc_uapi->local_instance_in_use) { + priv_data->mc_io = mc_uapi->static_mc_io; + mc_uapi->local_instance_in_use = 1; + } else { + error = fsl_mc_portal_allocate(root_mc_device, 0, + &dynamic_mc_io); + if (error) { + dev_dbg(&root_mc_device->dev, + "Could not allocate MC portal\n"); + goto error_portal_allocate; + } + + priv_data->mc_io = dynamic_mc_io; + } + priv_data->uapi = mc_uapi; + filep->private_data = priv_data; + + mutex_unlock(&mc_uapi->mutex); + + return 0; + +error_portal_allocate: + mutex_unlock(&mc_uapi->mutex); + kfree(priv_data); + + return error; +} + +static int fsl_mc_uapi_dev_release(struct inode *inode, struct file *filep) +{ + struct uapi_priv_data *priv_data; + struct fsl_mc_uapi *mc_uapi; + struct fsl_mc_io *mc_io; + + priv_data = filep->private_data; + mc_uapi = priv_data->uapi; + mc_io = priv_data->mc_io; + + mutex_lock(&mc_uapi->mutex); + + if (mc_io == mc_uapi->static_mc_io) + mc_uapi->local_instance_in_use = 0; + else + fsl_mc_portal_free(mc_io); + + kfree(filep->private_data); + filep->private_data = NULL; + + mutex_unlock(&mc_uapi->mutex); + + return 0; +} + +static long fsl_mc_uapi_dev_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg) +{ + struct uapi_priv_data *priv_data = file->private_data; + struct fsl_mc_device *root_mc_device; + struct fsl_mc_bus *mc_bus; + int error; + + mc_bus = container_of(priv_data->uapi, struct fsl_mc_bus, uapi_misc); + root_mc_device = &mc_bus->mc_dev; + + switch (cmd) { + case FSL_MC_SEND_MC_COMMAND: + error = fsl_mc_uapi_send_command(root_mc_device, arg, priv_data->mc_io); + break; + default: + dev_dbg(&root_mc_device->dev, "unexpected ioctl call number\n"); + error = -EINVAL; + } + + return error; +} + +static const struct file_operations fsl_mc_uapi_dev_fops = { + .owner = THIS_MODULE, + .open = fsl_mc_uapi_dev_open, + .release = fsl_mc_uapi_dev_release, + .unlocked_ioctl = fsl_mc_uapi_dev_ioctl, +}; + +int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus) +{ + struct fsl_mc_device *mc_dev = &mc_bus->mc_dev; + struct fsl_mc_uapi *mc_uapi = &mc_bus->uapi_misc; + int error; + + mc_uapi->misc.minor = MISC_DYNAMIC_MINOR; + mc_uapi->misc.name = dev_name(&mc_dev->dev); + mc_uapi->misc.fops = &fsl_mc_uapi_dev_fops; + + error = misc_register(&mc_uapi->misc); + if (error) + return error; + + mc_uapi->static_mc_io = mc_bus->mc_dev.mc_io; + + mutex_init(&mc_uapi->mutex); + + return 0; +} + +void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus) +{ + misc_deregister(&mc_bus->uapi_misc.misc); +} diff --git a/drivers/bus/fsl-mc/mc-sys.c b/drivers/bus/fsl-mc/mc-sys.c index 85a0225db522..b291b35e3884 100644 --- a/drivers/bus/fsl-mc/mc-sys.c +++ b/drivers/bus/fsl-mc/mc-sys.c @@ -35,7 +35,7 @@ static enum mc_cmd_status mc_cmd_hdr_read_status(struct fsl_mc_command *cmd) return (enum mc_cmd_status)hdr->status; } -static u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd) +u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd) { struct mc_cmd_header *hdr = (struct mc_cmd_header *)&cmd->header; u16 cmd_id = le16_to_cpu(hdr->cmd_id); diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c index f0697f433c2f..be4eebb0971b 100644 --- a/drivers/bus/mhi/core/init.c +++ b/drivers/bus/mhi/core/init.c @@ -151,12 +151,17 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) { struct mhi_event *mhi_event = mhi_cntrl->mhi_event; struct device *dev = &mhi_cntrl->mhi_dev->dev; + unsigned long irq_flags = IRQF_SHARED | IRQF_NO_SUSPEND; int i, ret; + /* if controller driver has set irq_flags, use it */ + if (mhi_cntrl->irq_flags) + irq_flags = mhi_cntrl->irq_flags; + /* Setup BHI_INTVEC IRQ */ ret = request_threaded_irq(mhi_cntrl->irq[0], mhi_intvec_handler, mhi_intvec_threaded_handler, - IRQF_SHARED | IRQF_NO_SUSPEND, + irq_flags, "bhi", mhi_cntrl); if (ret) return ret; @@ -174,7 +179,7 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) ret = request_irq(mhi_cntrl->irq[mhi_event->irq], mhi_irq_handler, - IRQF_SHARED | IRQF_NO_SUSPEND, + irq_flags, "mhi", mhi_event); if (ret) { dev_err(dev, "Error requesting irq:%d for ev:%d\n", @@ -552,6 +557,9 @@ void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl, tre_ring = &mhi_chan->tre_ring; chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan]; + if (!chan_ctxt->rbase) /* Already uninitialized */ + return; + mhi_free_coherent(mhi_cntrl, tre_ring->alloc_size, tre_ring->pre_aligned, tre_ring->dma_handle); vfree(buf_ring->base); diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index 1202433ecf98..4e0131b94056 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -111,7 +111,14 @@ void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl, dma_addr_t db; db = ring->iommu_base + (ring->wp - ring->base); + + /* + * Writes to the new ring element must be visible to the hardware + * before letting h/w know there is new element to fetch. + */ + dma_wmb(); *ring->ctxt_wp = db; + mhi_chan->db_cfg.process_db(mhi_cntrl, &mhi_chan->db_cfg, ring->db_addr, db); } @@ -135,6 +142,19 @@ enum mhi_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl) } EXPORT_SYMBOL_GPL(mhi_get_mhi_state); +void mhi_soc_reset(struct mhi_controller *mhi_cntrl) +{ + if (mhi_cntrl->reset) { + mhi_cntrl->reset(mhi_cntrl); + return; + } + + /* Generic MHI SoC reset */ + mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, + MHI_SOC_RESET_REQ); +} +EXPORT_SYMBOL_GPL(mhi_soc_reset); + int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl, struct mhi_buf_info *buf_info) { @@ -959,118 +979,88 @@ static bool mhi_is_ring_full(struct mhi_controller *mhi_cntrl, return (tmp == ring->rp); } -int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir, - struct sk_buff *skb, size_t len, enum mhi_flags mflags) +static int mhi_queue(struct mhi_device *mhi_dev, struct mhi_buf_info *buf_info, + enum dma_data_direction dir, enum mhi_flags mflags) { struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : mhi_dev->dl_chan; struct mhi_ring *tre_ring = &mhi_chan->tre_ring; - struct mhi_buf_info buf_info = { }; + unsigned long flags; int ret; - /* If MHI host pre-allocates buffers then client drivers cannot queue */ - if (mhi_chan->pre_alloc) - return -EINVAL; - - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; - - read_lock_bh(&mhi_cntrl->pm_lock); - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { - read_unlock_bh(&mhi_cntrl->pm_lock); + if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) return -EIO; + + read_lock_irqsave(&mhi_cntrl->pm_lock, flags); + + ret = mhi_is_ring_full(mhi_cntrl, tre_ring); + if (unlikely(ret)) { + ret = -ENOMEM; + goto exit_unlock; } - /* we're in M3 or transitioning to M3 */ + ret = mhi_gen_tre(mhi_cntrl, mhi_chan, buf_info, mflags); + if (unlikely(ret)) + goto exit_unlock; + + /* trigger M3 exit if necessary */ if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) mhi_trigger_resume(mhi_cntrl); - /* Toggle wake to exit out of M2 */ + /* Assert dev_wake (to exit/prevent M1/M2)*/ mhi_cntrl->wake_toggle(mhi_cntrl); + if (mhi_chan->dir == DMA_TO_DEVICE) + atomic_inc(&mhi_cntrl->pending_pkts); + + if (unlikely(!MHI_DB_ACCESS_VALID(mhi_cntrl))) { + ret = -EIO; + goto exit_unlock; + } + + mhi_ring_chan_db(mhi_cntrl, mhi_chan); + +exit_unlock: + read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); + + return ret; +} + +int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir, + struct sk_buff *skb, size_t len, enum mhi_flags mflags) +{ + struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : + mhi_dev->dl_chan; + struct mhi_buf_info buf_info = { }; + buf_info.v_addr = skb->data; buf_info.cb_buf = skb; buf_info.len = len; - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) { - read_unlock_bh(&mhi_cntrl->pm_lock); - return ret; - } + if (unlikely(mhi_chan->pre_alloc)) + return -EINVAL; - if (mhi_chan->dir == DMA_TO_DEVICE) - atomic_inc(&mhi_cntrl->pending_pkts); - - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - read_lock_bh(&mhi_chan->lock); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_bh(&mhi_chan->lock); - } - - read_unlock_bh(&mhi_cntrl->pm_lock); - - return 0; + return mhi_queue(mhi_dev, &buf_info, dir, mflags); } EXPORT_SYMBOL_GPL(mhi_queue_skb); int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir, struct mhi_buf *mhi_buf, size_t len, enum mhi_flags mflags) { - struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : mhi_dev->dl_chan; - struct device *dev = &mhi_cntrl->mhi_dev->dev; - struct mhi_ring *tre_ring = &mhi_chan->tre_ring; struct mhi_buf_info buf_info = { }; - int ret; - - /* If MHI host pre-allocates buffers then client drivers cannot queue */ - if (mhi_chan->pre_alloc) - return -EINVAL; - - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; - - read_lock_bh(&mhi_cntrl->pm_lock); - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { - dev_err(dev, "MHI is not in activate state, PM state: %s\n", - to_mhi_pm_state_str(mhi_cntrl->pm_state)); - read_unlock_bh(&mhi_cntrl->pm_lock); - - return -EIO; - } - - /* we're in M3 or transitioning to M3 */ - if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) - mhi_trigger_resume(mhi_cntrl); - - /* Toggle wake to exit out of M2 */ - mhi_cntrl->wake_toggle(mhi_cntrl); buf_info.p_addr = mhi_buf->dma_addr; buf_info.cb_buf = mhi_buf; buf_info.pre_mapped = true; buf_info.len = len; - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) { - read_unlock_bh(&mhi_cntrl->pm_lock); - return ret; - } + if (unlikely(mhi_chan->pre_alloc)) + return -EINVAL; - if (mhi_chan->dir == DMA_TO_DEVICE) - atomic_inc(&mhi_cntrl->pending_pkts); - - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - read_lock_bh(&mhi_chan->lock); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_bh(&mhi_chan->lock); - } - - read_unlock_bh(&mhi_cntrl->pm_lock); - - return 0; + return mhi_queue(mhi_dev, &buf_info, dir, mflags); } EXPORT_SYMBOL_GPL(mhi_queue_dma); @@ -1124,57 +1114,13 @@ int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan, int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir, void *buf, size_t len, enum mhi_flags mflags) { - struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; - struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : - mhi_dev->dl_chan; - struct mhi_ring *tre_ring; struct mhi_buf_info buf_info = { }; - unsigned long flags; - int ret; - - /* - * this check here only as a guard, it's always - * possible mhi can enter error while executing rest of function, - * which is not fatal so we do not need to hold pm_lock - */ - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) - return -EIO; - - tre_ring = &mhi_chan->tre_ring; - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; buf_info.v_addr = buf; buf_info.cb_buf = buf; buf_info.len = len; - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) - return ret; - - read_lock_irqsave(&mhi_cntrl->pm_lock, flags); - - /* we're in M3 or transitioning to M3 */ - if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) - mhi_trigger_resume(mhi_cntrl); - - /* Toggle wake to exit out of M2 */ - mhi_cntrl->wake_toggle(mhi_cntrl); - - if (mhi_chan->dir == DMA_TO_DEVICE) - atomic_inc(&mhi_cntrl->pending_pkts); - - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - unsigned long flags; - - read_lock_irqsave(&mhi_chan->lock, flags); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_irqrestore(&mhi_chan->lock, flags); - } - - read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); - - return 0; + return mhi_queue(mhi_dev, &buf_info, dir, mflags); } EXPORT_SYMBOL_GPL(mhi_queue_buf); diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index f5bee76ea061..20673a4b4a3c 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -8,13 +8,21 @@ * Copyright (C) 2020 Linaro Ltd */ +#include +#include #include #include #include #include +#include +#include #define MHI_PCI_DEFAULT_BAR_NUM 0 +#define MHI_POST_RESET_DELAY_MS 500 + +#define HEALTH_CHECK_PERIOD (HZ * 2) + /** * struct mhi_pci_dev_info - MHI PCI device specific information * @config: MHI controller configuration @@ -76,6 +84,36 @@ struct mhi_pci_dev_info { .offload_channel = false, \ } +#define MHI_CHANNEL_CONFIG_HW_UL(ch_num, ch_name, el_count, ev_ring) \ + { \ + .num = ch_num, \ + .name = ch_name, \ + .num_elements = el_count, \ + .event_ring = ev_ring, \ + .dir = DMA_TO_DEVICE, \ + .ee_mask = BIT(MHI_EE_AMSS), \ + .pollcfg = 0, \ + .doorbell = MHI_DB_BRST_ENABLE, \ + .lpm_notify = false, \ + .offload_channel = false, \ + .doorbell_mode_switch = true, \ + } \ + +#define MHI_CHANNEL_CONFIG_HW_DL(ch_num, ch_name, el_count, ev_ring) \ + { \ + .num = ch_num, \ + .name = ch_name, \ + .num_elements = el_count, \ + .event_ring = ev_ring, \ + .dir = DMA_FROM_DEVICE, \ + .ee_mask = BIT(MHI_EE_AMSS), \ + .pollcfg = 0, \ + .doorbell = MHI_DB_BRST_ENABLE, \ + .lpm_notify = false, \ + .offload_channel = false, \ + .doorbell_mode_switch = true, \ + } + #define MHI_EVENT_CONFIG_DATA(ev_ring) \ { \ .num_elements = 128, \ @@ -91,8 +129,8 @@ struct mhi_pci_dev_info { #define MHI_EVENT_CONFIG_HW_DATA(ev_ring, ch_num) \ { \ - .num_elements = 128, \ - .irq_moderation_ms = 5, \ + .num_elements = 2048, \ + .irq_moderation_ms = 1, \ .irq = (ev_ring) + 1, \ .priority = 1, \ .mode = MHI_DB_BRST_DISABLE, \ @@ -104,27 +142,31 @@ struct mhi_pci_dev_info { } static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = { + MHI_CHANNEL_CONFIG_UL(4, "DIAG", 16, 1), + MHI_CHANNEL_CONFIG_DL(5, "DIAG", 16, 1), MHI_CHANNEL_CONFIG_UL(12, "MBIM", 4, 0), MHI_CHANNEL_CONFIG_DL(13, "MBIM", 4, 0), MHI_CHANNEL_CONFIG_UL(14, "QMI", 4, 0), MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0), MHI_CHANNEL_CONFIG_DL(21, "IPCR", 8, 0), - MHI_CHANNEL_CONFIG_UL(100, "IP_HW0", 128, 1), - MHI_CHANNEL_CONFIG_DL(101, "IP_HW0", 128, 2), + MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 2), + MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 3), }; -static const struct mhi_event_config modem_qcom_v1_mhi_events[] = { +static struct mhi_event_config modem_qcom_v1_mhi_events[] = { /* first ring is control+data ring */ MHI_EVENT_CONFIG_CTRL(0), + /* DIAG dedicated event ring */ + MHI_EVENT_CONFIG_DATA(1), /* Hardware channels request dedicated hardware event rings */ - MHI_EVENT_CONFIG_HW_DATA(1, 100), - MHI_EVENT_CONFIG_HW_DATA(2, 101) + MHI_EVENT_CONFIG_HW_DATA(2, 100), + MHI_EVENT_CONFIG_HW_DATA(3, 101) }; -static const struct mhi_controller_config modem_qcom_v1_mhiv_config = { +static struct mhi_controller_config modem_qcom_v1_mhiv_config = { .max_channels = 128, - .timeout_ms = 5000, + .timeout_ms = 8000, .num_channels = ARRAY_SIZE(modem_qcom_v1_mhi_channels), .ch_cfg = modem_qcom_v1_mhi_channels, .num_events = ARRAY_SIZE(modem_qcom_v1_mhi_events), @@ -147,6 +189,18 @@ static const struct pci_device_id mhi_pci_id_table[] = { }; MODULE_DEVICE_TABLE(pci, mhi_pci_id_table); +enum mhi_pci_device_status { + MHI_PCI_DEV_STARTED, +}; + +struct mhi_pci_device { + struct mhi_controller mhi_cntrl; + struct pci_saved_state *pci_state; + struct work_struct recovery_work; + struct timer_list health_check_timer; + unsigned long status; +}; + static int mhi_pci_read_reg(struct mhi_controller *mhi_cntrl, void __iomem *addr, u32 *out) { @@ -163,7 +217,31 @@ static void mhi_pci_write_reg(struct mhi_controller *mhi_cntrl, static void mhi_pci_status_cb(struct mhi_controller *mhi_cntrl, enum mhi_callback cb) { + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + /* Nothing to do for now */ + switch (cb) { + case MHI_CB_FATAL_ERROR: + case MHI_CB_SYS_ERROR: + dev_warn(&pdev->dev, "firmware crashed (%u)\n", cb); + break; + default: + break; + } +} + +static bool mhi_pci_is_alive(struct mhi_controller *mhi_cntrl) +{ + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + u16 vendor = 0; + + if (pci_read_config_word(pdev, PCI_VENDOR_ID, &vendor)) + return false; + + if (vendor == (u16) ~0 || vendor == 0) + return false; + + return true; } static int mhi_pci_claim(struct mhi_controller *mhi_cntrl, @@ -227,8 +305,12 @@ static int mhi_pci_get_irqs(struct mhi_controller *mhi_cntrl, } if (nr_vectors < mhi_cntrl->nr_irqs) { - dev_warn(&pdev->dev, "Not enough MSI vectors (%d/%d), use shared MSI\n", - nr_vectors, mhi_cntrl_config->num_events); + dev_warn(&pdev->dev, "using shared MSI\n"); + + /* Patch msi vectors, use only one (shared) */ + for (i = 0; i < mhi_cntrl_config->num_events; i++) + mhi_cntrl_config->event_cfg[i].irq = 0; + mhi_cntrl->nr_irqs = 1; } irq = devm_kcalloc(&pdev->dev, mhi_cntrl->nr_irqs, sizeof(int), GFP_KERNEL); @@ -257,20 +339,89 @@ static void mhi_pci_runtime_put(struct mhi_controller *mhi_cntrl) /* no PM for now */ } +static void mhi_pci_recovery_work(struct work_struct *work) +{ + struct mhi_pci_device *mhi_pdev = container_of(work, struct mhi_pci_device, + recovery_work); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + int err; + + dev_warn(&pdev->dev, "device recovery started\n"); + + del_timer(&mhi_pdev->health_check_timer); + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } + + /* Check if we can recover without full reset */ + pci_set_power_state(pdev, PCI_D0); + pci_load_saved_state(pdev, mhi_pdev->pci_state); + pci_restore_state(pdev); + + if (!mhi_pci_is_alive(mhi_cntrl)) + goto err_try_reset; + + err = mhi_prepare_for_power_up(mhi_cntrl); + if (err) + goto err_try_reset; + + err = mhi_sync_power_up(mhi_cntrl); + if (err) + goto err_unprepare; + + dev_dbg(&pdev->dev, "Recovery completed\n"); + + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + return; + +err_unprepare: + mhi_unprepare_after_power_down(mhi_cntrl); +err_try_reset: + if (pci_reset_function(pdev)) + dev_err(&pdev->dev, "Recovery failed\n"); +} + +static void health_check(struct timer_list *t) +{ + struct mhi_pci_device *mhi_pdev = from_timer(mhi_pdev, t, health_check_timer); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + if (!mhi_pci_is_alive(mhi_cntrl)) { + dev_err(mhi_cntrl->cntrl_dev, "Device died\n"); + queue_work(system_long_wq, &mhi_pdev->recovery_work); + return; + } + + /* reschedule in two seconds */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); +} + static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { const struct mhi_pci_dev_info *info = (struct mhi_pci_dev_info *) id->driver_data; const struct mhi_controller_config *mhi_cntrl_config; + struct mhi_pci_device *mhi_pdev; struct mhi_controller *mhi_cntrl; int err; dev_dbg(&pdev->dev, "MHI PCI device found: %s\n", info->name); - mhi_cntrl = mhi_alloc_controller(); - if (!mhi_cntrl) + /* mhi_pdev.mhi_cntrl must be zero-initialized */ + mhi_pdev = devm_kzalloc(&pdev->dev, sizeof(*mhi_pdev), GFP_KERNEL); + if (!mhi_pdev) return -ENOMEM; + INIT_WORK(&mhi_pdev->recovery_work, mhi_pci_recovery_work); + timer_setup(&mhi_pdev->health_check_timer, health_check, 0); + mhi_cntrl_config = info->config; + mhi_cntrl = &mhi_pdev->mhi_cntrl; + mhi_cntrl->cntrl_dev = &pdev->dev; mhi_cntrl->iova_start = 0; mhi_cntrl->iova_stop = (dma_addr_t)DMA_BIT_MASK(info->dma_data_width); @@ -285,17 +436,23 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = mhi_pci_claim(mhi_cntrl, info->bar_num, DMA_BIT_MASK(info->dma_data_width)); if (err) - goto err_release; + return err; err = mhi_pci_get_irqs(mhi_cntrl, mhi_cntrl_config); if (err) - goto err_release; + return err; - pci_set_drvdata(pdev, mhi_cntrl); + pci_set_drvdata(pdev, mhi_pdev); + + /* Have stored pci confspace at hand for restore in sudden PCI error */ + pci_save_state(pdev); + mhi_pdev->pci_state = pci_store_saved_state(pdev); + + pci_enable_pcie_error_reporting(pdev); err = mhi_register_controller(mhi_cntrl, mhi_cntrl_config); if (err) - goto err_release; + return err; /* MHI bus does not power up the controller by default */ err = mhi_prepare_for_power_up(mhi_cntrl); @@ -310,33 +467,209 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_unprepare; } + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + + /* start health check */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + return 0; err_unprepare: mhi_unprepare_after_power_down(mhi_cntrl); err_unregister: mhi_unregister_controller(mhi_cntrl); -err_release: - mhi_free_controller(mhi_cntrl); return err; } static void mhi_pci_remove(struct pci_dev *pdev) { - struct mhi_controller *mhi_cntrl = pci_get_drvdata(pdev); + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + del_timer(&mhi_pdev->health_check_timer); + cancel_work_sync(&mhi_pdev->recovery_work); + + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, true); + mhi_unprepare_after_power_down(mhi_cntrl); + } - mhi_power_down(mhi_cntrl, true); - mhi_unprepare_after_power_down(mhi_cntrl); mhi_unregister_controller(mhi_cntrl); - mhi_free_controller(mhi_cntrl); } +static void mhi_pci_reset_prepare(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + dev_info(&pdev->dev, "reset\n"); + + del_timer(&mhi_pdev->health_check_timer); + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } + + /* cause internal device reset */ + mhi_soc_reset(mhi_cntrl); + + /* Be sure device reset has been executed */ + msleep(MHI_POST_RESET_DELAY_MS); +} + +static void mhi_pci_reset_done(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + int err; + + /* Restore initial known working PCI state */ + pci_load_saved_state(pdev, mhi_pdev->pci_state); + pci_restore_state(pdev); + + /* Is device status available ? */ + if (!mhi_pci_is_alive(mhi_cntrl)) { + dev_err(&pdev->dev, "reset failed\n"); + return; + } + + err = mhi_prepare_for_power_up(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to prepare MHI controller\n"); + return; + } + + err = mhi_sync_power_up(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to power up MHI controller\n"); + mhi_unprepare_after_power_down(mhi_cntrl); + return; + } + + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); +} + +static pci_ers_result_t mhi_pci_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + dev_err(&pdev->dev, "PCI error detected, state = %u\n", state); + + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } else { + /* Nothing to do */ + return PCI_ERS_RESULT_RECOVERED; + } + + pci_disable_device(pdev); + + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t mhi_pci_slot_reset(struct pci_dev *pdev) +{ + if (pci_enable_device(pdev)) { + dev_err(&pdev->dev, "Cannot re-enable PCI device after reset.\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + return PCI_ERS_RESULT_RECOVERED; +} + +static void mhi_pci_io_resume(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + + dev_err(&pdev->dev, "PCI slot reset done\n"); + + queue_work(system_long_wq, &mhi_pdev->recovery_work); +} + +static const struct pci_error_handlers mhi_pci_err_handler = { + .error_detected = mhi_pci_error_detected, + .slot_reset = mhi_pci_slot_reset, + .resume = mhi_pci_io_resume, + .reset_prepare = mhi_pci_reset_prepare, + .reset_done = mhi_pci_reset_done, +}; + +static int __maybe_unused mhi_pci_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + del_timer(&mhi_pdev->health_check_timer); + cancel_work_sync(&mhi_pdev->recovery_work); + + /* Transition to M3 state */ + mhi_pm_suspend(mhi_cntrl); + + pci_save_state(pdev); + pci_disable_device(pdev); + pci_wake_from_d3(pdev, true); + pci_set_power_state(pdev, PCI_D3hot); + + return 0; +} + +static int __maybe_unused mhi_pci_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + int err; + + pci_set_power_state(pdev, PCI_D0); + pci_restore_state(pdev); + pci_set_master(pdev); + + err = pci_enable_device(pdev); + if (err) + goto err_recovery; + + /* Exit M3, transition to M0 state */ + err = mhi_pm_resume(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to resume device: %d\n", err); + goto err_recovery; + } + + /* Resume health check */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + + return 0; + +err_recovery: + /* The device may have loose power or crashed, try recovering it */ + queue_work(system_long_wq, &mhi_pdev->recovery_work); + + return err; +} + +static const struct dev_pm_ops mhi_pci_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mhi_pci_suspend, mhi_pci_resume) +}; + static struct pci_driver mhi_pci_driver = { .name = "mhi-pci-generic", .id_table = mhi_pci_id_table, .probe = mhi_pci_probe, - .remove = mhi_pci_remove + .remove = mhi_pci_remove, + .err_handler = &mhi_pci_err_handler, + .driver.pm = &mhi_pci_pm_ops }; module_pci_driver(mhi_pci_driver); diff --git a/drivers/char/hw_random/ingenic-trng.c b/drivers/char/hw_random/ingenic-trng.c index 954a8411d67d..0eb80f786f4d 100644 --- a/drivers/char/hw_random/ingenic-trng.c +++ b/drivers/char/hw_random/ingenic-trng.c @@ -113,13 +113,17 @@ static int ingenic_trng_probe(struct platform_device *pdev) ret = hwrng_register(&trng->rng); if (ret) { dev_err(&pdev->dev, "Failed to register hwrng\n"); - return ret; + goto err_unprepare_clk; } platform_set_drvdata(pdev, trng); dev_info(&pdev->dev, "Ingenic DTRNG driver registered\n"); return 0; + +err_unprepare_clk: + clk_disable_unprepare(trng->clk); + return ret; } static int ingenic_trng_remove(struct platform_device *pdev) diff --git a/drivers/char/hw_random/iproc-rng200.c b/drivers/char/hw_random/iproc-rng200.c index 01583faf9893..a43743887db1 100644 --- a/drivers/char/hw_random/iproc-rng200.c +++ b/drivers/char/hw_random/iproc-rng200.c @@ -28,7 +28,6 @@ #define RNG_CTRL_OFFSET 0x00 #define RNG_CTRL_RNG_RBGEN_MASK 0x00001FFF #define RNG_CTRL_RNG_RBGEN_ENABLE 0x00000001 -#define RNG_CTRL_RNG_RBGEN_DISABLE 0x00000000 #define RNG_SOFT_RESET_OFFSET 0x04 #define RNG_SOFT_RESET 0x00000001 @@ -54,15 +53,24 @@ struct iproc_rng200_dev { #define to_rng_priv(rng) container_of(rng, struct iproc_rng200_dev, rng) +static void iproc_rng200_enable_set(void __iomem *rng_base, bool enable) +{ + u32 val; + + val = ioread32(rng_base + RNG_CTRL_OFFSET); + val &= ~RNG_CTRL_RNG_RBGEN_MASK; + + if (enable) + val |= RNG_CTRL_RNG_RBGEN_ENABLE; + + iowrite32(val, rng_base + RNG_CTRL_OFFSET); +} + static void iproc_rng200_restart(void __iomem *rng_base) { uint32_t val; - /* Disable RBG */ - val = ioread32(rng_base + RNG_CTRL_OFFSET); - val &= ~RNG_CTRL_RNG_RBGEN_MASK; - val |= RNG_CTRL_RNG_RBGEN_DISABLE; - iowrite32(val, rng_base + RNG_CTRL_OFFSET); + iproc_rng200_enable_set(rng_base, false); /* Clear all interrupt status */ iowrite32(0xFFFFFFFFUL, rng_base + RNG_INT_STATUS_OFFSET); @@ -84,11 +92,7 @@ static void iproc_rng200_restart(void __iomem *rng_base) val &= ~RBG_SOFT_RESET; iowrite32(val, rng_base + RBG_SOFT_RESET_OFFSET); - /* Enable RBG */ - val = ioread32(rng_base + RNG_CTRL_OFFSET); - val &= ~RNG_CTRL_RNG_RBGEN_MASK; - val |= RNG_CTRL_RNG_RBGEN_ENABLE; - iowrite32(val, rng_base + RNG_CTRL_OFFSET); + iproc_rng200_enable_set(rng_base, true); } static int iproc_rng200_read(struct hwrng *rng, void *buf, size_t max, @@ -155,13 +159,8 @@ static int iproc_rng200_read(struct hwrng *rng, void *buf, size_t max, static int iproc_rng200_init(struct hwrng *rng) { struct iproc_rng200_dev *priv = to_rng_priv(rng); - uint32_t val; - /* Setup RNG. */ - val = ioread32(priv->base + RNG_CTRL_OFFSET); - val &= ~RNG_CTRL_RNG_RBGEN_MASK; - val |= RNG_CTRL_RNG_RBGEN_ENABLE; - iowrite32(val, priv->base + RNG_CTRL_OFFSET); + iproc_rng200_enable_set(priv->base, true); return 0; } @@ -169,13 +168,8 @@ static int iproc_rng200_init(struct hwrng *rng) static void iproc_rng200_cleanup(struct hwrng *rng) { struct iproc_rng200_dev *priv = to_rng_priv(rng); - uint32_t val; - /* Disable RNG hardware */ - val = ioread32(priv->base + RNG_CTRL_OFFSET); - val &= ~RNG_CTRL_RNG_RBGEN_MASK; - val |= RNG_CTRL_RNG_RBGEN_DISABLE; - iowrite32(val, priv->base + RNG_CTRL_OFFSET); + iproc_rng200_enable_set(priv->base, false); } static int iproc_rng200_probe(struct platform_device *pdev) diff --git a/drivers/char/hw_random/nomadik-rng.c b/drivers/char/hw_random/nomadik-rng.c index b0ded41eb865..67947a19aa22 100644 --- a/drivers/char/hw_random/nomadik-rng.c +++ b/drivers/char/hw_random/nomadik-rng.c @@ -69,11 +69,10 @@ out_clk: return ret; } -static int nmk_rng_remove(struct amba_device *dev) +static void nmk_rng_remove(struct amba_device *dev) { amba_release_regions(dev); clk_disable(rng_clk); - return 0; } static const struct amba_id nmk_rng_ids[] = { diff --git a/drivers/char/hw_random/optee-rng.c b/drivers/char/hw_random/optee-rng.c index a99d82949981..135a82590923 100644 --- a/drivers/char/hw_random/optee-rng.c +++ b/drivers/char/hw_random/optee-rng.c @@ -243,7 +243,7 @@ static int optee_rng_probe(struct device *dev) if (err) goto out_sess; - err = hwrng_register(&pvt_data.optee_rng); + err = devm_hwrng_register(dev, &pvt_data.optee_rng); if (err) { dev_err(dev, "hwrng registration failed (%d)\n", err); goto out_sess; @@ -263,7 +263,6 @@ out_ctx: static int optee_rng_remove(struct device *dev) { - hwrng_unregister(&pvt_data.optee_rng); tee_client_close_session(pvt_data.ctx, pvt_data.session_id); tee_client_close_context(pvt_data.ctx); diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index e262445fed5f..8ea1fc831eb7 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -69,7 +69,7 @@ static int timeriomem_rng_read(struct hwrng *hwrng, void *data, */ if (retval > 0) usleep_range(period_us, - period_us + min(1, period_us / 100)); + period_us + max(1, period_us / 100)); *(u32 *)data = readl(priv->io_base); retval += sizeof(u32); @@ -169,7 +169,7 @@ static int timeriomem_rng_probe(struct platform_device *pdev) priv->present = 1; complete(&priv->completion); - err = hwrng_register(&priv->rng_ops); + err = devm_hwrng_register(&pdev->dev, &priv->rng_ops); if (err) { dev_err(&pdev->dev, "problem registering\n"); return err; @@ -185,7 +185,6 @@ static int timeriomem_rng_remove(struct platform_device *pdev) { struct timeriomem_rng_private *priv = platform_get_drvdata(pdev); - hwrng_unregister(&priv->rng_ops); hrtimer_cancel(&priv->timer); return 0; diff --git a/drivers/char/ipmi/ipmb_dev_int.c b/drivers/char/ipmi/ipmb_dev_int.c index 382b28f1cf2f..49b8f22fdcf0 100644 --- a/drivers/char/ipmi/ipmb_dev_int.c +++ b/drivers/char/ipmi/ipmb_dev_int.c @@ -137,7 +137,7 @@ static ssize_t ipmb_write(struct file *file, const char __user *buf, { struct ipmb_dev *ipmb_dev = to_ipmb_dev(file); u8 rq_sa, netf_rq_lun, msg_len; - union i2c_smbus_data data; + struct i2c_client *temp_client; u8 msg[MAX_MSG_LEN]; ssize_t ret; @@ -160,21 +160,21 @@ static ssize_t ipmb_write(struct file *file, const char __user *buf, } /* - * subtract rq_sa and netf_rq_lun from the length of the msg passed to - * i2c_smbus_xfer + * subtract rq_sa and netf_rq_lun from the length of the msg. Fill the + * temporary client. Note that its use is an exception for IPMI. */ msg_len = msg[IPMB_MSG_LEN_IDX] - SMBUS_MSG_HEADER_LENGTH; - if (msg_len > I2C_SMBUS_BLOCK_MAX) - msg_len = I2C_SMBUS_BLOCK_MAX; + temp_client = kmemdup(ipmb_dev->client, sizeof(*temp_client), GFP_KERNEL); + if (!temp_client) + return -ENOMEM; - data.block[0] = msg_len; - memcpy(&data.block[1], msg + SMBUS_MSG_IDX_OFFSET, msg_len); - ret = i2c_smbus_xfer(ipmb_dev->client->adapter, rq_sa, - ipmb_dev->client->flags, - I2C_SMBUS_WRITE, netf_rq_lun, - I2C_SMBUS_BLOCK_DATA, &data); + temp_client->addr = rq_sa; - return ret ? : count; + ret = i2c_smbus_write_block_data(temp_client, netf_rq_lun, msg_len, + msg + SMBUS_MSG_IDX_OFFSET); + kfree(temp_client); + + return ret < 0 ? ret : count; } static __poll_t ipmb_poll(struct file *file, poll_table *wait) diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 887ffca3f47f..869b9f5e8e03 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -31,9 +31,6 @@ #include #include #include -#include -#include -#include #ifdef CONFIG_IA64 # include @@ -829,42 +826,6 @@ static loff_t memory_lseek(struct file *file, loff_t offset, int orig) return ret; } -static struct inode *devmem_inode; - -#ifdef CONFIG_IO_STRICT_DEVMEM -void revoke_devmem(struct resource *res) -{ - /* pairs with smp_store_release() in devmem_init_inode() */ - struct inode *inode = smp_load_acquire(&devmem_inode); - - /* - * Check that the initialization has completed. Losing the race - * is ok because it means drivers are claiming resources before - * the fs_initcall level of init and prevent /dev/mem from - * establishing mappings. - */ - if (!inode) - return; - - /* - * The expectation is that the driver has successfully marked - * the resource busy by this point, so devmem_is_allowed() - * should start returning false, however for performance this - * does not iterate the entire resource range. - */ - if (devmem_is_allowed(PHYS_PFN(res->start)) && - devmem_is_allowed(PHYS_PFN(res->end))) { - /* - * *cringe* iomem=relaxed says "go ahead, what's the - * worst that can happen?" - */ - return; - } - - unmap_mapping_range(inode->i_mapping, res->start, resource_size(res), 1); -} -#endif - static int open_port(struct inode *inode, struct file *filp) { int rc; @@ -884,8 +845,7 @@ static int open_port(struct inode *inode, struct file *filp) * revocations when drivers want to take over a /dev/mem mapped * range. */ - inode->i_mapping = devmem_inode->i_mapping; - filp->f_mapping = inode->i_mapping; + filp->f_mapping = iomem_get_mapping(); return 0; } @@ -1017,48 +977,6 @@ static char *mem_devnode(struct device *dev, umode_t *mode) static struct class *mem_class; -static int devmem_fs_init_fs_context(struct fs_context *fc) -{ - return init_pseudo(fc, DEVMEM_MAGIC) ? 0 : -ENOMEM; -} - -static struct file_system_type devmem_fs_type = { - .name = "devmem", - .owner = THIS_MODULE, - .init_fs_context = devmem_fs_init_fs_context, - .kill_sb = kill_anon_super, -}; - -static int devmem_init_inode(void) -{ - static struct vfsmount *devmem_vfs_mount; - static int devmem_fs_cnt; - struct inode *inode; - int rc; - - rc = simple_pin_fs(&devmem_fs_type, &devmem_vfs_mount, &devmem_fs_cnt); - if (rc < 0) { - pr_err("Cannot mount /dev/mem pseudo filesystem: %d\n", rc); - return rc; - } - - inode = alloc_anon_inode(devmem_vfs_mount->mnt_sb); - if (IS_ERR(inode)) { - rc = PTR_ERR(inode); - pr_err("Cannot allocate inode for /dev/mem: %d\n", rc); - simple_release_fs(&devmem_vfs_mount, &devmem_fs_cnt); - return rc; - } - - /* - * Publish /dev/mem initialized. - * Pairs with smp_load_acquire() in revoke_devmem(). - */ - smp_store_release(&devmem_inode, inode); - - return 0; -} - static int __init chr_dev_init(void) { int minor; @@ -1080,8 +998,6 @@ static int __init chr_dev_init(void) */ if ((minor == DEVPORT_MINOR) && !arch_has_dev_port()) continue; - if ((minor == DEVMEM_MINOR) && devmem_init_inode() != 0) - continue; device_create(mem_class, NULL, MKDEV(MEM_MAJOR, minor), NULL, devlist[minor].name); diff --git a/drivers/char/random.c b/drivers/char/random.c index 84e24986a97a..0fe9e200e4c8 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1959,7 +1959,7 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg) return -EPERM; if (crng_init < 2) return -ENODATA; - crng_reseed(&primary_crng, NULL); + crng_reseed(&primary_crng, &input_pool); crng_global_init_time = jiffies - 1; return 0; default: diff --git a/drivers/char/tpm/Kconfig b/drivers/char/tpm/Kconfig index a18c314da211..4308f9ca7a43 100644 --- a/drivers/char/tpm/Kconfig +++ b/drivers/char/tpm/Kconfig @@ -86,6 +86,16 @@ config TCG_TIS_SYNQUACER To compile this driver as a module, choose M here; the module will be called tpm_tis_synquacer. +config TCG_TIS_I2C_CR50 + tristate "TPM Interface Specification 2.0 Interface (I2C - CR50)" + depends on I2C + select TCG_CR50 + help + This is a driver for the Google cr50 I2C TPM interface which is a + custom microcontroller and requires a custom i2c protocol interface + to handle the limitations of the hardware. To compile this driver + as a module, choose M here; the module will be called tcg_tis_i2c_cr50. + config TCG_TIS_I2C_ATMEL tristate "TPM Interface Specification 1.2 Interface (I2C - Atmel)" depends on I2C diff --git a/drivers/char/tpm/Makefile b/drivers/char/tpm/Makefile index 84db4fb3a9c9..66d39ea6bd10 100644 --- a/drivers/char/tpm/Makefile +++ b/drivers/char/tpm/Makefile @@ -27,6 +27,8 @@ obj-$(CONFIG_TCG_TIS_SPI) += tpm_tis_spi.o tpm_tis_spi-y := tpm_tis_spi_main.o tpm_tis_spi-$(CONFIG_TCG_TIS_SPI_CR50) += tpm_tis_spi_cr50.o +obj-$(CONFIG_TCG_TIS_I2C_CR50) += tpm_tis_i2c_cr50.o + obj-$(CONFIG_TCG_TIS_I2C_ATMEL) += tpm_i2c_atmel.o obj-$(CONFIG_TCG_TIS_I2C_INFINEON) += tpm_i2c_infineon.o obj-$(CONFIG_TCG_TIS_I2C_NUVOTON) += tpm_i2c_nuvoton.o diff --git a/drivers/char/tpm/eventlog/tpm1.c b/drivers/char/tpm/eventlog/tpm1.c index 2c96977ad080..8aa9057601d6 100644 --- a/drivers/char/tpm/eventlog/tpm1.c +++ b/drivers/char/tpm/eventlog/tpm1.c @@ -210,6 +210,7 @@ static int get_event_name(char *dest, struct tcpa_event *event, default: break; } + break; default: break; } diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index ddaeceb7e109..19e23fcc6bc8 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -278,6 +278,8 @@ static void tpm_devs_release(struct device *dev) { struct tpm_chip *chip = container_of(dev, struct tpm_chip, devs); + dump_stack(); + /* release the master device reference */ put_device(&chip->dev); } diff --git a/drivers/char/tpm/tpm-dev-common.c b/drivers/char/tpm/tpm-dev-common.c index 1784530b8387..c08cbb306636 100644 --- a/drivers/char/tpm/tpm-dev-common.c +++ b/drivers/char/tpm/tpm-dev-common.c @@ -20,7 +20,6 @@ #include "tpm-dev.h" static struct workqueue_struct *tpm_dev_wq; -static DEFINE_MUTEX(tpm_dev_wq_lock); static ssize_t tpm_dev_transmit(struct tpm_chip *chip, struct tpm_space *space, u8 *buf, size_t bufsiz) diff --git a/drivers/char/tpm/tpm-sysfs.c b/drivers/char/tpm/tpm-sysfs.c index e2ff0b273a0f..63f03cfb8e6a 100644 --- a/drivers/char/tpm/tpm-sysfs.c +++ b/drivers/char/tpm/tpm-sysfs.c @@ -337,11 +337,190 @@ static const struct attribute_group tpm2_dev_group = { .attrs = tpm2_dev_attrs, }; +struct tpm_pcr_attr { + int alg_id; + int pcr; + struct device_attribute attr; +}; + +#define to_tpm_pcr_attr(a) container_of(a, struct tpm_pcr_attr, attr) + +static ssize_t pcr_value_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tpm_pcr_attr *ha = to_tpm_pcr_attr(attr); + struct tpm_chip *chip = to_tpm_chip(dev); + struct tpm_digest digest; + int i; + int digest_size = 0; + int rc; + char *str = buf; + + for (i = 0; i < chip->nr_allocated_banks; i++) + if (ha->alg_id == chip->allocated_banks[i].alg_id) + digest_size = chip->allocated_banks[i].digest_size; + /* should never happen */ + if (!digest_size) + return -EINVAL; + + digest.alg_id = ha->alg_id; + rc = tpm_pcr_read(chip, ha->pcr, &digest); + if (rc) + return rc; + for (i = 0; i < digest_size; i++) + str += sprintf(str, "%02X", digest.digest[i]); + str += sprintf(str, "\n"); + + return str - buf; +} + +/* + * The following set of defines represents all the magic to build + * the per hash attribute groups for displaying each bank of PCRs. + * The only slight problem with this approach is that every PCR is + * hard coded to be present, so you don't know if an PCR is missing + * until a cat of the file returns -EINVAL + * + * Also note you must ignore checkpatch warnings in this macro + * code. This is deep macro magic that checkpatch.pl doesn't + * understand. + */ + +/* Note, this must match TPM2_PLATFORM_PCR which is fixed at 24. */ +#define _TPM_HELPER(_alg, _hash, F) \ + F(_alg, _hash, 0) \ + F(_alg, _hash, 1) \ + F(_alg, _hash, 2) \ + F(_alg, _hash, 3) \ + F(_alg, _hash, 4) \ + F(_alg, _hash, 5) \ + F(_alg, _hash, 6) \ + F(_alg, _hash, 7) \ + F(_alg, _hash, 8) \ + F(_alg, _hash, 9) \ + F(_alg, _hash, 10) \ + F(_alg, _hash, 11) \ + F(_alg, _hash, 12) \ + F(_alg, _hash, 13) \ + F(_alg, _hash, 14) \ + F(_alg, _hash, 15) \ + F(_alg, _hash, 16) \ + F(_alg, _hash, 17) \ + F(_alg, _hash, 18) \ + F(_alg, _hash, 19) \ + F(_alg, _hash, 20) \ + F(_alg, _hash, 21) \ + F(_alg, _hash, 22) \ + F(_alg, _hash, 23) + +/* ignore checkpatch warning about trailing ; in macro. */ +#define PCR_ATTR(_alg, _hash, _pcr) \ + static struct tpm_pcr_attr dev_attr_pcr_##_hash##_##_pcr = { \ + .alg_id = _alg, \ + .pcr = _pcr, \ + .attr = { \ + .attr = { \ + .name = __stringify(_pcr), \ + .mode = 0444 \ + }, \ + .show = pcr_value_show \ + } \ + }; + +#define PCR_ATTRS(_alg, _hash) \ + _TPM_HELPER(_alg, _hash, PCR_ATTR) + +/* ignore checkpatch warning about trailing , in macro. */ +#define PCR_ATTR_VAL(_alg, _hash, _pcr) \ + &dev_attr_pcr_##_hash##_##_pcr.attr.attr, + +#define PCR_ATTR_GROUP_ARRAY(_alg, _hash) \ + static struct attribute *pcr_group_attrs_##_hash[] = { \ + _TPM_HELPER(_alg, _hash, PCR_ATTR_VAL) \ + NULL \ + } + +#define PCR_ATTR_GROUP(_alg, _hash) \ + static struct attribute_group pcr_group_##_hash = { \ + .name = "pcr-" __stringify(_hash), \ + .attrs = pcr_group_attrs_##_hash \ + } + +#define PCR_ATTR_BUILD(_alg, _hash) \ + PCR_ATTRS(_alg, _hash) \ + PCR_ATTR_GROUP_ARRAY(_alg, _hash); \ + PCR_ATTR_GROUP(_alg, _hash) +/* + * End of macro structure to build an attribute group containing 24 + * PCR value files for each supported hash algorithm + */ + +/* + * The next set of macros implements the cleverness for each hash to + * build a static attribute group called pcr_group_ which can be + * added to chip->groups[]. + * + * The first argument is the TPM algorithm id and the second is the + * hash used as both the suffix and the group name. Note: the group + * name is a directory in the top level tpm class with the name + * pcr-, so it must not clash with any other names already + * in the sysfs directory. + */ +PCR_ATTR_BUILD(TPM_ALG_SHA1, sha1); +PCR_ATTR_BUILD(TPM_ALG_SHA256, sha256); +PCR_ATTR_BUILD(TPM_ALG_SHA384, sha384); +PCR_ATTR_BUILD(TPM_ALG_SHA512, sha512); +PCR_ATTR_BUILD(TPM_ALG_SM3_256, sm3); + + void tpm_sysfs_add_device(struct tpm_chip *chip) { + int i; + WARN_ON(chip->groups_cnt != 0); + if (chip->flags & TPM_CHIP_FLAG_TPM2) chip->groups[chip->groups_cnt++] = &tpm2_dev_group; else chip->groups[chip->groups_cnt++] = &tpm1_dev_group; + + /* add one group for each bank hash */ + for (i = 0; i < chip->nr_allocated_banks; i++) { + switch (chip->allocated_banks[i].alg_id) { + case TPM_ALG_SHA1: + chip->groups[chip->groups_cnt++] = &pcr_group_sha1; + break; + case TPM_ALG_SHA256: + chip->groups[chip->groups_cnt++] = &pcr_group_sha256; + break; + case TPM_ALG_SHA384: + chip->groups[chip->groups_cnt++] = &pcr_group_sha384; + break; + case TPM_ALG_SHA512: + chip->groups[chip->groups_cnt++] = &pcr_group_sha512; + break; + case TPM_ALG_SM3_256: + chip->groups[chip->groups_cnt++] = &pcr_group_sm3; + break; + default: + /* + * If triggers, send a patch to add both a + * PCR_ATTR_BUILD() macro above for the + * missing algorithm as well as an additional + * case in this switch statement. + */ + dev_err(&chip->dev, + "TPM with unsupported bank algorithm 0x%04x", + chip->allocated_banks[i].alg_id); + break; + } + } + + /* + * This will only trigger if someone has added an additional + * hash to the tpm_algorithms enum without incrementing + * TPM_MAX_HASHES. + */ + WARN_ON(chip->groups_cnt > TPM_MAX_HASHES + 1); } diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h index 947d1db0a5cc..283f78211c3a 100644 --- a/drivers/char/tpm/tpm.h +++ b/drivers/char/tpm/tpm.h @@ -164,8 +164,6 @@ extern const struct file_operations tpmrm_fops; extern struct idr dev_nums_idr; ssize_t tpm_transmit(struct tpm_chip *chip, u8 *buf, size_t bufsiz); -ssize_t tpm_transmit_cmd(struct tpm_chip *chip, struct tpm_buf *buf, - size_t min_rsp_body_length, const char *desc); int tpm_get_timeouts(struct tpm_chip *); int tpm_auto_startup(struct tpm_chip *chip); @@ -194,8 +192,6 @@ static inline void tpm_msleep(unsigned int delay_msec) int tpm_chip_start(struct tpm_chip *chip); void tpm_chip_stop(struct tpm_chip *chip); struct tpm_chip *tpm_find_get_ops(struct tpm_chip *chip); -__must_check int tpm_try_get_ops(struct tpm_chip *chip); -void tpm_put_ops(struct tpm_chip *chip); struct tpm_chip *tpm_chip_alloc(struct device *dev, const struct tpm_class_ops *ops); diff --git a/drivers/char/tpm/tpm_ppi.c b/drivers/char/tpm/tpm_ppi.c index b2dab941cb7f..40018a73b3cb 100644 --- a/drivers/char/tpm/tpm_ppi.c +++ b/drivers/char/tpm/tpm_ppi.c @@ -358,7 +358,7 @@ static struct attribute *ppi_attrs[] = { &dev_attr_tcg_operations.attr, &dev_attr_vs_operations.attr, NULL, }; -static struct attribute_group ppi_attr_grp = { +static const struct attribute_group ppi_attr_grp = { .name = "ppi", .attrs = ppi_attrs }; diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index 92c51c6cfd1b..431919d5f48a 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -125,7 +125,8 @@ static bool check_locality(struct tpm_chip *chip, int l) if (rc < 0) return false; - if ((access & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) == + if ((access & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID + | TPM_ACCESS_REQUEST_USE)) == (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) { priv->locality = l; return true; @@ -134,58 +135,13 @@ static bool check_locality(struct tpm_chip *chip, int l) return false; } -static bool locality_inactive(struct tpm_chip *chip, int l) -{ - struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); - int rc; - u8 access; - - rc = tpm_tis_read8(priv, TPM_ACCESS(l), &access); - if (rc < 0) - return false; - - if ((access & (TPM_ACCESS_VALID | TPM_ACCESS_ACTIVE_LOCALITY)) - == TPM_ACCESS_VALID) - return true; - - return false; -} - static int release_locality(struct tpm_chip *chip, int l) { struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); - unsigned long stop, timeout; - long rc; tpm_tis_write8(priv, TPM_ACCESS(l), TPM_ACCESS_ACTIVE_LOCALITY); - stop = jiffies + chip->timeout_a; - - if (chip->flags & TPM_CHIP_FLAG_IRQ) { -again: - timeout = stop - jiffies; - if ((long)timeout <= 0) - return -1; - - rc = wait_event_interruptible_timeout(priv->int_queue, - (locality_inactive(chip, l)), - timeout); - - if (rc > 0) - return 0; - - if (rc == -ERESTARTSYS && freezing(current)) { - clear_thread_flag(TIF_SIGPENDING); - goto again; - } - } else { - do { - if (locality_inactive(chip, l)) - return 0; - tpm_msleep(TPM_TIMEOUT); - } while (time_before(jiffies, stop)); - } - return -1; + return 0; } static int request_locality(struct tpm_chip *chip, int l) diff --git a/drivers/char/tpm/tpm_tis_i2c_cr50.c b/drivers/char/tpm/tpm_tis_i2c_cr50.c new file mode 100644 index 000000000000..ec9a65e7887d --- /dev/null +++ b/drivers/char/tpm/tpm_tis_i2c_cr50.c @@ -0,0 +1,790 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2020 Google Inc. + * + * Based on Infineon TPM driver by Peter Huewe. + * + * cr50 is a firmware for H1 secure modules that requires special + * handling for the I2C interface. + * + * - Use an interrupt for transaction status instead of hardcoded delays. + * - Must use write+wait+read read protocol. + * - All 4 bytes of status register must be read/written at once. + * - Burst count max is 63 bytes, and burst count behaves slightly differently + * than other I2C TPMs. + * - When reading from FIFO the full burstcnt must be read instead of just + * reading header and determining the remainder. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tpm_tis_core.h" + +#define TPM_CR50_MAX_BUFSIZE 64 +#define TPM_CR50_TIMEOUT_SHORT_MS 2 /* Short timeout during transactions */ +#define TPM_CR50_TIMEOUT_NOIRQ_MS 20 /* Timeout for TPM ready without IRQ */ +#define TPM_CR50_I2C_DID_VID 0x00281ae0L /* Device and vendor ID reg value */ +#define TPM_CR50_I2C_MAX_RETRIES 3 /* Max retries due to I2C errors */ +#define TPM_CR50_I2C_RETRY_DELAY_LO 55 /* Min usecs between retries on I2C */ +#define TPM_CR50_I2C_RETRY_DELAY_HI 65 /* Max usecs between retries on I2C */ + +#define TPM_I2C_ACCESS(l) (0x0000 | ((l) << 4)) +#define TPM_I2C_STS(l) (0x0001 | ((l) << 4)) +#define TPM_I2C_DATA_FIFO(l) (0x0005 | ((l) << 4)) +#define TPM_I2C_DID_VID(l) (0x0006 | ((l) << 4)) + +/** + * struct tpm_i2c_cr50_priv_data - Driver private data. + * @irq: Irq number used for this chip. + * If irq <= 0, then a fixed timeout is used instead of waiting for irq. + * @tpm_ready: Struct used by irq handler to signal R/W readiness. + * @buf: Buffer used for i2c writes, with i2c address prepended to content. + * + * Private driver struct used by kernel threads and interrupt context. + */ +struct tpm_i2c_cr50_priv_data { + int irq; + struct completion tpm_ready; + u8 buf[TPM_CR50_MAX_BUFSIZE]; +}; + +/** + * tpm_cr50_i2c_int_handler() - cr50 interrupt handler. + * @dummy: Unused parameter. + * @tpm_info: TPM chip information. + * + * The cr50 interrupt handler signals waiting threads that the + * interrupt has been asserted. It does not do any interrupt triggered + * processing but is instead used to avoid fixed delays. + * + * Return: + * IRQ_HANDLED signifies irq was handled by this device. + */ +static irqreturn_t tpm_cr50_i2c_int_handler(int dummy, void *tpm_info) +{ + struct tpm_chip *chip = tpm_info; + struct tpm_i2c_cr50_priv_data *priv = dev_get_drvdata(&chip->dev); + + complete(&priv->tpm_ready); + + return IRQ_HANDLED; +} + +/** + * tpm_cr50_i2c_wait_tpm_ready() - Wait for tpm to signal ready. + * @chip: A TPM chip. + * + * Wait for completion interrupt if available, otherwise use a fixed + * delay for the TPM to be ready. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_wait_tpm_ready(struct tpm_chip *chip) +{ + struct tpm_i2c_cr50_priv_data *priv = dev_get_drvdata(&chip->dev); + + /* Use a safe fixed delay if interrupt is not supported */ + if (priv->irq <= 0) { + msleep(TPM_CR50_TIMEOUT_NOIRQ_MS); + return 0; + } + + /* Wait for interrupt to indicate TPM is ready to respond */ + if (!wait_for_completion_timeout(&priv->tpm_ready, + msecs_to_jiffies(chip->timeout_a))) { + dev_warn(&chip->dev, "Timeout waiting for TPM ready\n"); + return -ETIMEDOUT; + } + + return 0; +} + +/** + * tpm_cr50_i2c_enable_tpm_irq() - Enable TPM irq. + * @chip: A TPM chip. + */ +static void tpm_cr50_i2c_enable_tpm_irq(struct tpm_chip *chip) +{ + struct tpm_i2c_cr50_priv_data *priv = dev_get_drvdata(&chip->dev); + + if (priv->irq > 0) { + reinit_completion(&priv->tpm_ready); + enable_irq(priv->irq); + } +} + +/** + * tpm_cr50_i2c_disable_tpm_irq() - Disable TPM irq. + * @chip: A TPM chip. + */ +static void tpm_cr50_i2c_disable_tpm_irq(struct tpm_chip *chip) +{ + struct tpm_i2c_cr50_priv_data *priv = dev_get_drvdata(&chip->dev); + + if (priv->irq > 0) + disable_irq(priv->irq); +} + +/** + * tpm_cr50_i2c_transfer_message() - Transfer a message over i2c. + * @dev: Device information. + * @adapter: I2C adapter. + * @msg: Message to transfer. + * + * Call unlocked i2c transfer routine with the provided parameters and + * retry in case of bus errors. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_transfer_message(struct device *dev, + struct i2c_adapter *adapter, + struct i2c_msg *msg) +{ + unsigned int try; + int rc; + + for (try = 0; try < TPM_CR50_I2C_MAX_RETRIES; try++) { + rc = __i2c_transfer(adapter, msg, 1); + if (rc == 1) + return 0; /* Successfully transferred the message */ + if (try) + dev_warn(dev, "i2c transfer failed (attempt %d/%d): %d\n", + try + 1, TPM_CR50_I2C_MAX_RETRIES, rc); + usleep_range(TPM_CR50_I2C_RETRY_DELAY_LO, TPM_CR50_I2C_RETRY_DELAY_HI); + } + + /* No i2c message transferred */ + return -EIO; +} + +/** + * tpm_cr50_i2c_read() - Read from TPM register. + * @chip: A TPM chip. + * @addr: Register address to read from. + * @buffer: Read destination, provided by caller. + * @len: Number of bytes to read. + * + * Sends the register address byte to the TPM, then waits until TPM + * is ready via interrupt signal or timeout expiration, then 'len' + * bytes are read from TPM response into the provided 'buffer'. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_read(struct tpm_chip *chip, u8 addr, u8 *buffer, size_t len) +{ + struct i2c_client *client = to_i2c_client(chip->dev.parent); + struct i2c_msg msg_reg_addr = { + .addr = client->addr, + .len = 1, + .buf = &addr + }; + struct i2c_msg msg_response = { + .addr = client->addr, + .flags = I2C_M_RD, + .len = len, + .buf = buffer + }; + int rc; + + i2c_lock_bus(client->adapter, I2C_LOCK_SEGMENT); + + /* Prepare for completion interrupt */ + tpm_cr50_i2c_enable_tpm_irq(chip); + + /* Send the register address byte to the TPM */ + rc = tpm_cr50_i2c_transfer_message(&chip->dev, client->adapter, &msg_reg_addr); + if (rc < 0) + goto out; + + /* Wait for TPM to be ready with response data */ + rc = tpm_cr50_i2c_wait_tpm_ready(chip); + if (rc < 0) + goto out; + + /* Read response data from the TPM */ + rc = tpm_cr50_i2c_transfer_message(&chip->dev, client->adapter, &msg_response); + +out: + tpm_cr50_i2c_disable_tpm_irq(chip); + i2c_unlock_bus(client->adapter, I2C_LOCK_SEGMENT); + + if (rc < 0) + return rc; + + return 0; +} + +/** + * tpm_cr50_i2c_write()- Write to TPM register. + * @chip: A TPM chip. + * @addr: Register address to write to. + * @buffer: Data to write. + * @len: Number of bytes to write. + * + * The provided address is prepended to the data in 'buffer', the + * cobined address+data is sent to the TPM, then wait for TPM to + * indicate it is done writing. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_write(struct tpm_chip *chip, u8 addr, u8 *buffer, + size_t len) +{ + struct tpm_i2c_cr50_priv_data *priv = dev_get_drvdata(&chip->dev); + struct i2c_client *client = to_i2c_client(chip->dev.parent); + struct i2c_msg msg = { + .addr = client->addr, + .len = len + 1, + .buf = priv->buf + }; + int rc; + + if (len > TPM_CR50_MAX_BUFSIZE - 1) + return -EINVAL; + + /* Prepend the 'register address' to the buffer */ + priv->buf[0] = addr; + memcpy(priv->buf + 1, buffer, len); + + i2c_lock_bus(client->adapter, I2C_LOCK_SEGMENT); + + /* Prepare for completion interrupt */ + tpm_cr50_i2c_enable_tpm_irq(chip); + + /* Send write request buffer with address */ + rc = tpm_cr50_i2c_transfer_message(&chip->dev, client->adapter, &msg); + if (rc < 0) + goto out; + + /* Wait for TPM to be ready, ignore timeout */ + tpm_cr50_i2c_wait_tpm_ready(chip); + +out: + tpm_cr50_i2c_disable_tpm_irq(chip); + i2c_unlock_bus(client->adapter, I2C_LOCK_SEGMENT); + + if (rc < 0) + return rc; + + return 0; +} + +/** + * tpm_cr50_check_locality() - Verify TPM locality 0 is active. + * @chip: A TPM chip. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_check_locality(struct tpm_chip *chip) +{ + u8 mask = TPM_ACCESS_VALID | TPM_ACCESS_ACTIVE_LOCALITY; + u8 buf; + int rc; + + rc = tpm_cr50_i2c_read(chip, TPM_I2C_ACCESS(0), &buf, sizeof(buf)); + if (rc < 0) + return rc; + + if ((buf & mask) == mask) + return 0; + + return -EIO; +} + +/** + * tpm_cr50_release_locality() - Release TPM locality. + * @chip: A TPM chip. + * @force: Flag to force release if set. + */ +static void tpm_cr50_release_locality(struct tpm_chip *chip, bool force) +{ + u8 mask = TPM_ACCESS_VALID | TPM_ACCESS_REQUEST_PENDING; + u8 addr = TPM_I2C_ACCESS(0); + u8 buf; + + if (tpm_cr50_i2c_read(chip, addr, &buf, sizeof(buf)) < 0) + return; + + if (force || (buf & mask) == mask) { + buf = TPM_ACCESS_ACTIVE_LOCALITY; + tpm_cr50_i2c_write(chip, addr, &buf, sizeof(buf)); + } +} + +/** + * tpm_cr50_request_locality() - Request TPM locality 0. + * @chip: A TPM chip. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_request_locality(struct tpm_chip *chip) +{ + u8 buf = TPM_ACCESS_REQUEST_USE; + unsigned long stop; + int rc; + + if (!tpm_cr50_check_locality(chip)) + return 0; + + rc = tpm_cr50_i2c_write(chip, TPM_I2C_ACCESS(0), &buf, sizeof(buf)); + if (rc < 0) + return rc; + + stop = jiffies + chip->timeout_a; + do { + if (!tpm_cr50_check_locality(chip)) + return 0; + + msleep(TPM_CR50_TIMEOUT_SHORT_MS); + } while (time_before(jiffies, stop)); + + return -ETIMEDOUT; +} + +/** + * tpm_cr50_i2c_tis_status() - Read cr50 tis status. + * @chip: A TPM chip. + * + * cr50 requires all 4 bytes of status register to be read. + * + * Return: + * TPM status byte. + */ +static u8 tpm_cr50_i2c_tis_status(struct tpm_chip *chip) +{ + u8 buf[4]; + + if (tpm_cr50_i2c_read(chip, TPM_I2C_STS(0), buf, sizeof(buf)) < 0) + return 0; + + return buf[0]; +} + +/** + * tpm_cr50_i2c_tis_set_ready() - Set status register to ready. + * @chip: A TPM chip. + * + * cr50 requires all 4 bytes of status register to be written. + */ +static void tpm_cr50_i2c_tis_set_ready(struct tpm_chip *chip) +{ + u8 buf[4] = { TPM_STS_COMMAND_READY }; + + tpm_cr50_i2c_write(chip, TPM_I2C_STS(0), buf, sizeof(buf)); + msleep(TPM_CR50_TIMEOUT_SHORT_MS); +} + +/** + * tpm_cr50_i2c_get_burst_and_status() - Get burst count and status. + * @chip: A TPM chip. + * @mask: Status mask. + * @burst: Return value for burst. + * @status: Return value for status. + * + * cr50 uses bytes 3:2 of status register for burst count and + * all 4 bytes must be read. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_get_burst_and_status(struct tpm_chip *chip, u8 mask, + size_t *burst, u32 *status) +{ + unsigned long stop; + u8 buf[4]; + + *status = 0; + + /* wait for burstcount */ + stop = jiffies + chip->timeout_b; + + do { + if (tpm_cr50_i2c_read(chip, TPM_I2C_STS(0), buf, sizeof(buf)) < 0) { + msleep(TPM_CR50_TIMEOUT_SHORT_MS); + continue; + } + + *status = *buf; + *burst = le16_to_cpup((__le16 *)(buf + 1)); + + if ((*status & mask) == mask && + *burst > 0 && *burst <= TPM_CR50_MAX_BUFSIZE - 1) + return 0; + + msleep(TPM_CR50_TIMEOUT_SHORT_MS); + } while (time_before(jiffies, stop)); + + dev_err(&chip->dev, "Timeout reading burst and status\n"); + return -ETIMEDOUT; +} + +/** + * tpm_cr50_i2c_tis_recv() - TPM reception callback. + * @chip: A TPM chip. + * @buf: Reception buffer. + * @buf_len: Buffer length to read. + * + * Return: + * - >= 0: Number of read bytes. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_tis_recv(struct tpm_chip *chip, u8 *buf, size_t buf_len) +{ + + u8 mask = TPM_STS_VALID | TPM_STS_DATA_AVAIL; + size_t burstcnt, cur, len, expected; + u8 addr = TPM_I2C_DATA_FIFO(0); + u32 status; + int rc; + + if (buf_len < TPM_HEADER_SIZE) + return -EINVAL; + + rc = tpm_cr50_i2c_get_burst_and_status(chip, mask, &burstcnt, &status); + if (rc < 0) + goto out_err; + + if (burstcnt > buf_len || burstcnt < TPM_HEADER_SIZE) { + dev_err(&chip->dev, + "Unexpected burstcnt: %zu (max=%zu, min=%d)\n", + burstcnt, buf_len, TPM_HEADER_SIZE); + rc = -EIO; + goto out_err; + } + + /* Read first chunk of burstcnt bytes */ + rc = tpm_cr50_i2c_read(chip, addr, buf, burstcnt); + if (rc < 0) { + dev_err(&chip->dev, "Read of first chunk failed\n"); + goto out_err; + } + + /* Determine expected data in the return buffer */ + expected = be32_to_cpup((__be32 *)(buf + 2)); + if (expected > buf_len) { + dev_err(&chip->dev, "Buffer too small to receive i2c data\n"); + goto out_err; + } + + /* Now read the rest of the data */ + cur = burstcnt; + while (cur < expected) { + /* Read updated burst count and check status */ + rc = tpm_cr50_i2c_get_burst_and_status(chip, mask, &burstcnt, &status); + if (rc < 0) + goto out_err; + + len = min_t(size_t, burstcnt, expected - cur); + rc = tpm_cr50_i2c_read(chip, addr, buf + cur, len); + if (rc < 0) { + dev_err(&chip->dev, "Read failed\n"); + goto out_err; + } + + cur += len; + } + + /* Ensure TPM is done reading data */ + rc = tpm_cr50_i2c_get_burst_and_status(chip, TPM_STS_VALID, &burstcnt, &status); + if (rc < 0) + goto out_err; + if (status & TPM_STS_DATA_AVAIL) { + dev_err(&chip->dev, "Data still available\n"); + rc = -EIO; + goto out_err; + } + + tpm_cr50_release_locality(chip, false); + return cur; + +out_err: + /* Abort current transaction if still pending */ + if (tpm_cr50_i2c_tis_status(chip) & TPM_STS_COMMAND_READY) + tpm_cr50_i2c_tis_set_ready(chip); + + tpm_cr50_release_locality(chip, false); + return rc; +} + +/** + * tpm_cr50_i2c_tis_send() - TPM transmission callback. + * @chip: A TPM chip. + * @buf: Buffer to send. + * @len: Buffer length. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_tis_send(struct tpm_chip *chip, u8 *buf, size_t len) +{ + size_t burstcnt, limit, sent = 0; + u8 tpm_go[4] = { TPM_STS_GO }; + unsigned long stop; + u32 status; + int rc; + + rc = tpm_cr50_request_locality(chip); + if (rc < 0) + return rc; + + /* Wait until TPM is ready for a command */ + stop = jiffies + chip->timeout_b; + while (!(tpm_cr50_i2c_tis_status(chip) & TPM_STS_COMMAND_READY)) { + if (time_after(jiffies, stop)) { + rc = -ETIMEDOUT; + goto out_err; + } + + tpm_cr50_i2c_tis_set_ready(chip); + } + + while (len > 0) { + u8 mask = TPM_STS_VALID; + + /* Wait for data if this is not the first chunk */ + if (sent > 0) + mask |= TPM_STS_DATA_EXPECT; + + /* Read burst count and check status */ + rc = tpm_cr50_i2c_get_burst_and_status(chip, mask, &burstcnt, &status); + if (rc < 0) + goto out_err; + + /* + * Use burstcnt - 1 to account for the address byte + * that is inserted by tpm_cr50_i2c_write() + */ + limit = min_t(size_t, burstcnt - 1, len); + rc = tpm_cr50_i2c_write(chip, TPM_I2C_DATA_FIFO(0), &buf[sent], limit); + if (rc < 0) { + dev_err(&chip->dev, "Write failed\n"); + goto out_err; + } + + sent += limit; + len -= limit; + } + + /* Ensure TPM is not expecting more data */ + rc = tpm_cr50_i2c_get_burst_and_status(chip, TPM_STS_VALID, &burstcnt, &status); + if (rc < 0) + goto out_err; + if (status & TPM_STS_DATA_EXPECT) { + dev_err(&chip->dev, "Data still expected\n"); + rc = -EIO; + goto out_err; + } + + /* Start the TPM command */ + rc = tpm_cr50_i2c_write(chip, TPM_I2C_STS(0), tpm_go, + sizeof(tpm_go)); + if (rc < 0) { + dev_err(&chip->dev, "Start command failed\n"); + goto out_err; + } + return 0; + +out_err: + /* Abort current transaction if still pending */ + if (tpm_cr50_i2c_tis_status(chip) & TPM_STS_COMMAND_READY) + tpm_cr50_i2c_tis_set_ready(chip); + + tpm_cr50_release_locality(chip, false); + return rc; +} + +/** + * tpm_cr50_i2c_req_canceled() - Callback to notify a request cancel. + * @chip: A TPM chip. + * @status: Status given by the cancel callback. + * + * Return: + * True if command is ready, False otherwise. + */ +static bool tpm_cr50_i2c_req_canceled(struct tpm_chip *chip, u8 status) +{ + return status == TPM_STS_COMMAND_READY; +} + +static const struct tpm_class_ops cr50_i2c = { + .flags = TPM_OPS_AUTO_STARTUP, + .status = &tpm_cr50_i2c_tis_status, + .recv = &tpm_cr50_i2c_tis_recv, + .send = &tpm_cr50_i2c_tis_send, + .cancel = &tpm_cr50_i2c_tis_set_ready, + .req_complete_mask = TPM_STS_DATA_AVAIL | TPM_STS_VALID, + .req_complete_val = TPM_STS_DATA_AVAIL | TPM_STS_VALID, + .req_canceled = &tpm_cr50_i2c_req_canceled, +}; + +static const struct i2c_device_id cr50_i2c_table[] = { + {"cr50_i2c", 0}, + {} +}; +MODULE_DEVICE_TABLE(i2c, cr50_i2c_table); + +#ifdef CONFIG_ACPI +static const struct acpi_device_id cr50_i2c_acpi_id[] = { + { "GOOG0005", 0 }, + {} +}; +MODULE_DEVICE_TABLE(acpi, cr50_i2c_acpi_id); +#endif + +#ifdef CONFIG_OF +static const struct of_device_id of_cr50_i2c_match[] = { + { .compatible = "google,cr50", }, + {} +}; +MODULE_DEVICE_TABLE(of, of_cr50_i2c_match); +#endif + +/** + * tpm_cr50_i2c_probe() - Driver probe function. + * @client: I2C client information. + * @id: I2C device id. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct tpm_i2c_cr50_priv_data *priv; + struct device *dev = &client->dev; + struct tpm_chip *chip; + u32 vendor; + u8 buf[4]; + int rc; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) + return -ENODEV; + + chip = tpmm_chip_alloc(dev, &cr50_i2c); + if (IS_ERR(chip)) + return PTR_ERR(chip); + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + /* cr50 is a TPM 2.0 chip */ + chip->flags |= TPM_CHIP_FLAG_TPM2; + chip->flags |= TPM_CHIP_FLAG_FIRMWARE_POWER_MANAGED; + + /* Default timeouts */ + chip->timeout_a = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->timeout_b = msecs_to_jiffies(TIS_LONG_TIMEOUT); + chip->timeout_c = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->timeout_d = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + + dev_set_drvdata(&chip->dev, priv); + init_completion(&priv->tpm_ready); + + if (client->irq > 0) { + rc = devm_request_irq(dev, client->irq, tpm_cr50_i2c_int_handler, + IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + dev->driver->name, chip); + if (rc < 0) { + dev_err(dev, "Failed to probe IRQ %d\n", client->irq); + return rc; + } + + disable_irq(client->irq); + priv->irq = client->irq; + } else { + dev_warn(dev, "No IRQ, will use %ums delay for TPM ready\n", + TPM_CR50_TIMEOUT_NOIRQ_MS); + } + + rc = tpm_cr50_request_locality(chip); + if (rc < 0) { + dev_err(dev, "Could not request locality\n"); + return rc; + } + + /* Read four bytes from DID_VID register */ + rc = tpm_cr50_i2c_read(chip, TPM_I2C_DID_VID(0), buf, sizeof(buf)); + if (rc < 0) { + dev_err(dev, "Could not read vendor id\n"); + tpm_cr50_release_locality(chip, true); + return rc; + } + + vendor = le32_to_cpup((__le32 *)buf); + if (vendor != TPM_CR50_I2C_DID_VID) { + dev_err(dev, "Vendor ID did not match! ID was %08x\n", vendor); + tpm_cr50_release_locality(chip, true); + return -ENODEV; + } + + dev_info(dev, "cr50 TPM 2.0 (i2c 0x%02x irq %d id 0x%x)\n", + client->addr, client->irq, vendor >> 16); + + return tpm_chip_register(chip); +} + +/** + * tpm_cr50_i2c_remove() - Driver remove function. + * @client: I2C client information. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_remove(struct i2c_client *client) +{ + struct tpm_chip *chip = i2c_get_clientdata(client); + struct device *dev = &client->dev; + + if (!chip) { + dev_err(dev, "Could not get client data at remove\n"); + return -ENODEV; + } + + tpm_chip_unregister(chip); + tpm_cr50_release_locality(chip, true); + + return 0; +} + +static SIMPLE_DEV_PM_OPS(cr50_i2c_pm, tpm_pm_suspend, tpm_pm_resume); + +static struct i2c_driver cr50_i2c_driver = { + .id_table = cr50_i2c_table, + .probe = tpm_cr50_i2c_probe, + .remove = tpm_cr50_i2c_remove, + .driver = { + .name = "cr50_i2c", + .pm = &cr50_i2c_pm, + .acpi_match_table = ACPI_PTR(cr50_i2c_acpi_id), + .of_match_table = of_match_ptr(of_cr50_i2c_match), + }, +}; + +module_i2c_driver(cr50_i2c_driver); + +MODULE_DESCRIPTION("cr50 TPM I2C Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 85856cff506c..a588d56502d4 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -247,7 +247,8 @@ config CLK_TWL6040 config COMMON_CLK_AXI_CLKGEN tristate "AXI clkgen driver" - depends on ARCH_ZYNQ || MICROBLAZE || COMPILE_TEST + depends on HAS_IOMEM || COMPILE_TEST + depends on OF help Support for the Analog Devices axi-clkgen pcore clock generator for Xilinx FPGAs. It is commonly used in Analog Devices' reference designs. @@ -368,6 +369,13 @@ config COMMON_CLK_FIXED_MMIO help Support for Memory Mapped IO Fixed clocks +config COMMON_CLK_K210 + bool "Clock driver for the Canaan Kendryte K210 SoC" + depends on OF && RISCV && SOC_CANAAN + default SOC_CANAAN + help + Support for the Canaan Kendryte K210 RISC-V SoC clocks. + source "drivers/clk/actions/Kconfig" source "drivers/clk/analogbits/Kconfig" source "drivers/clk/baikal-t1/Kconfig" @@ -379,6 +387,7 @@ source "drivers/clk/ingenic/Kconfig" source "drivers/clk/keystone/Kconfig" source "drivers/clk/mediatek/Kconfig" source "drivers/clk/meson/Kconfig" +source "drivers/clk/mstar/Kconfig" source "drivers/clk/mvebu/Kconfig" source "drivers/clk/qcom/Kconfig" source "drivers/clk/renesas/Kconfig" @@ -392,6 +401,7 @@ source "drivers/clk/tegra/Kconfig" source "drivers/clk/ti/Kconfig" source "drivers/clk/uniphier/Kconfig" source "drivers/clk/x86/Kconfig" +source "drivers/clk/xilinx/Kconfig" source "drivers/clk/zynqmp/Kconfig" endif diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index dbdc590e7de3..b22ae4f81e0b 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -27,7 +27,6 @@ obj-$(CONFIG_COMMON_CLK_CDCE706) += clk-cdce706.o obj-$(CONFIG_COMMON_CLK_CDCE925) += clk-cdce925.o obj-$(CONFIG_ARCH_CLPS711X) += clk-clps711x.o obj-$(CONFIG_COMMON_CLK_CS2000_CP) += clk-cs2000-cp.o -obj-$(CONFIG_ARCH_EFM32) += clk-efm32gg.o obj-$(CONFIG_ARCH_SPARX5) += clk-sparx5.o obj-$(CONFIG_COMMON_CLK_FIXED_MMIO) += clk-fixed-mmio.o obj-$(CONFIG_COMMON_CLK_FSL_FLEXSPI) += clk-fsl-flexspi.o @@ -37,6 +36,7 @@ obj-$(CONFIG_COMMON_CLK_ASPEED) += clk-aspeed.o obj-$(CONFIG_MACH_ASPEED_G6) += clk-ast2600.o obj-$(CONFIG_ARCH_HIGHBANK) += clk-highbank.o obj-$(CONFIG_CLK_HSDK) += clk-hsdk-pll.o +obj-$(CONFIG_COMMON_CLK_K210) += clk-k210.o obj-$(CONFIG_COMMON_CLK_LOCHNAGAR) += clk-lochnagar.o obj-$(CONFIG_COMMON_CLK_MAX77686) += clk-max77686.o obj-$(CONFIG_COMMON_CLK_MAX9485) += clk-max9485.o @@ -63,9 +63,7 @@ obj-$(CONFIG_COMMON_CLK_SI570) += clk-si570.o obj-$(CONFIG_COMMON_CLK_STM32F) += clk-stm32f4.o obj-$(CONFIG_COMMON_CLK_STM32H7) += clk-stm32h7.o obj-$(CONFIG_COMMON_CLK_STM32MP157) += clk-stm32mp1.o -obj-$(CONFIG_ARCH_TANGO) += clk-tango4.o obj-$(CONFIG_CLK_TWL6040) += clk-twl6040.o -obj-$(CONFIG_ARCH_U300) += clk-u300.o obj-$(CONFIG_ARCH_VT8500) += clk-vt8500.o obj-$(CONFIG_COMMON_CLK_VC5) += clk-versaclock5.o obj-$(CONFIG_COMMON_CLK_WM831X) += clk-wm831x.o @@ -95,6 +93,7 @@ obj-$(CONFIG_MACH_PIC32) += microchip/ ifeq ($(CONFIG_COMMON_CLK), y) obj-$(CONFIG_ARCH_MMP) += mmp/ endif +obj-y += mstar/ obj-y += mvebu/ obj-$(CONFIG_ARCH_MXS) += mxs/ obj-$(CONFIG_COMMON_CLK_NXP) += nxp/ @@ -105,7 +104,6 @@ obj-y += renesas/ obj-$(CONFIG_ARCH_ROCKCHIP) += rockchip/ obj-$(CONFIG_COMMON_CLK_SAMSUNG) += samsung/ obj-$(CONFIG_CLK_SIFIVE) += sifive/ -obj-$(CONFIG_ARCH_SIRF) += sirf/ obj-$(CONFIG_ARCH_SOCFPGA) += socfpga/ obj-$(CONFIG_ARCH_AGILEX) += socfpga/ obj-$(CONFIG_ARCH_STRATIX10) += socfpga/ @@ -122,6 +120,6 @@ obj-y += versatile/ ifeq ($(CONFIG_COMMON_CLK), y) obj-$(CONFIG_X86) += x86/ endif -obj-$(CONFIG_ARCH_ZX) += zte/ +obj-y += xilinx/ obj-$(CONFIG_ARCH_ZYNQ) += zynq/ obj-$(CONFIG_COMMON_CLK_ZYNQMP) += zynqmp/ diff --git a/drivers/clk/at91/at91rm9200.c b/drivers/clk/at91/at91rm9200.c index 0fad1009f315..428a6f4b9ebc 100644 --- a/drivers/clk/at91/at91rm9200.c +++ b/drivers/clk/at91/at91rm9200.c @@ -215,5 +215,4 @@ err_free: * deferring properly. Once this is fixed, this can be switched to a platform * driver. */ -CLK_OF_DECLARE_DRIVER(at91rm9200_pmc, "atmel,at91rm9200-pmc", - at91rm9200_pmc_setup); +CLK_OF_DECLARE(at91rm9200_pmc, "atmel,at91rm9200-pmc", at91rm9200_pmc_setup); diff --git a/drivers/clk/at91/at91sam9260.c b/drivers/clk/at91/at91sam9260.c index ceb5495f723a..b29843bea278 100644 --- a/drivers/clk/at91/at91sam9260.c +++ b/drivers/clk/at91/at91sam9260.c @@ -491,26 +491,26 @@ static void __init at91sam9260_pmc_setup(struct device_node *np) { at91sam926x_pmc_setup(np, &at91sam9260_data); } -CLK_OF_DECLARE_DRIVER(at91sam9260_pmc, "atmel,at91sam9260-pmc", - at91sam9260_pmc_setup); + +CLK_OF_DECLARE(at91sam9260_pmc, "atmel,at91sam9260-pmc", at91sam9260_pmc_setup); static void __init at91sam9261_pmc_setup(struct device_node *np) { at91sam926x_pmc_setup(np, &at91sam9261_data); } -CLK_OF_DECLARE_DRIVER(at91sam9261_pmc, "atmel,at91sam9261-pmc", - at91sam9261_pmc_setup); + +CLK_OF_DECLARE(at91sam9261_pmc, "atmel,at91sam9261-pmc", at91sam9261_pmc_setup); static void __init at91sam9263_pmc_setup(struct device_node *np) { at91sam926x_pmc_setup(np, &at91sam9263_data); } -CLK_OF_DECLARE_DRIVER(at91sam9263_pmc, "atmel,at91sam9263-pmc", - at91sam9263_pmc_setup); + +CLK_OF_DECLARE(at91sam9263_pmc, "atmel,at91sam9263-pmc", at91sam9263_pmc_setup); static void __init at91sam9g20_pmc_setup(struct device_node *np) { at91sam926x_pmc_setup(np, &at91sam9g20_data); } -CLK_OF_DECLARE_DRIVER(at91sam9g20_pmc, "atmel,at91sam9g20-pmc", - at91sam9g20_pmc_setup); + +CLK_OF_DECLARE(at91sam9g20_pmc, "atmel,at91sam9g20-pmc", at91sam9g20_pmc_setup); diff --git a/drivers/clk/at91/at91sam9g45.c b/drivers/clk/at91/at91sam9g45.c index 0214333dedd3..15da0dfe3ef2 100644 --- a/drivers/clk/at91/at91sam9g45.c +++ b/drivers/clk/at91/at91sam9g45.c @@ -228,5 +228,4 @@ err_free: * The TCB is used as the clocksource so its clock is needed early. This means * this can't be a platform driver. */ -CLK_OF_DECLARE_DRIVER(at91sam9g45_pmc, "atmel,at91sam9g45-pmc", - at91sam9g45_pmc_setup); +CLK_OF_DECLARE(at91sam9g45_pmc, "atmel,at91sam9g45-pmc", at91sam9g45_pmc_setup); diff --git a/drivers/clk/at91/at91sam9n12.c b/drivers/clk/at91/at91sam9n12.c index f9db5316a7f1..7fe435f4b46b 100644 --- a/drivers/clk/at91/at91sam9n12.c +++ b/drivers/clk/at91/at91sam9n12.c @@ -255,5 +255,4 @@ err_free: * The TCB is used as the clocksource so its clock is needed early. This means * this can't be a platform driver. */ -CLK_OF_DECLARE_DRIVER(at91sam9n12_pmc, "atmel,at91sam9n12-pmc", - at91sam9n12_pmc_setup); +CLK_OF_DECLARE(at91sam9n12_pmc, "atmel,at91sam9n12-pmc", at91sam9n12_pmc_setup); diff --git a/drivers/clk/at91/at91sam9rl.c b/drivers/clk/at91/at91sam9rl.c index 66736e03cfef..ecbabf5162bd 100644 --- a/drivers/clk/at91/at91sam9rl.c +++ b/drivers/clk/at91/at91sam9rl.c @@ -186,4 +186,5 @@ static void __init at91sam9rl_pmc_setup(struct device_node *np) err_free: kfree(at91sam9rl_pmc); } -CLK_OF_DECLARE_DRIVER(at91sam9rl_pmc, "atmel,at91sam9rl-pmc", at91sam9rl_pmc_setup); + +CLK_OF_DECLARE(at91sam9rl_pmc, "atmel,at91sam9rl-pmc", at91sam9rl_pmc_setup); diff --git a/drivers/clk/at91/at91sam9x5.c b/drivers/clk/at91/at91sam9x5.c index 79b9d3667228..5cce48c64ea2 100644 --- a/drivers/clk/at91/at91sam9x5.c +++ b/drivers/clk/at91/at91sam9x5.c @@ -302,33 +302,33 @@ static void __init at91sam9g15_pmc_setup(struct device_node *np) { at91sam9x5_pmc_setup(np, at91sam9g15_periphck, true); } -CLK_OF_DECLARE_DRIVER(at91sam9g15_pmc, "atmel,at91sam9g15-pmc", - at91sam9g15_pmc_setup); + +CLK_OF_DECLARE(at91sam9g15_pmc, "atmel,at91sam9g15-pmc", at91sam9g15_pmc_setup); static void __init at91sam9g25_pmc_setup(struct device_node *np) { at91sam9x5_pmc_setup(np, at91sam9g25_periphck, false); } -CLK_OF_DECLARE_DRIVER(at91sam9g25_pmc, "atmel,at91sam9g25-pmc", - at91sam9g25_pmc_setup); + +CLK_OF_DECLARE(at91sam9g25_pmc, "atmel,at91sam9g25-pmc", at91sam9g25_pmc_setup); static void __init at91sam9g35_pmc_setup(struct device_node *np) { at91sam9x5_pmc_setup(np, at91sam9g35_periphck, true); } -CLK_OF_DECLARE_DRIVER(at91sam9g35_pmc, "atmel,at91sam9g35-pmc", - at91sam9g35_pmc_setup); + +CLK_OF_DECLARE(at91sam9g35_pmc, "atmel,at91sam9g35-pmc", at91sam9g35_pmc_setup); static void __init at91sam9x25_pmc_setup(struct device_node *np) { at91sam9x5_pmc_setup(np, at91sam9x25_periphck, false); } -CLK_OF_DECLARE_DRIVER(at91sam9x25_pmc, "atmel,at91sam9x25-pmc", - at91sam9x25_pmc_setup); + +CLK_OF_DECLARE(at91sam9x25_pmc, "atmel,at91sam9x25-pmc", at91sam9x25_pmc_setup); static void __init at91sam9x35_pmc_setup(struct device_node *np) { at91sam9x5_pmc_setup(np, at91sam9x35_periphck, true); } -CLK_OF_DECLARE_DRIVER(at91sam9x35_pmc, "atmel,at91sam9x35-pmc", - at91sam9x35_pmc_setup); + +CLK_OF_DECLARE(at91sam9x35_pmc, "atmel,at91sam9x35-pmc", at91sam9x35_pmc_setup); diff --git a/drivers/clk/at91/sama5d2.c b/drivers/clk/at91/sama5d2.c index 9a5cbc7cd55a..3d1f78176c3e 100644 --- a/drivers/clk/at91/sama5d2.c +++ b/drivers/clk/at91/sama5d2.c @@ -372,4 +372,5 @@ static void __init sama5d2_pmc_setup(struct device_node *np) err_free: kfree(sama5d2_pmc); } -CLK_OF_DECLARE_DRIVER(sama5d2_pmc, "atmel,sama5d2-pmc", sama5d2_pmc_setup); + +CLK_OF_DECLARE(sama5d2_pmc, "atmel,sama5d2-pmc", sama5d2_pmc_setup); diff --git a/drivers/clk/at91/sama5d3.c b/drivers/clk/at91/sama5d3.c index 87009ee8effc..d376257807d2 100644 --- a/drivers/clk/at91/sama5d3.c +++ b/drivers/clk/at91/sama5d3.c @@ -255,4 +255,4 @@ err_free: * The TCB is used as the clocksource so its clock is needed early. This means * this can't be a platform driver. */ -CLK_OF_DECLARE_DRIVER(sama5d3_pmc, "atmel,sama5d3-pmc", sama5d3_pmc_setup); +CLK_OF_DECLARE(sama5d3_pmc, "atmel,sama5d3-pmc", sama5d3_pmc_setup); diff --git a/drivers/clk/at91/sama5d4.c b/drivers/clk/at91/sama5d4.c index 57fff790188b..5cbaac68da44 100644 --- a/drivers/clk/at91/sama5d4.c +++ b/drivers/clk/at91/sama5d4.c @@ -286,4 +286,5 @@ static void __init sama5d4_pmc_setup(struct device_node *np) err_free: kfree(sama5d4_pmc); } -CLK_OF_DECLARE_DRIVER(sama5d4_pmc, "atmel,sama5d4-pmc", sama5d4_pmc_setup); + +CLK_OF_DECLARE(sama5d4_pmc, "atmel,sama5d4-pmc", sama5d4_pmc_setup); diff --git a/drivers/clk/bcm/clk-iproc-pll.c b/drivers/clk/bcm/clk-iproc-pll.c index 274441e2ddb2..33da30f99c79 100644 --- a/drivers/clk/bcm/clk-iproc-pll.c +++ b/drivers/clk/bcm/clk-iproc-pll.c @@ -704,7 +704,7 @@ static const struct clk_ops iproc_clk_ops = { .set_rate = iproc_clk_set_rate, }; -/** +/* * Some PLLs require the PLL SW override bit to be set before changes can be * applied to the PLL */ diff --git a/drivers/clk/clk-ast2600.c b/drivers/clk/clk-ast2600.c index 177368cac6dd..a55b37fc2c8b 100644 --- a/drivers/clk/clk-ast2600.c +++ b/drivers/clk/clk-ast2600.c @@ -17,7 +17,8 @@ #define ASPEED_G6_NUM_CLKS 71 -#define ASPEED_G6_SILICON_REV 0x004 +#define ASPEED_G6_SILICON_REV 0x014 +#define CHIP_REVISION_ID GENMASK(23, 16) #define ASPEED_G6_RESET_CTRL 0x040 #define ASPEED_G6_RESET_CTRL2 0x050 @@ -190,18 +191,34 @@ static struct clk_hw *ast2600_calc_pll(const char *name, u32 val) static struct clk_hw *ast2600_calc_apll(const char *name, u32 val) { unsigned int mult, div; + u32 chip_id = readl(scu_g6_base + ASPEED_G6_SILICON_REV); - if (val & BIT(20)) { - /* Pass through mode */ - mult = div = 1; + if (((chip_id & CHIP_REVISION_ID) >> 16) >= 2) { + if (val & BIT(24)) { + /* Pass through mode */ + mult = div = 1; + } else { + /* F = 25Mhz * [(m + 1) / (n + 1)] / (p + 1) */ + u32 m = val & 0x1fff; + u32 n = (val >> 13) & 0x3f; + u32 p = (val >> 19) & 0xf; + + mult = (m + 1); + div = (n + 1) * (p + 1); + } } else { - /* F = 25Mhz * (2-od) * [(m + 2) / (n + 1)] */ - u32 m = (val >> 5) & 0x3f; - u32 od = (val >> 4) & 0x1; - u32 n = val & 0xf; + if (val & BIT(20)) { + /* Pass through mode */ + mult = div = 1; + } else { + /* F = 25Mhz * (2-od) * [(m + 2) / (n + 1)] */ + u32 m = (val >> 5) & 0x3f; + u32 od = (val >> 4) & 0x1; + u32 n = val & 0xf; - mult = (2 - od) * (m + 2); - div = n + 1; + mult = (2 - od) * (m + 2); + div = n + 1; + } } return clk_hw_register_fixed_factor(NULL, name, "clkin", 0, mult, div); diff --git a/drivers/clk/clk-axi-clkgen.c b/drivers/clk/clk-axi-clkgen.c index ad86e031ba3e..ac6ff736ac8f 100644 --- a/drivers/clk/clk-axi-clkgen.c +++ b/drivers/clk/clk-axi-clkgen.c @@ -108,6 +108,13 @@ static uint32_t axi_clkgen_lookup_lock(unsigned int m) return 0x1f1f00fa; } +static const struct axi_clkgen_limits axi_clkgen_zynqmp_default_limits = { + .fpfd_min = 10000, + .fpfd_max = 450000, + .fvco_min = 800000, + .fvco_max = 1600000, +}; + static const struct axi_clkgen_limits axi_clkgen_zynq_default_limits = { .fpfd_min = 10000, .fpfd_max = 300000, @@ -503,7 +510,6 @@ static int axi_clkgen_probe(struct platform_device *pdev) struct clk_init_data init; const char *parent_names[2]; const char *clk_name; - struct resource *mem; unsigned int i; int ret; @@ -515,8 +521,7 @@ static int axi_clkgen_probe(struct platform_device *pdev) if (!axi_clkgen) return -ENOMEM; - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - axi_clkgen->base = devm_ioremap_resource(&pdev->dev, mem); + axi_clkgen->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(axi_clkgen->base)) return PTR_ERR(axi_clkgen->base); @@ -560,6 +565,10 @@ static int axi_clkgen_remove(struct platform_device *pdev) } static const struct of_device_id axi_clkgen_ids[] = { + { + .compatible = "adi,zynqmp-axi-clkgen-2.00.a", + .data = &axi_clkgen_zynqmp_default_limits, + }, { .compatible = "adi,axi-clkgen-2.00.a", .data = &axi_clkgen_zynq_default_limits, diff --git a/drivers/clk/clk-bd718x7.c b/drivers/clk/clk-bd718x7.c index b52e8d6f660c..17d90e09f1c0 100644 --- a/drivers/clk/clk-bd718x7.c +++ b/drivers/clk/clk-bd718x7.c @@ -31,12 +31,12 @@ struct bd718xx_clk { u8 reg; u8 mask; struct platform_device *pdev; - struct rohm_regmap_dev *mfd; + struct regmap *regmap; }; static int bd71837_clk_set(struct bd718xx_clk *c, unsigned int status) { - return regmap_update_bits(c->mfd->regmap, c->reg, c->mask, status); + return regmap_update_bits(c->regmap, c->reg, c->mask, status); } static void bd71837_clk_disable(struct clk_hw *hw) @@ -62,7 +62,7 @@ static int bd71837_clk_is_enabled(struct clk_hw *hw) int rval; struct bd718xx_clk *c = container_of(hw, struct bd718xx_clk, hw); - rval = regmap_read(c->mfd->regmap, c->reg, &enabled); + rval = regmap_read(c->regmap, c->reg, &enabled); if (rval) return rval; @@ -82,7 +82,6 @@ static int bd71837_clk_probe(struct platform_device *pdev) int rval = -ENOMEM; const char *parent_clk; struct device *parent = pdev->dev.parent; - struct rohm_regmap_dev *mfd = dev_get_drvdata(parent); struct clk_init_data init = { .name = "bd718xx-32k-out", .ops = &bd71837_clk_ops, @@ -93,6 +92,10 @@ static int bd71837_clk_probe(struct platform_device *pdev) if (!c) return -ENOMEM; + c->regmap = dev_get_regmap(pdev->dev.parent, NULL); + if (!c->regmap) + return -ENODEV; + init.num_parents = 1; parent_clk = of_clk_get_parent_name(parent->of_node, 0); @@ -119,7 +122,6 @@ static int bd71837_clk_probe(struct platform_device *pdev) dev_err(&pdev->dev, "Unknown clk chip\n"); return -EINVAL; } - c->mfd = mfd; c->pdev = pdev; c->hw.init = &init; diff --git a/drivers/clk/clk-divider.c b/drivers/clk/clk-divider.c index c499799693cc..344997203f0e 100644 --- a/drivers/clk/clk-divider.c +++ b/drivers/clk/clk-divider.c @@ -494,8 +494,13 @@ struct clk_hw *__clk_hw_register_divider(struct device *dev, else init.ops = &clk_divider_ops; init.flags = flags; - init.parent_names = (parent_name ? &parent_name: NULL); - init.num_parents = (parent_name ? 1 : 0); + init.parent_names = parent_name ? &parent_name : NULL; + init.parent_hws = parent_hw ? &parent_hw : NULL; + init.parent_data = parent_data; + if (parent_name || parent_hw || parent_data) + init.num_parents = 1; + else + init.num_parents = 0; /* struct clk_divider assignments */ div->reg = reg; diff --git a/drivers/clk/clk-efm32gg.c b/drivers/clk/clk-efm32gg.c deleted file mode 100644 index 85beaacb4088..000000000000 --- a/drivers/clk/clk-efm32gg.c +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2013 Pengutronix - * Uwe Kleine-Koenig - */ -#include -#include -#include -#include -#include - -#include - -#define CMU_HFPERCLKEN0 0x44 -#define CMU_MAX_CLKS 37 - -static struct clk_hw_onecell_data *clk_data; - -static void __init efm32gg_cmu_init(struct device_node *np) -{ - int i; - void __iomem *base; - struct clk_hw **hws; - - clk_data = kzalloc(struct_size(clk_data, hws, CMU_MAX_CLKS), - GFP_KERNEL); - - if (!clk_data) - return; - - hws = clk_data->hws; - - for (i = 0; i < CMU_MAX_CLKS; ++i) - hws[i] = ERR_PTR(-ENOENT); - - base = of_iomap(np, 0); - if (!base) { - pr_warn("Failed to map address range for efm32gg,cmu node\n"); - return; - } - - hws[clk_HFXO] = clk_hw_register_fixed_rate(NULL, "HFXO", NULL, 0, - 48000000); - - hws[clk_HFPERCLKUSART0] = clk_hw_register_gate(NULL, "HFPERCLK.USART0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 0, 0, NULL); - hws[clk_HFPERCLKUSART1] = clk_hw_register_gate(NULL, "HFPERCLK.USART1", - "HFXO", 0, base + CMU_HFPERCLKEN0, 1, 0, NULL); - hws[clk_HFPERCLKUSART2] = clk_hw_register_gate(NULL, "HFPERCLK.USART2", - "HFXO", 0, base + CMU_HFPERCLKEN0, 2, 0, NULL); - hws[clk_HFPERCLKUART0] = clk_hw_register_gate(NULL, "HFPERCLK.UART0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 3, 0, NULL); - hws[clk_HFPERCLKUART1] = clk_hw_register_gate(NULL, "HFPERCLK.UART1", - "HFXO", 0, base + CMU_HFPERCLKEN0, 4, 0, NULL); - hws[clk_HFPERCLKTIMER0] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 5, 0, NULL); - hws[clk_HFPERCLKTIMER1] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER1", - "HFXO", 0, base + CMU_HFPERCLKEN0, 6, 0, NULL); - hws[clk_HFPERCLKTIMER2] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER2", - "HFXO", 0, base + CMU_HFPERCLKEN0, 7, 0, NULL); - hws[clk_HFPERCLKTIMER3] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER3", - "HFXO", 0, base + CMU_HFPERCLKEN0, 8, 0, NULL); - hws[clk_HFPERCLKACMP0] = clk_hw_register_gate(NULL, "HFPERCLK.ACMP0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 9, 0, NULL); - hws[clk_HFPERCLKACMP1] = clk_hw_register_gate(NULL, "HFPERCLK.ACMP1", - "HFXO", 0, base + CMU_HFPERCLKEN0, 10, 0, NULL); - hws[clk_HFPERCLKI2C0] = clk_hw_register_gate(NULL, "HFPERCLK.I2C0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 11, 0, NULL); - hws[clk_HFPERCLKI2C1] = clk_hw_register_gate(NULL, "HFPERCLK.I2C1", - "HFXO", 0, base + CMU_HFPERCLKEN0, 12, 0, NULL); - hws[clk_HFPERCLKGPIO] = clk_hw_register_gate(NULL, "HFPERCLK.GPIO", - "HFXO", 0, base + CMU_HFPERCLKEN0, 13, 0, NULL); - hws[clk_HFPERCLKVCMP] = clk_hw_register_gate(NULL, "HFPERCLK.VCMP", - "HFXO", 0, base + CMU_HFPERCLKEN0, 14, 0, NULL); - hws[clk_HFPERCLKPRS] = clk_hw_register_gate(NULL, "HFPERCLK.PRS", - "HFXO", 0, base + CMU_HFPERCLKEN0, 15, 0, NULL); - hws[clk_HFPERCLKADC0] = clk_hw_register_gate(NULL, "HFPERCLK.ADC0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 16, 0, NULL); - hws[clk_HFPERCLKDAC0] = clk_hw_register_gate(NULL, "HFPERCLK.DAC0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 17, 0, NULL); - - of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); -} -CLK_OF_DECLARE(efm32ggcmu, "efm32gg,cmu", efm32gg_cmu_init); diff --git a/drivers/clk/clk-fixed-factor.c b/drivers/clk/clk-fixed-factor.c index 910e6e74ae90..4f7bf3929d6d 100644 --- a/drivers/clk/clk-fixed-factor.c +++ b/drivers/clk/clk-fixed-factor.c @@ -64,10 +64,16 @@ const struct clk_ops clk_fixed_factor_ops = { }; EXPORT_SYMBOL_GPL(clk_fixed_factor_ops); +static void devm_clk_hw_register_fixed_factor_release(struct device *dev, void *res) +{ + clk_hw_unregister_fixed_factor(&((struct clk_fixed_factor *)res)->hw); +} + static struct clk_hw * __clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, const char *name, const char *parent_name, int index, - unsigned long flags, unsigned int mult, unsigned int div) + unsigned long flags, unsigned int mult, unsigned int div, + bool devm) { struct clk_fixed_factor *fix; struct clk_init_data init = { }; @@ -75,7 +81,15 @@ __clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, struct clk_hw *hw; int ret; - fix = kmalloc(sizeof(*fix), GFP_KERNEL); + /* You can't use devm without a dev */ + if (devm && !dev) + return ERR_PTR(-EINVAL); + + if (devm) + fix = devres_alloc(devm_clk_hw_register_fixed_factor_release, + sizeof(*fix), GFP_KERNEL); + else + fix = kmalloc(sizeof(*fix), GFP_KERNEL); if (!fix) return ERR_PTR(-ENOMEM); @@ -99,9 +113,13 @@ __clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, else ret = of_clk_hw_register(np, hw); if (ret) { - kfree(fix); + if (devm) + devres_free(fix); + else + kfree(fix); hw = ERR_PTR(ret); - } + } else if (devm) + devres_add(dev, fix); return hw; } @@ -111,7 +129,7 @@ struct clk_hw *clk_hw_register_fixed_factor(struct device *dev, unsigned int mult, unsigned int div) { return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, -1, - flags, mult, div); + flags, mult, div, false); } EXPORT_SYMBOL_GPL(clk_hw_register_fixed_factor); @@ -153,6 +171,15 @@ void clk_hw_unregister_fixed_factor(struct clk_hw *hw) } EXPORT_SYMBOL_GPL(clk_hw_unregister_fixed_factor); +struct clk_hw *devm_clk_hw_register_fixed_factor(struct device *dev, + const char *name, const char *parent_name, unsigned long flags, + unsigned int mult, unsigned int div) +{ + return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, -1, + flags, mult, div, true); +} +EXPORT_SYMBOL_GPL(devm_clk_hw_register_fixed_factor); + #ifdef CONFIG_OF static const struct of_device_id set_rate_parent_matches[] = { { .compatible = "allwinner,sun4i-a10-pll3-2x-clk" }, @@ -185,7 +212,7 @@ static struct clk_hw *_of_fixed_factor_clk_setup(struct device_node *node) flags |= CLK_SET_RATE_PARENT; hw = __clk_hw_register_fixed_factor(NULL, node, clk_name, NULL, 0, - flags, mult, div); + flags, mult, div, false); if (IS_ERR(hw)) { /* * Clear OF_POPULATED flag so that clock registration can be diff --git a/drivers/clk/clk-fixed-mmio.c b/drivers/clk/clk-fixed-mmio.c index 51f26619b6a2..5225d17d6b3f 100644 --- a/drivers/clk/clk-fixed-mmio.c +++ b/drivers/clk/clk-fixed-mmio.c @@ -55,7 +55,7 @@ static void __init of_fixed_mmio_clk_setup(struct device_node *node) } CLK_OF_DECLARE(fixed_mmio_clk, "fixed-mmio-clock", of_fixed_mmio_clk_setup); -/** +/* * This is not executed when of_fixed_mmio_clk_setup succeeded. */ static int of_fixed_mmio_clk_probe(struct platform_device *pdev) diff --git a/drivers/clk/clk-k210.c b/drivers/clk/clk-k210.c new file mode 100644 index 000000000000..6c84abf5b2e3 --- /dev/null +++ b/drivers/clk/clk-k210.c @@ -0,0 +1,1007 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (c) 2019 Western Digital Corporation or its affiliates. + */ +#define pr_fmt(fmt) "k210-clk: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +struct k210_sysclk; + +struct k210_clk { + int id; + struct k210_sysclk *ksc; + struct clk_hw hw; +}; + +struct k210_clk_cfg { + const char *name; + u8 gate_reg; + u8 gate_bit; + u8 div_reg; + u8 div_shift; + u8 div_width; + u8 div_type; + u8 mux_reg; + u8 mux_bit; +}; + +enum k210_clk_div_type { + K210_DIV_NONE, + K210_DIV_ONE_BASED, + K210_DIV_DOUBLE_ONE_BASED, + K210_DIV_POWER_OF_TWO, +}; + +#define K210_GATE(_reg, _bit) \ + .gate_reg = (_reg), \ + .gate_bit = (_bit) + +#define K210_DIV(_reg, _shift, _width, _type) \ + .div_reg = (_reg), \ + .div_shift = (_shift), \ + .div_width = (_width), \ + .div_type = (_type) + +#define K210_MUX(_reg, _bit) \ + .mux_reg = (_reg), \ + .mux_bit = (_bit) + +static struct k210_clk_cfg k210_clk_cfgs[K210_NUM_CLKS] = { + /* Gated clocks, no mux, no divider */ + [K210_CLK_CPU] = { + .name = "cpu", + K210_GATE(K210_SYSCTL_EN_CENT, 0) + }, + [K210_CLK_DMA] = { + .name = "dma", + K210_GATE(K210_SYSCTL_EN_PERI, 1) + }, + [K210_CLK_FFT] = { + .name = "fft", + K210_GATE(K210_SYSCTL_EN_PERI, 4) + }, + [K210_CLK_GPIO] = { + .name = "gpio", + K210_GATE(K210_SYSCTL_EN_PERI, 5) + }, + [K210_CLK_UART1] = { + .name = "uart1", + K210_GATE(K210_SYSCTL_EN_PERI, 16) + }, + [K210_CLK_UART2] = { + .name = "uart2", + K210_GATE(K210_SYSCTL_EN_PERI, 17) + }, + [K210_CLK_UART3] = { + .name = "uart3", + K210_GATE(K210_SYSCTL_EN_PERI, 18) + }, + [K210_CLK_FPIOA] = { + .name = "fpioa", + K210_GATE(K210_SYSCTL_EN_PERI, 20) + }, + [K210_CLK_SHA] = { + .name = "sha", + K210_GATE(K210_SYSCTL_EN_PERI, 26) + }, + [K210_CLK_AES] = { + .name = "aes", + K210_GATE(K210_SYSCTL_EN_PERI, 19) + }, + [K210_CLK_OTP] = { + .name = "otp", + K210_GATE(K210_SYSCTL_EN_PERI, 27) + }, + [K210_CLK_RTC] = { + .name = "rtc", + K210_GATE(K210_SYSCTL_EN_PERI, 29) + }, + + /* Gated divider clocks */ + [K210_CLK_SRAM0] = { + .name = "sram0", + K210_GATE(K210_SYSCTL_EN_CENT, 1), + K210_DIV(K210_SYSCTL_THR0, 0, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_SRAM1] = { + .name = "sram1", + K210_GATE(K210_SYSCTL_EN_CENT, 2), + K210_DIV(K210_SYSCTL_THR0, 4, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_ROM] = { + .name = "rom", + K210_GATE(K210_SYSCTL_EN_PERI, 0), + K210_DIV(K210_SYSCTL_THR0, 16, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_DVP] = { + .name = "dvp", + K210_GATE(K210_SYSCTL_EN_PERI, 3), + K210_DIV(K210_SYSCTL_THR0, 12, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_APB0] = { + .name = "apb0", + K210_GATE(K210_SYSCTL_EN_CENT, 3), + K210_DIV(K210_SYSCTL_SEL0, 3, 3, K210_DIV_ONE_BASED) + }, + [K210_CLK_APB1] = { + .name = "apb1", + K210_GATE(K210_SYSCTL_EN_CENT, 4), + K210_DIV(K210_SYSCTL_SEL0, 6, 3, K210_DIV_ONE_BASED) + }, + [K210_CLK_APB2] = { + .name = "apb2", + K210_GATE(K210_SYSCTL_EN_CENT, 5), + K210_DIV(K210_SYSCTL_SEL0, 9, 3, K210_DIV_ONE_BASED) + }, + [K210_CLK_AI] = { + .name = "ai", + K210_GATE(K210_SYSCTL_EN_PERI, 2), + K210_DIV(K210_SYSCTL_THR0, 8, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_SPI0] = { + .name = "spi0", + K210_GATE(K210_SYSCTL_EN_PERI, 6), + K210_DIV(K210_SYSCTL_THR1, 0, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_SPI1] = { + .name = "spi1", + K210_GATE(K210_SYSCTL_EN_PERI, 7), + K210_DIV(K210_SYSCTL_THR1, 8, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_SPI2] = { + .name = "spi2", + K210_GATE(K210_SYSCTL_EN_PERI, 8), + K210_DIV(K210_SYSCTL_THR1, 16, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2C0] = { + .name = "i2c0", + K210_GATE(K210_SYSCTL_EN_PERI, 13), + K210_DIV(K210_SYSCTL_THR5, 8, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2C1] = { + .name = "i2c1", + K210_GATE(K210_SYSCTL_EN_PERI, 14), + K210_DIV(K210_SYSCTL_THR5, 16, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2C2] = { + .name = "i2c2", + K210_GATE(K210_SYSCTL_EN_PERI, 15), + K210_DIV(K210_SYSCTL_THR5, 24, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_WDT0] = { + .name = "wdt0", + K210_GATE(K210_SYSCTL_EN_PERI, 24), + K210_DIV(K210_SYSCTL_THR6, 0, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_WDT1] = { + .name = "wdt1", + K210_GATE(K210_SYSCTL_EN_PERI, 25), + K210_DIV(K210_SYSCTL_THR6, 8, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S0] = { + .name = "i2s0", + K210_GATE(K210_SYSCTL_EN_PERI, 10), + K210_DIV(K210_SYSCTL_THR3, 0, 16, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S1] = { + .name = "i2s1", + K210_GATE(K210_SYSCTL_EN_PERI, 11), + K210_DIV(K210_SYSCTL_THR3, 16, 16, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S2] = { + .name = "i2s2", + K210_GATE(K210_SYSCTL_EN_PERI, 12), + K210_DIV(K210_SYSCTL_THR4, 0, 16, K210_DIV_DOUBLE_ONE_BASED) + }, + + /* Divider clocks, no gate, no mux */ + [K210_CLK_I2S0_M] = { + .name = "i2s0_m", + K210_DIV(K210_SYSCTL_THR4, 16, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S1_M] = { + .name = "i2s1_m", + K210_DIV(K210_SYSCTL_THR4, 24, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S2_M] = { + .name = "i2s2_m", + K210_DIV(K210_SYSCTL_THR4, 0, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + + /* Muxed gated divider clocks */ + [K210_CLK_SPI3] = { + .name = "spi3", + K210_GATE(K210_SYSCTL_EN_PERI, 9), + K210_DIV(K210_SYSCTL_THR1, 24, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 12) + }, + [K210_CLK_TIMER0] = { + .name = "timer0", + K210_GATE(K210_SYSCTL_EN_PERI, 21), + K210_DIV(K210_SYSCTL_THR2, 0, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 13) + }, + [K210_CLK_TIMER1] = { + .name = "timer1", + K210_GATE(K210_SYSCTL_EN_PERI, 22), + K210_DIV(K210_SYSCTL_THR2, 8, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 14) + }, + [K210_CLK_TIMER2] = { + .name = "timer2", + K210_GATE(K210_SYSCTL_EN_PERI, 23), + K210_DIV(K210_SYSCTL_THR2, 16, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 15) + }, +}; + +/* + * PLL control register bits. + */ +#define K210_PLL_CLKR GENMASK(3, 0) +#define K210_PLL_CLKF GENMASK(9, 4) +#define K210_PLL_CLKOD GENMASK(13, 10) +#define K210_PLL_BWADJ GENMASK(19, 14) +#define K210_PLL_RESET (1 << 20) +#define K210_PLL_PWRD (1 << 21) +#define K210_PLL_INTFB (1 << 22) +#define K210_PLL_BYPASS (1 << 23) +#define K210_PLL_TEST (1 << 24) +#define K210_PLL_EN (1 << 25) +#define K210_PLL_SEL GENMASK(27, 26) /* PLL2 only */ + +/* + * PLL lock register bits. + */ +#define K210_PLL_LOCK 0 +#define K210_PLL_CLEAR_SLIP 2 +#define K210_PLL_TEST_OUT 3 + +/* + * Clock selector register bits. + */ +#define K210_ACLK_SEL BIT(0) +#define K210_ACLK_DIV GENMASK(2, 1) + +/* + * PLLs. + */ +enum k210_pll_id { + K210_PLL0, K210_PLL1, K210_PLL2, K210_PLL_NUM +}; + +struct k210_pll { + enum k210_pll_id id; + struct k210_sysclk *ksc; + void __iomem *base; + void __iomem *reg; + void __iomem *lock; + u8 lock_shift; + u8 lock_width; + struct clk_hw hw; +}; +#define to_k210_pll(_hw) container_of(_hw, struct k210_pll, hw) + +/* + * PLLs configuration: by default PLL0 runs at 780 MHz and PLL1 at 299 MHz. + * The first 2 SRAM banks depend on ACLK/CPU clock which is by default PLL0 + * rate divided by 2. Set PLL1 to 390 MHz so that the third SRAM bank has the + * same clock as the first 2. + */ +struct k210_pll_cfg { + u32 reg; + u8 lock_shift; + u8 lock_width; + u32 r; + u32 f; + u32 od; + u32 bwadj; +}; + +static struct k210_pll_cfg k210_plls_cfg[] = { + { K210_SYSCTL_PLL0, 0, 2, 0, 59, 1, 59 }, /* 780 MHz */ + { K210_SYSCTL_PLL1, 8, 1, 0, 59, 3, 59 }, /* 390 MHz */ + { K210_SYSCTL_PLL2, 16, 1, 0, 22, 1, 22 }, /* 299 MHz */ +}; + +/** + * struct k210_sysclk - sysclk driver data + * @regs: system controller registers start address + * @clk_lock: clock setting spinlock + * @plls: SoC PLLs descriptors + * @aclk: ACLK clock + * @clks: All other clocks + */ +struct k210_sysclk { + void __iomem *regs; + spinlock_t clk_lock; + struct k210_pll plls[K210_PLL_NUM]; + struct clk_hw aclk; + struct k210_clk clks[K210_NUM_CLKS]; +}; + +#define to_k210_sysclk(_hw) container_of(_hw, struct k210_sysclk, aclk) + +/* + * Set ACLK parent selector: 0 for IN0, 1 for PLL0. + */ +static void k210_aclk_set_selector(void __iomem *regs, u8 sel) +{ + u32 reg = readl(regs + K210_SYSCTL_SEL0); + + if (sel) + reg |= K210_ACLK_SEL; + else + reg &= K210_ACLK_SEL; + writel(reg, regs + K210_SYSCTL_SEL0); +} + +static void k210_init_pll(void __iomem *regs, enum k210_pll_id pllid, + struct k210_pll *pll) +{ + pll->id = pllid; + pll->reg = regs + k210_plls_cfg[pllid].reg; + pll->lock = regs + K210_SYSCTL_PLL_LOCK; + pll->lock_shift = k210_plls_cfg[pllid].lock_shift; + pll->lock_width = k210_plls_cfg[pllid].lock_width; +} + +static void k210_pll_wait_for_lock(struct k210_pll *pll) +{ + u32 reg, mask = GENMASK(pll->lock_shift + pll->lock_width - 1, + pll->lock_shift); + + while (true) { + reg = readl(pll->lock); + if ((reg & mask) == mask) + break; + + reg |= BIT(pll->lock_shift + K210_PLL_CLEAR_SLIP); + writel(reg, pll->lock); + } +} + +static bool k210_pll_hw_is_enabled(struct k210_pll *pll) +{ + u32 reg = readl(pll->reg); + u32 mask = K210_PLL_PWRD | K210_PLL_EN; + + if (reg & K210_PLL_RESET) + return false; + + return (reg & mask) == mask; +} + +static void k210_pll_enable_hw(void __iomem *regs, struct k210_pll *pll) +{ + struct k210_pll_cfg *pll_cfg = &k210_plls_cfg[pll->id]; + u32 reg; + + if (k210_pll_hw_is_enabled(pll)) + return; + + /* + * For PLL0, we need to re-parent ACLK to IN0 to keep the CPU cores and + * SRAM running. + */ + if (pll->id == K210_PLL0) + k210_aclk_set_selector(regs, 0); + + /* Set PLL factors */ + reg = readl(pll->reg); + reg &= ~GENMASK(19, 0); + reg |= FIELD_PREP(K210_PLL_CLKR, pll_cfg->r); + reg |= FIELD_PREP(K210_PLL_CLKF, pll_cfg->f); + reg |= FIELD_PREP(K210_PLL_CLKOD, pll_cfg->od); + reg |= FIELD_PREP(K210_PLL_BWADJ, pll_cfg->bwadj); + reg |= K210_PLL_PWRD; + writel(reg, pll->reg); + + /* + * Reset the PLL: ensure reset is low before asserting it. + * The magic NOPs come from the Kendryte reference SDK. + */ + reg &= ~K210_PLL_RESET; + writel(reg, pll->reg); + reg |= K210_PLL_RESET; + writel(reg, pll->reg); + nop(); + nop(); + reg &= ~K210_PLL_RESET; + writel(reg, pll->reg); + + k210_pll_wait_for_lock(pll); + + reg &= ~K210_PLL_BYPASS; + reg |= K210_PLL_EN; + writel(reg, pll->reg); + + if (pll->id == K210_PLL0) + k210_aclk_set_selector(regs, 1); +} + +static int k210_pll_enable(struct clk_hw *hw) +{ + struct k210_pll *pll = to_k210_pll(hw); + struct k210_sysclk *ksc = pll->ksc; + unsigned long flags; + + spin_lock_irqsave(&ksc->clk_lock, flags); + + k210_pll_enable_hw(ksc->regs, pll); + + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static void k210_pll_disable(struct clk_hw *hw) +{ + struct k210_pll *pll = to_k210_pll(hw); + struct k210_sysclk *ksc = pll->ksc; + unsigned long flags; + u32 reg; + + /* + * Bypassing before powering off is important so child clocks do not + * stop working. This is especially important for pll0, the indirect + * parent of the cpu clock. + */ + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(pll->reg); + reg |= K210_PLL_BYPASS; + writel(reg, pll->reg); + + reg &= ~K210_PLL_PWRD; + reg &= ~K210_PLL_EN; + writel(reg, pll->reg); + spin_unlock_irqrestore(&ksc->clk_lock, flags); +} + +static int k210_pll_is_enabled(struct clk_hw *hw) +{ + return k210_pll_hw_is_enabled(to_k210_pll(hw)); +} + +static unsigned long k210_pll_get_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct k210_pll *pll = to_k210_pll(hw); + u32 reg = readl(pll->reg); + u32 r, f, od; + + if (reg & K210_PLL_BYPASS) + return parent_rate; + + if (!(reg & K210_PLL_PWRD)) + return 0; + + r = FIELD_GET(K210_PLL_CLKR, reg) + 1; + f = FIELD_GET(K210_PLL_CLKF, reg) + 1; + od = FIELD_GET(K210_PLL_CLKOD, reg) + 1; + + return (u64)parent_rate * f / (r * od); +} + +static const struct clk_ops k210_pll_ops = { + .enable = k210_pll_enable, + .disable = k210_pll_disable, + .is_enabled = k210_pll_is_enabled, + .recalc_rate = k210_pll_get_rate, +}; + +static int k210_pll2_set_parent(struct clk_hw *hw, u8 index) +{ + struct k210_pll *pll = to_k210_pll(hw); + struct k210_sysclk *ksc = pll->ksc; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&ksc->clk_lock, flags); + + reg = readl(pll->reg); + reg &= ~K210_PLL_SEL; + reg |= FIELD_PREP(K210_PLL_SEL, index); + writel(reg, pll->reg); + + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static u8 k210_pll2_get_parent(struct clk_hw *hw) +{ + struct k210_pll *pll = to_k210_pll(hw); + u32 reg = readl(pll->reg); + + return FIELD_GET(K210_PLL_SEL, reg); +} + +static const struct clk_ops k210_pll2_ops = { + .enable = k210_pll_enable, + .disable = k210_pll_disable, + .is_enabled = k210_pll_is_enabled, + .recalc_rate = k210_pll_get_rate, + .set_parent = k210_pll2_set_parent, + .get_parent = k210_pll2_get_parent, +}; + +static int __init k210_register_pll(struct device_node *np, + struct k210_sysclk *ksc, + enum k210_pll_id pllid, const char *name, + int num_parents, const struct clk_ops *ops) +{ + struct k210_pll *pll = &ksc->plls[pllid]; + struct clk_init_data init = {}; + const struct clk_parent_data parent_data[] = { + { /* .index = 0 for in0 */ }, + { .hw = &ksc->plls[K210_PLL0].hw }, + { .hw = &ksc->plls[K210_PLL1].hw }, + }; + + init.name = name; + init.parent_data = parent_data; + init.num_parents = num_parents; + init.ops = ops; + + pll->hw.init = &init; + pll->ksc = ksc; + + return of_clk_hw_register(np, &pll->hw); +} + +static int __init k210_register_plls(struct device_node *np, + struct k210_sysclk *ksc) +{ + int i, ret; + + for (i = 0; i < K210_PLL_NUM; i++) + k210_init_pll(ksc->regs, i, &ksc->plls[i]); + + /* PLL0 and PLL1 only have IN0 as parent */ + ret = k210_register_pll(np, ksc, K210_PLL0, "pll0", 1, &k210_pll_ops); + if (ret) { + pr_err("%pOFP: register PLL0 failed\n", np); + return ret; + } + ret = k210_register_pll(np, ksc, K210_PLL1, "pll1", 1, &k210_pll_ops); + if (ret) { + pr_err("%pOFP: register PLL1 failed\n", np); + return ret; + } + + /* PLL2 has IN0, PLL0 and PLL1 as parents */ + ret = k210_register_pll(np, ksc, K210_PLL2, "pll2", 3, &k210_pll2_ops); + if (ret) { + pr_err("%pOFP: register PLL2 failed\n", np); + return ret; + } + + return 0; +} + +static int k210_aclk_set_parent(struct clk_hw *hw, u8 index) +{ + struct k210_sysclk *ksc = to_k210_sysclk(hw); + unsigned long flags; + + spin_lock_irqsave(&ksc->clk_lock, flags); + + k210_aclk_set_selector(ksc->regs, index); + + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static u8 k210_aclk_get_parent(struct clk_hw *hw) +{ + struct k210_sysclk *ksc = to_k210_sysclk(hw); + u32 sel; + + sel = readl(ksc->regs + K210_SYSCTL_SEL0) & K210_ACLK_SEL; + + return sel ? 1 : 0; +} + +static unsigned long k210_aclk_get_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct k210_sysclk *ksc = to_k210_sysclk(hw); + u32 reg = readl(ksc->regs + K210_SYSCTL_SEL0); + unsigned int shift; + + if (!(reg & 0x1)) + return parent_rate; + + shift = FIELD_GET(K210_ACLK_DIV, reg); + + return parent_rate / (2UL << shift); +} + +static const struct clk_ops k210_aclk_ops = { + .set_parent = k210_aclk_set_parent, + .get_parent = k210_aclk_get_parent, + .recalc_rate = k210_aclk_get_rate, +}; + +/* + * ACLK has IN0 and PLL0 as parents. + */ +static int __init k210_register_aclk(struct device_node *np, + struct k210_sysclk *ksc) +{ + struct clk_init_data init = {}; + const struct clk_parent_data parent_data[] = { + { /* .index = 0 for in0 */ }, + { .hw = &ksc->plls[K210_PLL0].hw }, + }; + int ret; + + init.name = "aclk"; + init.parent_data = parent_data; + init.num_parents = 2; + init.ops = &k210_aclk_ops; + ksc->aclk.init = &init; + + ret = of_clk_hw_register(np, &ksc->aclk); + if (ret) { + pr_err("%pOFP: register aclk failed\n", np); + return ret; + } + + return 0; +} + +#define to_k210_clk(_hw) container_of(_hw, struct k210_clk, hw) + +static int k210_clk_enable(struct clk_hw *hw) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg; + + if (!cfg->gate_reg) + return 0; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->gate_reg); + reg |= BIT(cfg->gate_bit); + writel(reg, ksc->regs + cfg->gate_reg); + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static void k210_clk_disable(struct clk_hw *hw) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg; + + if (!cfg->gate_reg) + return; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->gate_reg); + reg &= ~BIT(cfg->gate_bit); + writel(reg, ksc->regs + cfg->gate_reg); + spin_unlock_irqrestore(&ksc->clk_lock, flags); +} + +static int k210_clk_set_parent(struct clk_hw *hw, u8 index) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->mux_reg); + if (index) + reg |= BIT(cfg->mux_bit); + else + reg &= ~BIT(cfg->mux_bit); + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static u8 k210_clk_get_parent(struct clk_hw *hw) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg, idx; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->mux_reg); + idx = (reg & BIT(cfg->mux_bit)) ? 1 : 0; + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return idx; +} + +static unsigned long k210_clk_get_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + u32 reg, div_val; + + if (!cfg->div_reg) + return parent_rate; + + reg = readl(ksc->regs + cfg->div_reg); + div_val = (reg >> cfg->div_shift) & GENMASK(cfg->div_width - 1, 0); + + switch (cfg->div_type) { + case K210_DIV_ONE_BASED: + return parent_rate / (div_val + 1); + case K210_DIV_DOUBLE_ONE_BASED: + return parent_rate / ((div_val + 1) * 2); + case K210_DIV_POWER_OF_TWO: + return parent_rate / (2UL << div_val); + case K210_DIV_NONE: + default: + return 0; + } +} + +static const struct clk_ops k210_clk_mux_ops = { + .enable = k210_clk_enable, + .disable = k210_clk_disable, + .set_parent = k210_clk_set_parent, + .get_parent = k210_clk_get_parent, + .recalc_rate = k210_clk_get_rate, +}; + +static const struct clk_ops k210_clk_ops = { + .enable = k210_clk_enable, + .disable = k210_clk_disable, + .recalc_rate = k210_clk_get_rate, +}; + +static void __init k210_register_clk(struct device_node *np, + struct k210_sysclk *ksc, int id, + const struct clk_parent_data *parent_data, + int num_parents, unsigned long flags) +{ + struct k210_clk *kclk = &ksc->clks[id]; + struct clk_init_data init = {}; + int ret; + + init.name = k210_clk_cfgs[id].name; + init.flags = flags; + init.parent_data = parent_data; + init.num_parents = num_parents; + if (num_parents > 1) + init.ops = &k210_clk_mux_ops; + else + init.ops = &k210_clk_ops; + + kclk->id = id; + kclk->ksc = ksc; + kclk->hw.init = &init; + + ret = of_clk_hw_register(np, &kclk->hw); + if (ret) { + pr_err("%pOFP: register clock %s failed\n", + np, k210_clk_cfgs[id].name); + kclk->id = -1; + } +} + +/* + * All muxed clocks have IN0 and PLL0 as parents. + */ +static inline void __init k210_register_mux_clk(struct device_node *np, + struct k210_sysclk *ksc, int id) +{ + const struct clk_parent_data parent_data[2] = { + { /* .index = 0 for in0 */ }, + { .hw = &ksc->plls[K210_PLL0].hw } + }; + + k210_register_clk(np, ksc, id, parent_data, 2, 0); +} + +static inline void __init k210_register_in0_child(struct device_node *np, + struct k210_sysclk *ksc, int id) +{ + const struct clk_parent_data parent_data = { + /* .index = 0 for in0 */ + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, 0); +} + +static inline void __init k210_register_pll_child(struct device_node *np, + struct k210_sysclk *ksc, int id, + enum k210_pll_id pllid, + unsigned long flags) +{ + const struct clk_parent_data parent_data = { + .hw = &ksc->plls[pllid].hw, + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, flags); +} + +static inline void __init k210_register_aclk_child(struct device_node *np, + struct k210_sysclk *ksc, int id, + unsigned long flags) +{ + const struct clk_parent_data parent_data = { + .hw = &ksc->aclk, + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, flags); +} + +static inline void __init k210_register_clk_child(struct device_node *np, + struct k210_sysclk *ksc, int id, + int parent_id) +{ + const struct clk_parent_data parent_data = { + .hw = &ksc->clks[parent_id].hw, + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, 0); +} + +static struct clk_hw *k210_clk_hw_onecell_get(struct of_phandle_args *clkspec, + void *data) +{ + struct k210_sysclk *ksc = data; + unsigned int idx = clkspec->args[0]; + + if (idx >= K210_NUM_CLKS) + return ERR_PTR(-EINVAL); + + return &ksc->clks[idx].hw; +} + +static void __init k210_clk_init(struct device_node *np) +{ + struct device_node *sysctl_np; + struct k210_sysclk *ksc; + int i, ret; + + ksc = kzalloc(sizeof(*ksc), GFP_KERNEL); + if (!ksc) + return; + + spin_lock_init(&ksc->clk_lock); + sysctl_np = of_get_parent(np); + ksc->regs = of_iomap(sysctl_np, 0); + of_node_put(sysctl_np); + if (!ksc->regs) { + pr_err("%pOFP: failed to map registers\n", np); + return; + } + + ret = k210_register_plls(np, ksc); + if (ret) + return; + + ret = k210_register_aclk(np, ksc); + if (ret) + return; + + /* + * Critical clocks: there are no consumers of the SRAM clocks, + * including the AI clock for the third SRAM bank. The CPU clock + * is only referenced by the uarths serial device and so would be + * disabled if the serial console is disabled to switch to another + * console. Mark all these clocks as critical so that they are never + * disabled by the core clock management. + */ + k210_register_aclk_child(np, ksc, K210_CLK_CPU, CLK_IS_CRITICAL); + k210_register_aclk_child(np, ksc, K210_CLK_SRAM0, CLK_IS_CRITICAL); + k210_register_aclk_child(np, ksc, K210_CLK_SRAM1, CLK_IS_CRITICAL); + k210_register_pll_child(np, ksc, K210_CLK_AI, K210_PLL1, + CLK_IS_CRITICAL); + + /* Clocks with aclk as source */ + k210_register_aclk_child(np, ksc, K210_CLK_DMA, 0); + k210_register_aclk_child(np, ksc, K210_CLK_FFT, 0); + k210_register_aclk_child(np, ksc, K210_CLK_ROM, 0); + k210_register_aclk_child(np, ksc, K210_CLK_DVP, 0); + k210_register_aclk_child(np, ksc, K210_CLK_APB0, 0); + k210_register_aclk_child(np, ksc, K210_CLK_APB1, 0); + k210_register_aclk_child(np, ksc, K210_CLK_APB2, 0); + + /* Clocks with PLL0 as source */ + k210_register_pll_child(np, ksc, K210_CLK_SPI0, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_SPI1, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_SPI2, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2C0, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2C1, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2C2, K210_PLL0, 0); + + /* Clocks with PLL2 as source */ + k210_register_pll_child(np, ksc, K210_CLK_I2S0, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S1, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S2, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S0_M, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S1_M, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S2_M, K210_PLL2, 0); + + /* Clocks with IN0 as source */ + k210_register_in0_child(np, ksc, K210_CLK_WDT0); + k210_register_in0_child(np, ksc, K210_CLK_WDT1); + k210_register_in0_child(np, ksc, K210_CLK_RTC); + + /* Clocks with APB0 as source */ + k210_register_clk_child(np, ksc, K210_CLK_GPIO, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_UART1, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_UART2, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_UART3, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_FPIOA, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_SHA, K210_CLK_APB0); + + /* Clocks with APB1 as source */ + k210_register_clk_child(np, ksc, K210_CLK_AES, K210_CLK_APB1); + k210_register_clk_child(np, ksc, K210_CLK_OTP, K210_CLK_APB1); + + /* Mux clocks with in0 or pll0 as source */ + k210_register_mux_clk(np, ksc, K210_CLK_SPI3); + k210_register_mux_clk(np, ksc, K210_CLK_TIMER0); + k210_register_mux_clk(np, ksc, K210_CLK_TIMER1); + k210_register_mux_clk(np, ksc, K210_CLK_TIMER2); + + /* Check for registration errors */ + for (i = 0; i < K210_NUM_CLKS; i++) { + if (ksc->clks[i].id != i) + return; + } + + ret = of_clk_add_hw_provider(np, k210_clk_hw_onecell_get, ksc); + if (ret) { + pr_err("%pOFP: add clock provider failed %d\n", np, ret); + return; + } + + pr_info("%pOFP: CPU running at %lu MHz\n", + np, clk_hw_get_rate(&ksc->clks[K210_CLK_CPU].hw) / 1000000); +} + +CLK_OF_DECLARE(k210_clk, "canaan,k210-clk", k210_clk_init); + +/* + * Enable PLL1 to be able to use the AI SRAM. + */ +void __init k210_clk_early_init(void __iomem *regs) +{ + struct k210_pll pll1; + + /* Make sure ACLK selector is set to PLL0 */ + k210_aclk_set_selector(regs, 1); + + /* Startup PLL1 to enable the aisram bank for general memory use */ + k210_init_pll(regs, K210_PLL1, &pll1); + k210_pll_enable_hw(regs, &pll1); +} diff --git a/drivers/clk/clk-npcm7xx.c b/drivers/clk/clk-npcm7xx.c index 27a86b7a34db..e677bb5a784b 100644 --- a/drivers/clk/clk-npcm7xx.c +++ b/drivers/clk/clk-npcm7xx.c @@ -361,13 +361,6 @@ static const struct npcm7xx_clk_mux_data npcm7xx_muxes[] __initconst = { dvcssel_mux_parents, ARRAY_SIZE(dvcssel_mux_parents), 0, -1}, }; -/* fixed ratio dividers (no register): */ -static const struct npcm7xx_clk_div_fixed_data npcm7xx_divs_fx[] __initconst = { - { 1, 2, NPCM7XX_CLK_S_MC, NPCM7XX_CLK_S_MC_MUX, 0, NPCM7XX_CLK_MC}, - { 1, 2, NPCM7XX_CLK_S_PLL1_DIV2, NPCM7XX_CLK_S_PLL1, 0, -1}, - { 1, 2, NPCM7XX_CLK_S_PLL2_DIV2, NPCM7XX_CLK_S_PLL2, 0, -1}, -}; - /* configurable dividers: */ static const struct npcm7xx_clk_div_data npcm7xx_divs[] __initconst = { {NPCM7XX_CLKDIV1, 28, 3, NPCM7XX_CLK_S_ADC, @@ -435,107 +428,6 @@ static const struct npcm7xx_clk_div_data npcm7xx_divs[] __initconst = { }; -static const struct npcm7xx_clk_gate_data npcm7xx_gates[] __initconst = { - {NPCM7XX_CLKEN1, 31, "smb1-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 30, "smb0-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 29, "smb7-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 28, "smb6-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 27, "adc-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 26, "wdt-gate", NPCM7XX_CLK_S_TIMER, 0}, - {NPCM7XX_CLKEN1, 25, "usbdev3-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 24, "usbdev6-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 23, "usbdev5-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 22, "usbdev4-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 21, "emc2-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 20, "timer5_9-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 19, "timer0_4-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 18, "pwmm0-gate", NPCM7XX_CLK_S_APB3, 0}, - {NPCM7XX_CLKEN1, 17, "huart-gate", NPCM7XX_CLK_S_UART, 0}, - {NPCM7XX_CLKEN1, 16, "smb5-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 15, "smb4-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 14, "smb3-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 13, "smb2-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 12, "mc-gate", NPCM7XX_CLK_S_MC, 0}, - {NPCM7XX_CLKEN1, 11, "uart01-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 10, "aes-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 9, "peci-gate", NPCM7XX_CLK_S_APB3, 0}, - {NPCM7XX_CLKEN1, 8, "usbdev2-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 7, "uart23-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 6, "emc1-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 5, "usbdev1-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 4, "shm-gate", NPCM7XX_CLK_S_AHB, 0}, - /* bit 3 is reserved */ - {NPCM7XX_CLKEN1, 2, "kcs-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 1, "spi3-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 0, "spi0-gate", NPCM7XX_CLK_S_AHB, 0}, - - {NPCM7XX_CLKEN2, 31, "cp-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 30, "tock-gate", NPCM7XX_CLK_S_TOCK, 0}, - /* bit 29 is reserved */ - {NPCM7XX_CLKEN2, 28, "gmac1-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 27, "usbif-gate", NPCM7XX_CLK_S_USBIF, 0}, - {NPCM7XX_CLKEN2, 26, "usbhost-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 25, "gmac2-gate", NPCM7XX_CLK_S_AHB, 0}, - /* bit 24 is reserved */ - {NPCM7XX_CLKEN2, 23, "pspi2-gate", NPCM7XX_CLK_S_APB5, 0}, - {NPCM7XX_CLKEN2, 22, "pspi1-gate", NPCM7XX_CLK_S_APB5, 0}, - {NPCM7XX_CLKEN2, 21, "3des-gate", NPCM7XX_CLK_S_AHB, 0}, - /* bit 20 is reserved */ - {NPCM7XX_CLKEN2, 19, "siox2-gate", NPCM7XX_CLK_S_APB3, 0}, - {NPCM7XX_CLKEN2, 18, "siox1-gate", NPCM7XX_CLK_S_APB3, 0}, - /* bit 17 is reserved */ - {NPCM7XX_CLKEN2, 16, "fuse-gate", NPCM7XX_CLK_S_APB4, 0}, - /* bit 15 is reserved */ - {NPCM7XX_CLKEN2, 14, "vcd-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 13, "ece-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 12, "vdma-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 11, "ahbpcibrg-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 10, "gfxsys-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN2, 9, "sdhc-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 8, "mmc-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 7, "mft7-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 6, "mft6-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 5, "mft5-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 4, "mft4-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 3, "mft3-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 2, "mft2-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 1, "mft1-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 0, "mft0-gate", NPCM7XX_CLK_S_APB4, 0}, - - {NPCM7XX_CLKEN3, 31, "gpiom7-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 30, "gpiom6-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 29, "gpiom5-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 28, "gpiom4-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 27, "gpiom3-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 26, "gpiom2-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 25, "gpiom1-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 24, "gpiom0-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 23, "espi-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 22, "smb11-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 21, "smb10-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 20, "smb9-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 19, "smb8-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 18, "smb15-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 17, "rng-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 16, "timer10_14-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 15, "pcirc-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 14, "sececc-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 13, "sha-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 12, "smb14-gate", NPCM7XX_CLK_S_APB2, 0}, - /* bit 11 is reserved */ - /* bit 10 is reserved */ - {NPCM7XX_CLKEN3, 9, "pcimbx-gate", NPCM7XX_CLK_S_AHB, 0}, - /* bit 8 is reserved */ - {NPCM7XX_CLKEN3, 7, "usbdev9-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 6, "usbdev8-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 5, "usbdev7-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 4, "usbdev0-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 3, "smb13-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 2, "spix-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 1, "smb12-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 0, "pwmm1-gate", NPCM7XX_CLK_S_APB3, 0}, -}; - static DEFINE_SPINLOCK(npcm7xx_clk_lock); static void __init npcm7xx_clk_init(struct device_node *clk_np) diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c index 70aa521e7e7f..88898b97a443 100644 --- a/drivers/clk/clk-qoriq.c +++ b/drivers/clk/clk-qoriq.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2013 Freescale Semiconductor, Inc. + * Copyright 2021 NXP * * clock driver for Freescale QorIQ SoCs. */ @@ -564,7 +565,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 1, 1, 1, -1 }, - .pll_mask = 0x3f, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | BIT(CGA_PLL3) | + BIT(CGB_PLL1) | BIT(CGB_PLL2), .flags = CG_PLL_8BIT, }, { @@ -580,7 +583,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 1, 1, 1, -1 }, - .pll_mask = 0x3f, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | BIT(CGA_PLL3) | + BIT(CGB_PLL1) | BIT(CGB_PLL2), .flags = CG_PLL_8BIT, }, { @@ -591,7 +596,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, -1 }, - .pll_mask = 0x03, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), }, { .compat = "fsl,ls1028a-clockgen", @@ -605,7 +611,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 0, 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_VER3 | CG_LITTLE_ENDIAN, }, { @@ -620,7 +627,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_PLL_8BIT, }, { @@ -635,7 +643,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_PLL_8BIT, }, { @@ -649,7 +658,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_VER3 | CG_LITTLE_ENDIAN, }, { @@ -660,7 +670,7 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, -1 }, - .pll_mask = 0x03, + .pll_mask = BIT(PLATFORM_PLL) | BIT(CGA_PLL1), }, { .compat = "fsl,ls2080a-clockgen", @@ -670,7 +680,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 1, 1, -1 }, - .pll_mask = 0x37, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | + BIT(CGB_PLL1) | BIT(CGB_PLL2), .flags = CG_VER3 | CG_LITTLE_ENDIAN, }, { @@ -681,7 +693,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 0, 0, 1, 1, 1, 1, -1 }, - .pll_mask = 0x37, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | + BIT(CGB_PLL1) | BIT(CGB_PLL2), .flags = CG_VER3 | CG_LITTLE_ENDIAN, }, { @@ -694,7 +708,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 1, 1, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), }, { .compat = "fsl,p3041-clockgen", @@ -706,7 +721,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 1, 1, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), }, { .compat = "fsl,p4080-clockgen", @@ -718,7 +734,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 0, 0, 1, 1, 1, 1, -1 }, - .pll_mask = 0x1f, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | + BIT(CGA_PLL3) | BIT(CGA_PLL4), }, { .compat = "fsl,p5020-clockgen", @@ -730,7 +748,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 1, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), }, { .compat = "fsl,p5040-clockgen", @@ -742,7 +761,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 1, 1, -1 }, - .pll_mask = 0x0f, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | BIT(CGA_PLL3), }, { .compat = "fsl,t1023-clockgen", @@ -757,7 +777,7 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, -1 }, - .pll_mask = 0x03, + .pll_mask = BIT(PLATFORM_PLL) | BIT(CGA_PLL1), .flags = CG_PLL_8BIT, }, { @@ -770,7 +790,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 0, 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_PLL_8BIT, }, { @@ -786,7 +807,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_PLL_8BIT, }, { @@ -802,7 +824,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 1, -1 }, - .pll_mask = 0x3f, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | BIT(CGA_PLL3) | + BIT(CGB_PLL1) | BIT(CGB_PLL2), .flags = CG_PLL_8BIT, }, {}, diff --git a/drivers/clk/clk-si570.c b/drivers/clk/clk-si570.c index 34b25609f55f..eea50121718a 100644 --- a/drivers/clk/clk-si570.c +++ b/drivers/clk/clk-si570.c @@ -4,7 +4,7 @@ * * Copyright (C) 2010, 2011 Ericsson AB. * Copyright (C) 2011 Guenter Roeck. - * Copyright (C) 2011 - 2013 Xilinx Inc. + * Copyright (C) 2011 - 2021 Xilinx Inc. * * Author: Guenter Roeck * Sören Brinkmann @@ -123,14 +123,18 @@ static int si570_get_divs(struct clk_si570 *data, u64 *rfreq, * si570_get_defaults() - Get default values * @data: Driver data structure * @fout: Factory frequency output + * @skip_recall: If true, don't recall NVM into RAM * Returns 0 on success, negative errno otherwise. */ -static int si570_get_defaults(struct clk_si570 *data, u64 fout) +static int si570_get_defaults(struct clk_si570 *data, u64 fout, + bool skip_recall) { int err; u64 fdco; - regmap_write(data->regmap, SI570_REG_CONTROL, SI570_CNTRL_RECALL); + if (!skip_recall) + regmap_write(data->regmap, SI570_REG_CONTROL, + SI570_CNTRL_RECALL); err = si570_get_divs(data, &data->rfreq, &data->n1, &data->hs_div); if (err) @@ -400,6 +404,7 @@ static int si570_probe(struct i2c_client *client, struct clk_si570 *data; struct clk_init_data init; u32 initial_fout, factory_fout, stability; + bool skip_recall; int err; enum clk_si570_variant variant = id->driver_data; @@ -441,6 +446,9 @@ static int si570_probe(struct i2c_client *client, return err; } + skip_recall = of_property_read_bool(client->dev.of_node, + "silabs,skip-recall"); + data->regmap = devm_regmap_init_i2c(client, &si570_regmap_config); if (IS_ERR(data->regmap)) { dev_err(&client->dev, "failed to allocate register map\n"); @@ -448,7 +456,7 @@ static int si570_probe(struct i2c_client *client, } i2c_set_clientdata(client, data); - err = si570_get_defaults(data, factory_fout); + err = si570_get_defaults(data, factory_fout, skip_recall); if (err) return err; diff --git a/drivers/clk/clk-tango4.c b/drivers/clk/clk-tango4.c deleted file mode 100644 index fe12a43f7a40..000000000000 --- a/drivers/clk/clk-tango4.c +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include - -#define CLK_COUNT 4 /* cpu_clk, sys_clk, usb_clk, sdio_clk */ -static struct clk *clks[CLK_COUNT]; -static struct clk_onecell_data clk_data = { clks, CLK_COUNT }; - -#define SYSCLK_DIV 0x20 -#define CPUCLK_DIV 0x24 -#define DIV_BYPASS BIT(23) - -/*** CLKGEN_PLL ***/ -#define extract_pll_n(val) ((val >> 0) & ((1u << 7) - 1)) -#define extract_pll_k(val) ((val >> 13) & ((1u << 3) - 1)) -#define extract_pll_m(val) ((val >> 16) & ((1u << 3) - 1)) -#define extract_pll_isel(val) ((val >> 24) & ((1u << 3) - 1)) - -static void __init make_pll(int idx, const char *parent, void __iomem *base) -{ - char name[8]; - u32 val, mul, div; - - sprintf(name, "pll%d", idx); - val = readl(base + idx * 8); - mul = extract_pll_n(val) + 1; - div = (extract_pll_m(val) + 1) << extract_pll_k(val); - clk_register_fixed_factor(NULL, name, parent, 0, mul, div); - if (extract_pll_isel(val) != 1) - panic("%s: input not set to XTAL_IN\n", name); -} - -static void __init make_cd(int idx, void __iomem *base) -{ - char name[8]; - u32 val, mul, div; - - sprintf(name, "cd%d", idx); - val = readl(base + idx * 8); - mul = 1 << 27; - div = (2 << 27) + val; - clk_register_fixed_factor(NULL, name, "pll2", 0, mul, div); - if (val > 0xf0000000) - panic("%s: unsupported divider %x\n", name, val); -} - -static void __init tango4_clkgen_setup(struct device_node *np) -{ - struct clk **pp = clk_data.clks; - void __iomem *base = of_iomap(np, 0); - const char *parent = of_clk_get_parent_name(np, 0); - - if (!base) - panic("%pOFn: invalid address\n", np); - - if (readl(base + CPUCLK_DIV) & DIV_BYPASS) - panic("%pOFn: unsupported cpuclk setup\n", np); - - if (readl(base + SYSCLK_DIV) & DIV_BYPASS) - panic("%pOFn: unsupported sysclk setup\n", np); - - writel(0x100, base + CPUCLK_DIV); /* disable frequency ramping */ - - make_pll(0, parent, base); - make_pll(1, parent, base); - make_pll(2, parent, base); - make_cd(2, base + 0x80); - make_cd(6, base + 0x80); - - pp[0] = clk_register_divider(NULL, "cpu_clk", "pll0", 0, - base + CPUCLK_DIV, 8, 8, CLK_DIVIDER_ONE_BASED, NULL); - pp[1] = clk_register_fixed_factor(NULL, "sys_clk", "pll1", 0, 1, 4); - pp[2] = clk_register_fixed_factor(NULL, "usb_clk", "cd2", 0, 1, 2); - pp[3] = clk_register_fixed_factor(NULL, "sdio_clk", "cd6", 0, 1, 2); - - if (IS_ERR(pp[0]) || IS_ERR(pp[1]) || IS_ERR(pp[2]) || IS_ERR(pp[3])) - panic("%pOFn: clk registration failed\n", np); - - if (of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data)) - panic("%pOFn: clk provider registration failed\n", np); -} -CLK_OF_DECLARE(tango4_clkgen, "sigma,tango4-clkgen", tango4_clkgen_setup); diff --git a/drivers/clk/clk-u300.c b/drivers/clk/clk-u300.c deleted file mode 100644 index e228c07c4c6e..000000000000 --- a/drivers/clk/clk-u300.c +++ /dev/null @@ -1,1199 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * U300 clock implementation - * Copyright (C) 2007-2012 ST-Ericsson AB - * Author: Linus Walleij - * Author: Jonas Aaberg - */ -#include -#include -#include -#include -#include -#include -#include -#include - -/* APP side SYSCON registers */ -/* CLK Control Register 16bit (R/W) */ -#define U300_SYSCON_CCR (0x0000) -#define U300_SYSCON_CCR_I2S1_USE_VCXO (0x0040) -#define U300_SYSCON_CCR_I2S0_USE_VCXO (0x0020) -#define U300_SYSCON_CCR_TURN_VCXO_ON (0x0008) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_MASK (0x0007) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER (0x04) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW (0x03) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE (0x02) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_HIGH (0x01) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_BEST (0x00) -/* CLK Status Register 16bit (R/W) */ -#define U300_SYSCON_CSR (0x0004) -#define U300_SYSCON_CSR_PLL208_LOCK_IND (0x0002) -#define U300_SYSCON_CSR_PLL13_LOCK_IND (0x0001) -/* Reset lines for SLOW devices 16bit (R/W) */ -#define U300_SYSCON_RSR (0x0014) -#define U300_SYSCON_RSR_PPM_RESET_EN (0x0200) -#define U300_SYSCON_RSR_ACC_TMR_RESET_EN (0x0100) -#define U300_SYSCON_RSR_APP_TMR_RESET_EN (0x0080) -#define U300_SYSCON_RSR_RTC_RESET_EN (0x0040) -#define U300_SYSCON_RSR_KEYPAD_RESET_EN (0x0020) -#define U300_SYSCON_RSR_GPIO_RESET_EN (0x0010) -#define U300_SYSCON_RSR_EH_RESET_EN (0x0008) -#define U300_SYSCON_RSR_BTR_RESET_EN (0x0004) -#define U300_SYSCON_RSR_UART_RESET_EN (0x0002) -#define U300_SYSCON_RSR_SLOW_BRIDGE_RESET_EN (0x0001) -/* Reset lines for FAST devices 16bit (R/W) */ -#define U300_SYSCON_RFR (0x0018) -#define U300_SYSCON_RFR_UART1_RESET_ENABLE (0x0080) -#define U300_SYSCON_RFR_SPI_RESET_ENABLE (0x0040) -#define U300_SYSCON_RFR_MMC_RESET_ENABLE (0x0020) -#define U300_SYSCON_RFR_PCM_I2S1_RESET_ENABLE (0x0010) -#define U300_SYSCON_RFR_PCM_I2S0_RESET_ENABLE (0x0008) -#define U300_SYSCON_RFR_I2C1_RESET_ENABLE (0x0004) -#define U300_SYSCON_RFR_I2C0_RESET_ENABLE (0x0002) -#define U300_SYSCON_RFR_FAST_BRIDGE_RESET_ENABLE (0x0001) -/* Reset lines for the rest of the peripherals 16bit (R/W) */ -#define U300_SYSCON_RRR (0x001c) -#define U300_SYSCON_RRR_CDS_RESET_EN (0x4000) -#define U300_SYSCON_RRR_ISP_RESET_EN (0x2000) -#define U300_SYSCON_RRR_INTCON_RESET_EN (0x1000) -#define U300_SYSCON_RRR_MSPRO_RESET_EN (0x0800) -#define U300_SYSCON_RRR_XGAM_RESET_EN (0x0100) -#define U300_SYSCON_RRR_XGAM_VC_SYNC_RESET_EN (0x0080) -#define U300_SYSCON_RRR_NANDIF_RESET_EN (0x0040) -#define U300_SYSCON_RRR_EMIF_RESET_EN (0x0020) -#define U300_SYSCON_RRR_DMAC_RESET_EN (0x0010) -#define U300_SYSCON_RRR_CPU_RESET_EN (0x0008) -#define U300_SYSCON_RRR_APEX_RESET_EN (0x0004) -#define U300_SYSCON_RRR_AHB_RESET_EN (0x0002) -#define U300_SYSCON_RRR_AAIF_RESET_EN (0x0001) -/* Clock enable for SLOW peripherals 16bit (R/W) */ -#define U300_SYSCON_CESR (0x0020) -#define U300_SYSCON_CESR_PPM_CLK_EN (0x0200) -#define U300_SYSCON_CESR_ACC_TMR_CLK_EN (0x0100) -#define U300_SYSCON_CESR_APP_TMR_CLK_EN (0x0080) -#define U300_SYSCON_CESR_KEYPAD_CLK_EN (0x0040) -#define U300_SYSCON_CESR_GPIO_CLK_EN (0x0010) -#define U300_SYSCON_CESR_EH_CLK_EN (0x0008) -#define U300_SYSCON_CESR_BTR_CLK_EN (0x0004) -#define U300_SYSCON_CESR_UART_CLK_EN (0x0002) -#define U300_SYSCON_CESR_SLOW_BRIDGE_CLK_EN (0x0001) -/* Clock enable for FAST peripherals 16bit (R/W) */ -#define U300_SYSCON_CEFR (0x0024) -#define U300_SYSCON_CEFR_UART1_CLK_EN (0x0200) -#define U300_SYSCON_CEFR_I2S1_CORE_CLK_EN (0x0100) -#define U300_SYSCON_CEFR_I2S0_CORE_CLK_EN (0x0080) -#define U300_SYSCON_CEFR_SPI_CLK_EN (0x0040) -#define U300_SYSCON_CEFR_MMC_CLK_EN (0x0020) -#define U300_SYSCON_CEFR_I2S1_CLK_EN (0x0010) -#define U300_SYSCON_CEFR_I2S0_CLK_EN (0x0008) -#define U300_SYSCON_CEFR_I2C1_CLK_EN (0x0004) -#define U300_SYSCON_CEFR_I2C0_CLK_EN (0x0002) -#define U300_SYSCON_CEFR_FAST_BRIDGE_CLK_EN (0x0001) -/* Clock enable for the rest of the peripherals 16bit (R/W) */ -#define U300_SYSCON_CERR (0x0028) -#define U300_SYSCON_CERR_CDS_CLK_EN (0x2000) -#define U300_SYSCON_CERR_ISP_CLK_EN (0x1000) -#define U300_SYSCON_CERR_MSPRO_CLK_EN (0x0800) -#define U300_SYSCON_CERR_AHB_SUBSYS_BRIDGE_CLK_EN (0x0400) -#define U300_SYSCON_CERR_SEMI_CLK_EN (0x0200) -#define U300_SYSCON_CERR_XGAM_CLK_EN (0x0100) -#define U300_SYSCON_CERR_VIDEO_ENC_CLK_EN (0x0080) -#define U300_SYSCON_CERR_NANDIF_CLK_EN (0x0040) -#define U300_SYSCON_CERR_EMIF_CLK_EN (0x0020) -#define U300_SYSCON_CERR_DMAC_CLK_EN (0x0010) -#define U300_SYSCON_CERR_CPU_CLK_EN (0x0008) -#define U300_SYSCON_CERR_APEX_CLK_EN (0x0004) -#define U300_SYSCON_CERR_AHB_CLK_EN (0x0002) -#define U300_SYSCON_CERR_AAIF_CLK_EN (0x0001) -/* Single block clock enable 16bit (-/W) */ -#define U300_SYSCON_SBCER (0x002c) -#define U300_SYSCON_SBCER_PPM_CLK_EN (0x0009) -#define U300_SYSCON_SBCER_ACC_TMR_CLK_EN (0x0008) -#define U300_SYSCON_SBCER_APP_TMR_CLK_EN (0x0007) -#define U300_SYSCON_SBCER_KEYPAD_CLK_EN (0x0006) -#define U300_SYSCON_SBCER_GPIO_CLK_EN (0x0004) -#define U300_SYSCON_SBCER_EH_CLK_EN (0x0003) -#define U300_SYSCON_SBCER_BTR_CLK_EN (0x0002) -#define U300_SYSCON_SBCER_UART_CLK_EN (0x0001) -#define U300_SYSCON_SBCER_SLOW_BRIDGE_CLK_EN (0x0000) -#define U300_SYSCON_SBCER_UART1_CLK_EN (0x0019) -#define U300_SYSCON_SBCER_I2S1_CORE_CLK_EN (0x0018) -#define U300_SYSCON_SBCER_I2S0_CORE_CLK_EN (0x0017) -#define U300_SYSCON_SBCER_SPI_CLK_EN (0x0016) -#define U300_SYSCON_SBCER_MMC_CLK_EN (0x0015) -#define U300_SYSCON_SBCER_I2S1_CLK_EN (0x0014) -#define U300_SYSCON_SBCER_I2S0_CLK_EN (0x0013) -#define U300_SYSCON_SBCER_I2C1_CLK_EN (0x0012) -#define U300_SYSCON_SBCER_I2C0_CLK_EN (0x0011) -#define U300_SYSCON_SBCER_FAST_BRIDGE_CLK_EN (0x0010) -#define U300_SYSCON_SBCER_CDS_CLK_EN (0x002D) -#define U300_SYSCON_SBCER_ISP_CLK_EN (0x002C) -#define U300_SYSCON_SBCER_MSPRO_CLK_EN (0x002B) -#define U300_SYSCON_SBCER_AHB_SUBSYS_BRIDGE_CLK_EN (0x002A) -#define U300_SYSCON_SBCER_SEMI_CLK_EN (0x0029) -#define U300_SYSCON_SBCER_XGAM_CLK_EN (0x0028) -#define U300_SYSCON_SBCER_VIDEO_ENC_CLK_EN (0x0027) -#define U300_SYSCON_SBCER_NANDIF_CLK_EN (0x0026) -#define U300_SYSCON_SBCER_EMIF_CLK_EN (0x0025) -#define U300_SYSCON_SBCER_DMAC_CLK_EN (0x0024) -#define U300_SYSCON_SBCER_CPU_CLK_EN (0x0023) -#define U300_SYSCON_SBCER_APEX_CLK_EN (0x0022) -#define U300_SYSCON_SBCER_AHB_CLK_EN (0x0021) -#define U300_SYSCON_SBCER_AAIF_CLK_EN (0x0020) -/* Single block clock disable 16bit (-/W) */ -#define U300_SYSCON_SBCDR (0x0030) -/* Same values as above for SBCER */ -/* Clock force SLOW peripherals 16bit (R/W) */ -#define U300_SYSCON_CFSR (0x003c) -#define U300_SYSCON_CFSR_PPM_CLK_FORCE_EN (0x0200) -#define U300_SYSCON_CFSR_ACC_TMR_CLK_FORCE_EN (0x0100) -#define U300_SYSCON_CFSR_APP_TMR_CLK_FORCE_EN (0x0080) -#define U300_SYSCON_CFSR_KEYPAD_CLK_FORCE_EN (0x0020) -#define U300_SYSCON_CFSR_GPIO_CLK_FORCE_EN (0x0010) -#define U300_SYSCON_CFSR_EH_CLK_FORCE_EN (0x0008) -#define U300_SYSCON_CFSR_BTR_CLK_FORCE_EN (0x0004) -#define U300_SYSCON_CFSR_UART_CLK_FORCE_EN (0x0002) -#define U300_SYSCON_CFSR_SLOW_BRIDGE_CLK_FORCE_EN (0x0001) -/* Clock force FAST peripherals 16bit (R/W) */ -#define U300_SYSCON_CFFR (0x40) -/* Values not defined. Define if you want to use them. */ -/* Clock force the rest of the peripherals 16bit (R/W) */ -#define U300_SYSCON_CFRR (0x44) -#define U300_SYSCON_CFRR_CDS_CLK_FORCE_EN (0x2000) -#define U300_SYSCON_CFRR_ISP_CLK_FORCE_EN (0x1000) -#define U300_SYSCON_CFRR_MSPRO_CLK_FORCE_EN (0x0800) -#define U300_SYSCON_CFRR_AHB_SUBSYS_BRIDGE_CLK_FORCE_EN (0x0400) -#define U300_SYSCON_CFRR_SEMI_CLK_FORCE_EN (0x0200) -#define U300_SYSCON_CFRR_XGAM_CLK_FORCE_EN (0x0100) -#define U300_SYSCON_CFRR_VIDEO_ENC_CLK_FORCE_EN (0x0080) -#define U300_SYSCON_CFRR_NANDIF_CLK_FORCE_EN (0x0040) -#define U300_SYSCON_CFRR_EMIF_CLK_FORCE_EN (0x0020) -#define U300_SYSCON_CFRR_DMAC_CLK_FORCE_EN (0x0010) -#define U300_SYSCON_CFRR_CPU_CLK_FORCE_EN (0x0008) -#define U300_SYSCON_CFRR_APEX_CLK_FORCE_EN (0x0004) -#define U300_SYSCON_CFRR_AHB_CLK_FORCE_EN (0x0002) -#define U300_SYSCON_CFRR_AAIF_CLK_FORCE_EN (0x0001) -/* PLL208 Frequency Control 16bit (R/W) */ -#define U300_SYSCON_PFCR (0x48) -#define U300_SYSCON_PFCR_DPLL_MULT_NUM (0x000F) -/* Power Management Control 16bit (R/W) */ -#define U300_SYSCON_PMCR (0x50) -#define U300_SYSCON_PMCR_DCON_ENABLE (0x0002) -#define U300_SYSCON_PMCR_PWR_MGNT_ENABLE (0x0001) -/* Reset Out 16bit (R/W) */ -#define U300_SYSCON_RCR (0x6c) -#define U300_SYSCON_RCR_RESOUT0_RST_N_DISABLE (0x0001) -/* EMIF Slew Rate Control 16bit (R/W) */ -#define U300_SYSCON_SRCLR (0x70) -#define U300_SYSCON_SRCLR_MASK (0x03FF) -#define U300_SYSCON_SRCLR_VALUE (0x03FF) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_5_B (0x0200) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_5_A (0x0100) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_4_B (0x0080) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_4_A (0x0040) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_3_B (0x0020) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_3_A (0x0010) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_2_B (0x0008) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_2_A (0x0004) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_1_B (0x0002) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_1_A (0x0001) -/* EMIF Clock Control Register 16bit (R/W) */ -#define U300_SYSCON_ECCR (0x0078) -#define U300_SYSCON_ECCR_MASK (0x000F) -#define U300_SYSCON_ECCR_EMIF_1_STATIC_CLK_EN_N_DISABLE (0x0008) -#define U300_SYSCON_ECCR_EMIF_1_RET_OUT_CLK_EN_N_DISABLE (0x0004) -#define U300_SYSCON_ECCR_EMIF_MEMCLK_RET_EN_N_DISABLE (0x0002) -#define U300_SYSCON_ECCR_EMIF_SDRCLK_RET_EN_N_DISABLE (0x0001) -/* MMC/MSPRO frequency divider register 0 16bit (R/W) */ -#define U300_SYSCON_MMF0R (0x90) -#define U300_SYSCON_MMF0R_MASK (0x00FF) -#define U300_SYSCON_MMF0R_FREQ_0_HIGH_MASK (0x00F0) -#define U300_SYSCON_MMF0R_FREQ_0_LOW_MASK (0x000F) -/* MMC/MSPRO frequency divider register 1 16bit (R/W) */ -#define U300_SYSCON_MMF1R (0x94) -#define U300_SYSCON_MMF1R_MASK (0x00FF) -#define U300_SYSCON_MMF1R_FREQ_1_HIGH_MASK (0x00F0) -#define U300_SYSCON_MMF1R_FREQ_1_LOW_MASK (0x000F) -/* Clock control for the MMC and MSPRO blocks 16bit (R/W) */ -#define U300_SYSCON_MMCR (0x9C) -#define U300_SYSCON_MMCR_MASK (0x0003) -#define U300_SYSCON_MMCR_MMC_FB_CLK_SEL_ENABLE (0x0002) -#define U300_SYSCON_MMCR_MSPRO_FREQSEL_ENABLE (0x0001) -/* SYS_0_CLK_CONTROL first clock control 16bit (R/W) */ -#define U300_SYSCON_S0CCR (0x120) -#define U300_SYSCON_S0CCR_FIELD_MASK (0x43FF) -#define U300_SYSCON_S0CCR_CLOCK_REQ (0x4000) -#define U300_SYSCON_S0CCR_CLOCK_REQ_MONITOR (0x2000) -#define U300_SYSCON_S0CCR_CLOCK_INV (0x0200) -#define U300_SYSCON_S0CCR_CLOCK_FREQ_MASK (0x01E0) -#define U300_SYSCON_S0CCR_CLOCK_SELECT_MASK (0x001E) -#define U300_SYSCON_S0CCR_CLOCK_ENABLE (0x0001) -#define U300_SYSCON_S0CCR_SEL_MCLK (0x8 << 1) -#define U300_SYSCON_S0CCR_SEL_ACC_FSM_CLK (0xA << 1) -#define U300_SYSCON_S0CCR_SEL_PLL60_48_CLK (0xC << 1) -#define U300_SYSCON_S0CCR_SEL_PLL60_60_CLK (0xD << 1) -#define U300_SYSCON_S0CCR_SEL_ACC_PLL208_CLK (0xE << 1) -#define U300_SYSCON_S0CCR_SEL_APP_PLL13_CLK (0x0 << 1) -#define U300_SYSCON_S0CCR_SEL_APP_FSM_CLK (0x2 << 1) -#define U300_SYSCON_S0CCR_SEL_RTC_CLK (0x4 << 1) -#define U300_SYSCON_S0CCR_SEL_APP_PLL208_CLK (0x6 << 1) -/* SYS_1_CLK_CONTROL second clock control 16 bit (R/W) */ -#define U300_SYSCON_S1CCR (0x124) -#define U300_SYSCON_S1CCR_FIELD_MASK (0x43FF) -#define U300_SYSCON_S1CCR_CLOCK_REQ (0x4000) -#define U300_SYSCON_S1CCR_CLOCK_REQ_MONITOR (0x2000) -#define U300_SYSCON_S1CCR_CLOCK_INV (0x0200) -#define U300_SYSCON_S1CCR_CLOCK_FREQ_MASK (0x01E0) -#define U300_SYSCON_S1CCR_CLOCK_SELECT_MASK (0x001E) -#define U300_SYSCON_S1CCR_CLOCK_ENABLE (0x0001) -#define U300_SYSCON_S1CCR_SEL_MCLK (0x8 << 1) -#define U300_SYSCON_S1CCR_SEL_ACC_FSM_CLK (0xA << 1) -#define U300_SYSCON_S1CCR_SEL_PLL60_48_CLK (0xC << 1) -#define U300_SYSCON_S1CCR_SEL_PLL60_60_CLK (0xD << 1) -#define U300_SYSCON_S1CCR_SEL_ACC_PLL208_CLK (0xE << 1) -#define U300_SYSCON_S1CCR_SEL_ACC_PLL13_CLK (0x0 << 1) -#define U300_SYSCON_S1CCR_SEL_APP_FSM_CLK (0x2 << 1) -#define U300_SYSCON_S1CCR_SEL_RTC_CLK (0x4 << 1) -#define U300_SYSCON_S1CCR_SEL_APP_PLL208_CLK (0x6 << 1) -/* SYS_2_CLK_CONTROL third clock control 16 bit (R/W) */ -#define U300_SYSCON_S2CCR (0x128) -#define U300_SYSCON_S2CCR_FIELD_MASK (0xC3FF) -#define U300_SYSCON_S2CCR_CLK_STEAL (0x8000) -#define U300_SYSCON_S2CCR_CLOCK_REQ (0x4000) -#define U300_SYSCON_S2CCR_CLOCK_REQ_MONITOR (0x2000) -#define U300_SYSCON_S2CCR_CLOCK_INV (0x0200) -#define U300_SYSCON_S2CCR_CLOCK_FREQ_MASK (0x01E0) -#define U300_SYSCON_S2CCR_CLOCK_SELECT_MASK (0x001E) -#define U300_SYSCON_S2CCR_CLOCK_ENABLE (0x0001) -#define U300_SYSCON_S2CCR_SEL_MCLK (0x8 << 1) -#define U300_SYSCON_S2CCR_SEL_ACC_FSM_CLK (0xA << 1) -#define U300_SYSCON_S2CCR_SEL_PLL60_48_CLK (0xC << 1) -#define U300_SYSCON_S2CCR_SEL_PLL60_60_CLK (0xD << 1) -#define U300_SYSCON_S2CCR_SEL_ACC_PLL208_CLK (0xE << 1) -#define U300_SYSCON_S2CCR_SEL_ACC_PLL13_CLK (0x0 << 1) -#define U300_SYSCON_S2CCR_SEL_APP_FSM_CLK (0x2 << 1) -#define U300_SYSCON_S2CCR_SEL_RTC_CLK (0x4 << 1) -#define U300_SYSCON_S2CCR_SEL_APP_PLL208_CLK (0x6 << 1) -/* SC_PLL_IRQ_CONTROL 16bit (R/W) */ -#define U300_SYSCON_PICR (0x0130) -#define U300_SYSCON_PICR_MASK (0x00FF) -#define U300_SYSCON_PICR_FORCE_PLL208_LOCK_LOW_ENABLE (0x0080) -#define U300_SYSCON_PICR_FORCE_PLL208_LOCK_HIGH_ENABLE (0x0040) -#define U300_SYSCON_PICR_FORCE_PLL13_LOCK_LOW_ENABLE (0x0020) -#define U300_SYSCON_PICR_FORCE_PLL13_LOCK_HIGH_ENABLE (0x0010) -#define U300_SYSCON_PICR_IRQMASK_PLL13_UNLOCK_ENABLE (0x0008) -#define U300_SYSCON_PICR_IRQMASK_PLL13_LOCK_ENABLE (0x0004) -#define U300_SYSCON_PICR_IRQMASK_PLL208_UNLOCK_ENABLE (0x0002) -#define U300_SYSCON_PICR_IRQMASK_PLL208_LOCK_ENABLE (0x0001) -/* SC_PLL_IRQ_STATUS 16 bit (R/-) */ -#define U300_SYSCON_PISR (0x0134) -#define U300_SYSCON_PISR_MASK (0x000F) -#define U300_SYSCON_PISR_PLL13_UNLOCK_IND (0x0008) -#define U300_SYSCON_PISR_PLL13_LOCK_IND (0x0004) -#define U300_SYSCON_PISR_PLL208_UNLOCK_IND (0x0002) -#define U300_SYSCON_PISR_PLL208_LOCK_IND (0x0001) -/* SC_PLL_IRQ_CLEAR 16 bit (-/W) */ -#define U300_SYSCON_PICLR (0x0138) -#define U300_SYSCON_PICLR_MASK (0x000F) -#define U300_SYSCON_PICLR_RWMASK (0x0000) -#define U300_SYSCON_PICLR_PLL13_UNLOCK_SC (0x0008) -#define U300_SYSCON_PICLR_PLL13_LOCK_SC (0x0004) -#define U300_SYSCON_PICLR_PLL208_UNLOCK_SC (0x0002) -#define U300_SYSCON_PICLR_PLL208_LOCK_SC (0x0001) -/* Clock activity observability register 0 */ -#define U300_SYSCON_C0OAR (0x140) -#define U300_SYSCON_C0OAR_MASK (0xFFFF) -#define U300_SYSCON_C0OAR_VALUE (0xFFFF) -#define U300_SYSCON_C0OAR_BT_H_CLK (0x8000) -#define U300_SYSCON_C0OAR_ASPB_P_CLK (0x4000) -#define U300_SYSCON_C0OAR_APP_SEMI_H_CLK (0x2000) -#define U300_SYSCON_C0OAR_APP_SEMI_CLK (0x1000) -#define U300_SYSCON_C0OAR_APP_MMC_MSPRO_CLK (0x0800) -#define U300_SYSCON_C0OAR_APP_I2S1_CLK (0x0400) -#define U300_SYSCON_C0OAR_APP_I2S0_CLK (0x0200) -#define U300_SYSCON_C0OAR_APP_CPU_CLK (0x0100) -#define U300_SYSCON_C0OAR_APP_52_CLK (0x0080) -#define U300_SYSCON_C0OAR_APP_208_CLK (0x0040) -#define U300_SYSCON_C0OAR_APP_104_CLK (0x0020) -#define U300_SYSCON_C0OAR_APEX_CLK (0x0010) -#define U300_SYSCON_C0OAR_AHPB_M_H_CLK (0x0008) -#define U300_SYSCON_C0OAR_AHB_CLK (0x0004) -#define U300_SYSCON_C0OAR_AFPB_P_CLK (0x0002) -#define U300_SYSCON_C0OAR_AAIF_CLK (0x0001) -/* Clock activity observability register 1 */ -#define U300_SYSCON_C1OAR (0x144) -#define U300_SYSCON_C1OAR_MASK (0x3FFE) -#define U300_SYSCON_C1OAR_VALUE (0x3FFE) -#define U300_SYSCON_C1OAR_NFIF_F_CLK (0x2000) -#define U300_SYSCON_C1OAR_MSPRO_CLK (0x1000) -#define U300_SYSCON_C1OAR_MMC_P_CLK (0x0800) -#define U300_SYSCON_C1OAR_MMC_CLK (0x0400) -#define U300_SYSCON_C1OAR_KP_P_CLK (0x0200) -#define U300_SYSCON_C1OAR_I2C1_P_CLK (0x0100) -#define U300_SYSCON_C1OAR_I2C0_P_CLK (0x0080) -#define U300_SYSCON_C1OAR_GPIO_CLK (0x0040) -#define U300_SYSCON_C1OAR_EMIF_MPMC_CLK (0x0020) -#define U300_SYSCON_C1OAR_EMIF_H_CLK (0x0010) -#define U300_SYSCON_C1OAR_EVHIST_CLK (0x0008) -#define U300_SYSCON_C1OAR_PPM_CLK (0x0004) -#define U300_SYSCON_C1OAR_DMA_CLK (0x0002) -/* Clock activity observability register 2 */ -#define U300_SYSCON_C2OAR (0x148) -#define U300_SYSCON_C2OAR_MASK (0x0FFF) -#define U300_SYSCON_C2OAR_VALUE (0x0FFF) -#define U300_SYSCON_C2OAR_XGAM_CDI_CLK (0x0800) -#define U300_SYSCON_C2OAR_XGAM_CLK (0x0400) -#define U300_SYSCON_C2OAR_VC_H_CLK (0x0200) -#define U300_SYSCON_C2OAR_VC_CLK (0x0100) -#define U300_SYSCON_C2OAR_UA_P_CLK (0x0080) -#define U300_SYSCON_C2OAR_TMR1_CLK (0x0040) -#define U300_SYSCON_C2OAR_TMR0_CLK (0x0020) -#define U300_SYSCON_C2OAR_SPI_P_CLK (0x0010) -#define U300_SYSCON_C2OAR_PCM_I2S1_CORE_CLK (0x0008) -#define U300_SYSCON_C2OAR_PCM_I2S1_CLK (0x0004) -#define U300_SYSCON_C2OAR_PCM_I2S0_CORE_CLK (0x0002) -#define U300_SYSCON_C2OAR_PCM_I2S0_CLK (0x0001) - - -/* - * The clocking hierarchy currently looks like this. - * NOTE: the idea is NOT to show how the clocks are routed on the chip! - * The ideas is to show dependencies, so a clock higher up in the - * hierarchy has to be on in order for another clock to be on. Now, - * both CPU and DMA can actually be on top of the hierarchy, and that - * is not modeled currently. Instead we have the backbone AMBA bus on - * top. This bus cannot be programmed in any way but conceptually it - * needs to be active for the bridges and devices to transport data. - * - * Please be aware that a few clocks are hw controlled, which mean that - * the hw itself can turn on/off or change the rate of the clock when - * needed! - * - * AMBA bus - * | - * +- CPU - * +- FSMC NANDIF NAND Flash interface - * +- SEMI Shared Memory interface - * +- ISP Image Signal Processor (U335 only) - * +- CDS (U335 only) - * +- DMA Direct Memory Access Controller - * +- AAIF APP/ACC Interface (Mobile Scalable Link, MSL) - * +- APEX - * +- VIDEO_ENC AVE2/3 Video Encoder - * +- XGAM Graphics Accelerator Controller - * +- AHB - * | - * +- ahb:0 AHB Bridge - * | | - * | +- ahb:1 INTCON Interrupt controller - * | +- ahb:3 MSPRO Memory Stick Pro controller - * | +- ahb:4 EMIF External Memory interface - * | - * +- fast:0 FAST bridge - * | | - * | +- fast:1 MMCSD MMC/SD card reader controller - * | +- fast:2 I2S0 PCM I2S channel 0 controller - * | +- fast:3 I2S1 PCM I2S channel 1 controller - * | +- fast:4 I2C0 I2C channel 0 controller - * | +- fast:5 I2C1 I2C channel 1 controller - * | +- fast:6 SPI SPI controller - * | +- fast:7 UART1 Secondary UART (U335 only) - * | - * +- slow:0 SLOW bridge - * | - * +- slow:1 SYSCON (not possible to control) - * +- slow:2 WDOG Watchdog - * +- slow:3 UART0 primary UART - * +- slow:4 TIMER_APP Application timer - used in Linux - * +- slow:5 KEYPAD controller - * +- slow:6 GPIO controller - * +- slow:7 RTC controller - * +- slow:8 BT Bus Tracer (not used currently) - * +- slow:9 EH Event Handler (not used currently) - * +- slow:a TIMER_ACC Access style timer (not used currently) - * +- slow:b PPM (U335 only, what is that?) - */ - -/* Global syscon virtual base */ -static void __iomem *syscon_vbase; - -/** - * struct clk_syscon - U300 syscon clock - * @hw: corresponding clock hardware entry - * @hw_ctrld: whether this clock is hardware controlled (for refcount etc) - * and does not need any magic pokes to be enabled/disabled - * @reset: state holder, whether this block's reset line is asserted or not - * @res_reg: reset line enable/disable flag register - * @res_bit: bit for resetting or taking this consumer out of reset - * @en_reg: clock line enable/disable flag register - * @en_bit: bit for enabling/disabling this consumer clock line - * @clk_val: magic value to poke in the register to enable/disable - * this one clock - */ -struct clk_syscon { - struct clk_hw hw; - bool hw_ctrld; - bool reset; - void __iomem *res_reg; - u8 res_bit; - void __iomem *en_reg; - u8 en_bit; - u16 clk_val; -}; - -#define to_syscon(_hw) container_of(_hw, struct clk_syscon, hw) - -static DEFINE_SPINLOCK(syscon_resetreg_lock); - -/* - * Reset control functions. We remember if a block has been - * taken out of reset and don't remove the reset assertion again - * and vice versa. Currently we only remove resets so the - * enablement function is defined out. - */ -static void syscon_block_reset_enable(struct clk_syscon *sclk) -{ - unsigned long iflags; - u16 val; - - /* Not all blocks support resetting */ - if (!sclk->res_reg) - return; - spin_lock_irqsave(&syscon_resetreg_lock, iflags); - val = readw(sclk->res_reg); - val |= BIT(sclk->res_bit); - writew(val, sclk->res_reg); - spin_unlock_irqrestore(&syscon_resetreg_lock, iflags); - sclk->reset = true; -} - -static void syscon_block_reset_disable(struct clk_syscon *sclk) -{ - unsigned long iflags; - u16 val; - - /* Not all blocks support resetting */ - if (!sclk->res_reg) - return; - spin_lock_irqsave(&syscon_resetreg_lock, iflags); - val = readw(sclk->res_reg); - val &= ~BIT(sclk->res_bit); - writew(val, sclk->res_reg); - spin_unlock_irqrestore(&syscon_resetreg_lock, iflags); - sclk->reset = false; -} - -static int syscon_clk_prepare(struct clk_hw *hw) -{ - struct clk_syscon *sclk = to_syscon(hw); - - /* If the block is in reset, bring it out */ - if (sclk->reset) - syscon_block_reset_disable(sclk); - return 0; -} - -static void syscon_clk_unprepare(struct clk_hw *hw) -{ - struct clk_syscon *sclk = to_syscon(hw); - - /* Please don't force the console into reset */ - if (sclk->clk_val == U300_SYSCON_SBCER_UART_CLK_EN) - return; - /* When unpreparing, force block into reset */ - if (!sclk->reset) - syscon_block_reset_enable(sclk); -} - -static int syscon_clk_enable(struct clk_hw *hw) -{ - struct clk_syscon *sclk = to_syscon(hw); - - /* Don't touch the hardware controlled clocks */ - if (sclk->hw_ctrld) - return 0; - /* These cannot be controlled */ - if (sclk->clk_val == 0xFFFFU) - return 0; - - writew(sclk->clk_val, syscon_vbase + U300_SYSCON_SBCER); - return 0; -} - -static void syscon_clk_disable(struct clk_hw *hw) -{ - struct clk_syscon *sclk = to_syscon(hw); - - /* Don't touch the hardware controlled clocks */ - if (sclk->hw_ctrld) - return; - if (sclk->clk_val == 0xFFFFU) - return; - /* Please don't disable the console port */ - if (sclk->clk_val == U300_SYSCON_SBCER_UART_CLK_EN) - return; - - writew(sclk->clk_val, syscon_vbase + U300_SYSCON_SBCDR); -} - -static int syscon_clk_is_enabled(struct clk_hw *hw) -{ - struct clk_syscon *sclk = to_syscon(hw); - u16 val; - - /* If no enable register defined, it's always-on */ - if (!sclk->en_reg) - return 1; - - val = readw(sclk->en_reg); - val &= BIT(sclk->en_bit); - - return val ? 1 : 0; -} - -static u16 syscon_get_perf(void) -{ - u16 val; - - val = readw(syscon_vbase + U300_SYSCON_CCR); - val &= U300_SYSCON_CCR_CLKING_PERFORMANCE_MASK; - return val; -} - -static unsigned long -syscon_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct clk_syscon *sclk = to_syscon(hw); - u16 perf = syscon_get_perf(); - - switch (sclk->clk_val) { - case U300_SYSCON_SBCER_FAST_BRIDGE_CLK_EN: - case U300_SYSCON_SBCER_I2C0_CLK_EN: - case U300_SYSCON_SBCER_I2C1_CLK_EN: - case U300_SYSCON_SBCER_MMC_CLK_EN: - case U300_SYSCON_SBCER_SPI_CLK_EN: - /* The FAST clocks have one progression */ - switch (perf) { - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW: - return 13000000; - default: - return parent_rate; /* 26 MHz */ - } - case U300_SYSCON_SBCER_DMAC_CLK_EN: - case U300_SYSCON_SBCER_NANDIF_CLK_EN: - case U300_SYSCON_SBCER_XGAM_CLK_EN: - /* AMBA interconnect peripherals */ - switch (perf) { - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW: - return 6500000; - case U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE: - return 26000000; - default: - return parent_rate; /* 52 MHz */ - } - case U300_SYSCON_SBCER_SEMI_CLK_EN: - case U300_SYSCON_SBCER_EMIF_CLK_EN: - /* EMIF speeds */ - switch (perf) { - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW: - return 13000000; - case U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE: - return 52000000; - default: - return 104000000; - } - case U300_SYSCON_SBCER_CPU_CLK_EN: - /* And the fast CPU clock */ - switch (perf) { - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW: - return 13000000; - case U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE: - return 52000000; - case U300_SYSCON_CCR_CLKING_PERFORMANCE_HIGH: - return 104000000; - default: - return parent_rate; /* 208 MHz */ - } - default: - /* - * The SLOW clocks and default just inherit the rate of - * their parent (typically PLL13 13 MHz). - */ - return parent_rate; - } -} - -static long -syscon_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - struct clk_syscon *sclk = to_syscon(hw); - - if (sclk->clk_val != U300_SYSCON_SBCER_CPU_CLK_EN) - return *prate; - /* We really only support setting the rate of the CPU clock */ - if (rate <= 13000000) - return 13000000; - if (rate <= 52000000) - return 52000000; - if (rate <= 104000000) - return 104000000; - return 208000000; -} - -static int syscon_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct clk_syscon *sclk = to_syscon(hw); - u16 val; - - /* We only support setting the rate of the CPU clock */ - if (sclk->clk_val != U300_SYSCON_SBCER_CPU_CLK_EN) - return -EINVAL; - switch (rate) { - case 13000000: - val = U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER; - break; - case 52000000: - val = U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE; - break; - case 104000000: - val = U300_SYSCON_CCR_CLKING_PERFORMANCE_HIGH; - break; - case 208000000: - val = U300_SYSCON_CCR_CLKING_PERFORMANCE_BEST; - break; - default: - return -EINVAL; - } - val |= readw(syscon_vbase + U300_SYSCON_CCR) & - ~U300_SYSCON_CCR_CLKING_PERFORMANCE_MASK ; - writew(val, syscon_vbase + U300_SYSCON_CCR); - return 0; -} - -static const struct clk_ops syscon_clk_ops = { - .prepare = syscon_clk_prepare, - .unprepare = syscon_clk_unprepare, - .enable = syscon_clk_enable, - .disable = syscon_clk_disable, - .is_enabled = syscon_clk_is_enabled, - .recalc_rate = syscon_clk_recalc_rate, - .round_rate = syscon_clk_round_rate, - .set_rate = syscon_clk_set_rate, -}; - -static struct clk_hw * __init -syscon_clk_register(struct device *dev, const char *name, - const char *parent_name, unsigned long flags, - bool hw_ctrld, - void __iomem *res_reg, u8 res_bit, - void __iomem *en_reg, u8 en_bit, - u16 clk_val) -{ - struct clk_hw *hw; - struct clk_syscon *sclk; - struct clk_init_data init; - int ret; - - sclk = kzalloc(sizeof(*sclk), GFP_KERNEL); - if (!sclk) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = &syscon_clk_ops; - init.flags = flags; - init.parent_names = (parent_name ? &parent_name : NULL); - init.num_parents = (parent_name ? 1 : 0); - sclk->hw.init = &init; - sclk->hw_ctrld = hw_ctrld; - /* Assume the block is in reset at registration */ - sclk->reset = true; - sclk->res_reg = res_reg; - sclk->res_bit = res_bit; - sclk->en_reg = en_reg; - sclk->en_bit = en_bit; - sclk->clk_val = clk_val; - - hw = &sclk->hw; - ret = clk_hw_register(dev, hw); - if (ret) { - kfree(sclk); - hw = ERR_PTR(ret); - } - - return hw; -} - -#define U300_CLK_TYPE_SLOW 0 -#define U300_CLK_TYPE_FAST 1 -#define U300_CLK_TYPE_REST 2 - -/** - * struct u300_clock - defines the bits and pieces for a certain clock - * @type: the clock type, slow fast or rest - * @id: the bit in the slow/fast/rest register for this clock - * @hw_ctrld: whether the clock is hardware controlled - * @clk_val: a value to poke in the one-write enable/disable registers - */ -struct u300_clock { - u8 type; - u8 id; - bool hw_ctrld; - u16 clk_val; -}; - -static struct u300_clock const u300_clk_lookup[] __initconst = { - { - .type = U300_CLK_TYPE_REST, - .id = 3, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_CPU_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 4, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_DMAC_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 5, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_EMIF_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 6, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_NANDIF_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 8, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_XGAM_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 9, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_SEMI_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 10, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_AHB_SUBSYS_BRIDGE_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 12, - .hw_ctrld = false, - /* INTCON: cannot be enabled, just taken out of reset */ - .clk_val = 0xFFFFU, - }, - { - .type = U300_CLK_TYPE_FAST, - .id = 0, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_FAST_BRIDGE_CLK_EN, - }, - { - .type = U300_CLK_TYPE_FAST, - .id = 1, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_I2C0_CLK_EN, - }, - { - .type = U300_CLK_TYPE_FAST, - .id = 2, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_I2C1_CLK_EN, - }, - { - .type = U300_CLK_TYPE_FAST, - .id = 5, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_MMC_CLK_EN, - }, - { - .type = U300_CLK_TYPE_FAST, - .id = 6, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_SPI_CLK_EN, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 0, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_SLOW_BRIDGE_CLK_EN, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 1, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_UART_CLK_EN, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 4, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_GPIO_CLK_EN, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 6, - .hw_ctrld = true, - /* No clock enable register bit */ - .clk_val = 0xFFFFU, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 7, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_APP_TMR_CLK_EN, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 8, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_ACC_TMR_CLK_EN, - }, -}; - -static void __init of_u300_syscon_clk_init(struct device_node *np) -{ - struct clk_hw *hw = ERR_PTR(-EINVAL); - const char *clk_name = np->name; - const char *parent_name; - void __iomem *res_reg; - void __iomem *en_reg; - u32 clk_type; - u32 clk_id; - int i; - - if (of_property_read_u32(np, "clock-type", &clk_type)) { - pr_err("%s: syscon clock \"%s\" missing clock-type property\n", - __func__, clk_name); - return; - } - if (of_property_read_u32(np, "clock-id", &clk_id)) { - pr_err("%s: syscon clock \"%s\" missing clock-id property\n", - __func__, clk_name); - return; - } - parent_name = of_clk_get_parent_name(np, 0); - - switch (clk_type) { - case U300_CLK_TYPE_SLOW: - res_reg = syscon_vbase + U300_SYSCON_RSR; - en_reg = syscon_vbase + U300_SYSCON_CESR; - break; - case U300_CLK_TYPE_FAST: - res_reg = syscon_vbase + U300_SYSCON_RFR; - en_reg = syscon_vbase + U300_SYSCON_CEFR; - break; - case U300_CLK_TYPE_REST: - res_reg = syscon_vbase + U300_SYSCON_RRR; - en_reg = syscon_vbase + U300_SYSCON_CERR; - break; - default: - pr_err("unknown clock type %x specified\n", clk_type); - return; - } - - for (i = 0; i < ARRAY_SIZE(u300_clk_lookup); i++) { - const struct u300_clock *u3clk = &u300_clk_lookup[i]; - - if (u3clk->type == clk_type && u3clk->id == clk_id) - hw = syscon_clk_register(NULL, clk_name, parent_name, - 0, u3clk->hw_ctrld, - res_reg, u3clk->id, - en_reg, u3clk->id, - u3clk->clk_val); - } - - if (!IS_ERR(hw)) { - of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); - - /* - * Some few system clocks - device tree does not - * represent clocks without a corresponding device node. - * for now we add these three clocks here. - */ - if (clk_type == U300_CLK_TYPE_REST && clk_id == 5) - clk_hw_register_clkdev(hw, NULL, "pl172"); - if (clk_type == U300_CLK_TYPE_REST && clk_id == 9) - clk_hw_register_clkdev(hw, NULL, "semi"); - if (clk_type == U300_CLK_TYPE_REST && clk_id == 12) - clk_hw_register_clkdev(hw, NULL, "intcon"); - } -} - -/** - * struct clk_mclk - U300 MCLK clock (MMC/SD clock) - * @hw: corresponding clock hardware entry - * @is_mspro: if this is the memory stick clock rather than MMC/SD - */ -struct clk_mclk { - struct clk_hw hw; - bool is_mspro; -}; - -#define to_mclk(_hw) container_of(_hw, struct clk_mclk, hw) - -static int mclk_clk_prepare(struct clk_hw *hw) -{ - struct clk_mclk *mclk = to_mclk(hw); - u16 val; - - /* The MMC and MSPRO clocks need some special set-up */ - if (!mclk->is_mspro) { - /* Set default MMC clock divisor to 18.9 MHz */ - writew(0x0054U, syscon_vbase + U300_SYSCON_MMF0R); - val = readw(syscon_vbase + U300_SYSCON_MMCR); - /* Disable the MMC feedback clock */ - val &= ~U300_SYSCON_MMCR_MMC_FB_CLK_SEL_ENABLE; - /* Disable MSPRO frequency */ - val &= ~U300_SYSCON_MMCR_MSPRO_FREQSEL_ENABLE; - writew(val, syscon_vbase + U300_SYSCON_MMCR); - } else { - val = readw(syscon_vbase + U300_SYSCON_MMCR); - /* Disable the MMC feedback clock */ - val &= ~U300_SYSCON_MMCR_MMC_FB_CLK_SEL_ENABLE; - /* Enable MSPRO frequency */ - val |= U300_SYSCON_MMCR_MSPRO_FREQSEL_ENABLE; - writew(val, syscon_vbase + U300_SYSCON_MMCR); - } - - return 0; -} - -static unsigned long -mclk_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - u16 perf = syscon_get_perf(); - - switch (perf) { - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER: - /* - * Here, the 208 MHz PLL gets shut down and the always - * on 13 MHz PLL used for RTC etc kicks into use - * instead. - */ - return 13000000; - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_HIGH: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_BEST: - { - /* - * This clock is under program control. The register is - * divided in two nybbles, bit 7-4 gives cycles-1 to count - * high, bit 3-0 gives cycles-1 to count low. Distribute - * these with no more than 1 cycle difference between - * low and high and add low and high to get the actual - * divisor. The base PLL is 208 MHz. Writing 0x00 will - * divide by 1 and 1 so the highest frequency possible - * is 104 MHz. - * - * e.g. 0x54 => - * f = 208 / ((5+1) + (4+1)) = 208 / 11 = 18.9 MHz - */ - u16 val = readw(syscon_vbase + U300_SYSCON_MMF0R) & - U300_SYSCON_MMF0R_MASK; - switch (val) { - case 0x0054: - return 18900000; - case 0x0044: - return 20800000; - case 0x0043: - return 23100000; - case 0x0033: - return 26000000; - case 0x0032: - return 29700000; - case 0x0022: - return 34700000; - case 0x0021: - return 41600000; - case 0x0011: - return 52000000; - case 0x0000: - return 104000000; - default: - break; - } - } - default: - break; - } - return parent_rate; -} - -static long -mclk_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - if (rate <= 18900000) - return 18900000; - if (rate <= 20800000) - return 20800000; - if (rate <= 23100000) - return 23100000; - if (rate <= 26000000) - return 26000000; - if (rate <= 29700000) - return 29700000; - if (rate <= 34700000) - return 34700000; - if (rate <= 41600000) - return 41600000; - /* Highest rate */ - return 52000000; -} - -static int mclk_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - u16 val; - u16 reg; - - switch (rate) { - case 18900000: - val = 0x0054; - break; - case 20800000: - val = 0x0044; - break; - case 23100000: - val = 0x0043; - break; - case 26000000: - val = 0x0033; - break; - case 29700000: - val = 0x0032; - break; - case 34700000: - val = 0x0022; - break; - case 41600000: - val = 0x0021; - break; - case 52000000: - val = 0x0011; - break; - case 104000000: - val = 0x0000; - break; - default: - return -EINVAL; - } - - reg = readw(syscon_vbase + U300_SYSCON_MMF0R) & - ~U300_SYSCON_MMF0R_MASK; - writew(reg | val, syscon_vbase + U300_SYSCON_MMF0R); - return 0; -} - -static const struct clk_ops mclk_ops = { - .prepare = mclk_clk_prepare, - .recalc_rate = mclk_clk_recalc_rate, - .round_rate = mclk_clk_round_rate, - .set_rate = mclk_clk_set_rate, -}; - -static struct clk_hw * __init -mclk_clk_register(struct device *dev, const char *name, - const char *parent_name, bool is_mspro) -{ - struct clk_hw *hw; - struct clk_mclk *mclk; - struct clk_init_data init; - int ret; - - mclk = kzalloc(sizeof(*mclk), GFP_KERNEL); - if (!mclk) - return ERR_PTR(-ENOMEM); - - init.name = "mclk"; - init.ops = &mclk_ops; - init.flags = 0; - init.parent_names = (parent_name ? &parent_name : NULL); - init.num_parents = (parent_name ? 1 : 0); - mclk->hw.init = &init; - mclk->is_mspro = is_mspro; - - hw = &mclk->hw; - ret = clk_hw_register(dev, hw); - if (ret) { - kfree(mclk); - hw = ERR_PTR(ret); - } - - return hw; -} - -static void __init of_u300_syscon_mclk_init(struct device_node *np) -{ - struct clk_hw *hw; - const char *clk_name = np->name; - const char *parent_name; - - parent_name = of_clk_get_parent_name(np, 0); - hw = mclk_clk_register(NULL, clk_name, parent_name, false); - if (!IS_ERR(hw)) - of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); -} - -static const struct of_device_id u300_clk_match[] __initconst = { - { - .compatible = "fixed-clock", - .data = of_fixed_clk_setup, - }, - { - .compatible = "fixed-factor-clock", - .data = of_fixed_factor_clk_setup, - }, - { - .compatible = "stericsson,u300-syscon-clk", - .data = of_u300_syscon_clk_init, - }, - { - .compatible = "stericsson,u300-syscon-mclk", - .data = of_u300_syscon_mclk_init, - }, - {} -}; - - -void __init u300_clk_init(void __iomem *base) -{ - u16 val; - - syscon_vbase = base; - - /* Set system to run at PLL208, max performance, a known state. */ - val = readw(syscon_vbase + U300_SYSCON_CCR); - val &= ~U300_SYSCON_CCR_CLKING_PERFORMANCE_MASK; - writew(val, syscon_vbase + U300_SYSCON_CCR); - /* Wait for the PLL208 to lock if not locked in yet */ - while (!(readw(syscon_vbase + U300_SYSCON_CSR) & - U300_SYSCON_CSR_PLL208_LOCK_IND)); - - /* Power management enable */ - val = readw(syscon_vbase + U300_SYSCON_PMCR); - val |= U300_SYSCON_PMCR_PWR_MGNT_ENABLE; - writew(val, syscon_vbase + U300_SYSCON_PMCR); - - of_clk_init(u300_clk_match); -} diff --git a/drivers/clk/clk-versaclock5.c b/drivers/clk/clk-versaclock5.c index 43db67337bc0..344cd6c61188 100644 --- a/drivers/clk/clk-versaclock5.c +++ b/drivers/clk/clk-versaclock5.c @@ -759,6 +759,63 @@ static int vc5_update_power(struct device_node *np_output, return 0; } +static int vc5_map_cap_value(u32 femtofarads) +{ + int mapped_value; + + /* + * The datasheet explicitly states 9000 - 25000 with 0.5pF + * steps, but the Programmer's guide shows the steps are 0.430pF. + * After getting feedback from Renesas, the .5pF steps were the + * goal, but 430nF was the actual values. + * Because of this, the actual range goes to 22760 instead of 25000 + */ + if (femtofarads < 9000 || femtofarads > 22760) + return -EINVAL; + + /* + * The Programmer's guide shows XTAL[5:0] but in reality, + * XTAL[0] and XTAL[1] are both LSB which makes the math + * strange. With clarfication from Renesas, setting the + * values should be simpler by ignoring XTAL[0] + */ + mapped_value = DIV_ROUND_CLOSEST(femtofarads - 9000, 430); + + /* + * Since the calculation ignores XTAL[0], there is one + * special case where mapped_value = 32. In reality, this means + * the real mapped value should be 111111b. In other cases, + * the mapped_value needs to be shifted 1 to the left. + */ + if (mapped_value > 31) + mapped_value = 0x3f; + else + mapped_value <<= 1; + + return mapped_value; +} +static int vc5_update_cap_load(struct device_node *node, struct vc5_driver_data *vc5) +{ + u32 value; + int mapped_value; + + if (!of_property_read_u32(node, "idt,xtal-load-femtofarads", &value)) { + mapped_value = vc5_map_cap_value(value); + if (mapped_value < 0) + return mapped_value; + + /* + * The mapped_value is really the high 6 bits of + * VC5_XTAL_X1_LOAD_CAP and VC5_XTAL_X2_LOAD_CAP, so + * shift the value 2 places. + */ + regmap_update_bits(vc5->regmap, VC5_XTAL_X1_LOAD_CAP, ~0x03, mapped_value << 2); + regmap_update_bits(vc5->regmap, VC5_XTAL_X2_LOAD_CAP, ~0x03, mapped_value << 2); + } + + return 0; +} + static int vc5_update_slew(struct device_node *np_output, struct vc5_out_data *clk_out) { @@ -884,6 +941,13 @@ static int vc5_probe(struct i2c_client *client, const struct i2c_device_id *id) return -EINVAL; } + /* Configure Optional Loading Capacitance for external XTAL */ + if (!(vc5->chip_info->flags & VC5_HAS_INTERNAL_XTAL)) { + ret = vc5_update_cap_load(client->dev.of_node, vc5); + if (ret) + goto err_clk_register; + } + init.name = kasprintf(GFP_KERNEL, "%pOFn.mux", client->dev.of_node); init.ops = &vc5_mux_ops; init.flags = 0; diff --git a/drivers/clk/clk-xgene.c b/drivers/clk/clk-xgene.c index 3fd53057c01f..857217cbcef8 100644 --- a/drivers/clk/clk-xgene.c +++ b/drivers/clk/clk-xgene.c @@ -206,17 +206,16 @@ static void xgene_pcppllclk_init(struct device_node *np) * @hw: handle between common and hardware-specific interfaces * @reg: register containing the fractional scale multiplier (scaler) * @shift: shift to the unit bit field + * @mask: mask to the unit bit field * @denom: 1/denominator unit * @lock: register lock - * Flags: - * XGENE_CLK_PMD_SCALE_INVERTED - By default the scaler is the value read + * @flags: XGENE_CLK_PMD_SCALE_INVERTED - By default the scaler is the value read * from the register plus one. For example, * 0 for (0 + 1) / denom, * 1 for (1 + 1) / denom and etc. * If this flag is set, it is * 0 for (denom - 0) / denom, * 1 for (denom - 1) / denom and etc. - * */ struct xgene_clk_pmd { struct clk_hw hw; diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 3d751ae5bc70..5052541a0986 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -4576,6 +4576,8 @@ int of_clk_add_provider(struct device_node *np, if (ret < 0) of_clk_del_provider(np); + fwnode_dev_initialized(&np->fwnode, true); + return ret; } EXPORT_SYMBOL_GPL(of_clk_add_provider); @@ -4693,6 +4695,7 @@ void of_clk_del_provider(struct device_node *np) list_for_each_entry(cp, &of_clk_providers, link) { if (cp->node == np) { list_del(&cp->link); + fwnode_dev_initialized(&np->fwnode, false); of_node_put(cp->node); kfree(cp); break; diff --git a/drivers/clk/imx/clk-imx31.c b/drivers/clk/imx/clk-imx31.c index 7b13fb57d842..c44e18c6f63f 100644 --- a/drivers/clk/imx/clk-imx31.c +++ b/drivers/clk/imx/clk-imx31.c @@ -51,16 +51,6 @@ enum mx31_clks { static struct clk *clk[clk_max]; static struct clk_onecell_data clk_data; -static struct clk ** const uart_clks[] __initconst = { - &clk[ipg], - &clk[uart1_gate], - &clk[uart2_gate], - &clk[uart3_gate], - &clk[uart4_gate], - &clk[uart5_gate], - NULL -}; - static void __init _mx31_clocks_init(void __iomem *base, unsigned long fref) { clk[dummy] = imx_clk_fixed("dummy", 0); diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c index b2ff187cedab..521d6136d22c 100644 --- a/drivers/clk/imx/clk-imx6q.c +++ b/drivers/clk/imx/clk-imx6q.c @@ -338,10 +338,10 @@ static void init_ldb_clks(struct device_node *np, void __iomem *ccm_base) of_assigned_ldb_sels(np, &sel[0][3], &sel[1][3]); for (i = 0; i < 2; i++) { - /* Warn if a glitch might have been introduced already */ + /* Print a notice if a glitch might have been introduced already */ if (sel[i][0] != 3) { - pr_warn("ccm: ldb_di%d_sel already changed from reset value: %d\n", - i, sel[i][0]); + pr_notice("ccm: possible glitch: ldb_di%d_sel already changed from reset value: %d\n", + i, sel[i][0]); } if (sel[i][0] == sel[i][3]) diff --git a/drivers/clk/imx/clk-imx6sl.c b/drivers/clk/imx/clk-imx6sl.c index 2f9361946a0e..29eab05c9068 100644 --- a/drivers/clk/imx/clk-imx6sl.c +++ b/drivers/clk/imx/clk-imx6sl.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/clk/imx/clk-imx8mm.c b/drivers/clk/imx/clk-imx8mm.c index 7c905861af5d..6a01eec36dd0 100644 --- a/drivers/clk/imx/clk-imx8mm.c +++ b/drivers/clk/imx/clk-imx8mm.c @@ -288,6 +288,11 @@ static const char *imx8mm_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "dummy", " static const char *imx8mm_clko2_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_400m", "sys_pll2_166m", "sys_pll3_out", "audio_pll1_out", "video_pll1_out", "osc_32k", }; +static const char * const clkout_sels[] = {"audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "dummy", "dummy", "gpu_pll_out", "vpu_pll_out", + "arm_pll_out", "sys_pll1", "sys_pll2", "sys_pll3", + "dummy", "dummy", "osc_24m", "dummy", "osc_32k"}; + static struct clk_hw_onecell_data *clk_hw_data; static struct clk_hw **hws; @@ -410,6 +415,13 @@ static int imx8mm_clocks_probe(struct platform_device *pdev) hws[IMX8MM_SYS_PLL2_500M] = imx_clk_hw_fixed_factor("sys_pll2_500m", "sys_pll2_500m_cg", 1, 2); hws[IMX8MM_SYS_PLL2_1000M] = imx_clk_hw_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1); + hws[IMX8MM_CLK_CLKOUT1_SEL] = imx_clk_hw_mux("clkout1_sel", base + 0x128, 4, 4, clkout_sels, ARRAY_SIZE(clkout_sels)); + hws[IMX8MM_CLK_CLKOUT1_DIV] = imx_clk_hw_divider("clkout1_div", "clkout1_sel", base + 0x128, 0, 4); + hws[IMX8MM_CLK_CLKOUT1] = imx_clk_hw_gate("clkout1", "clkout1_div", base + 0x128, 8); + hws[IMX8MM_CLK_CLKOUT2_SEL] = imx_clk_hw_mux("clkout2_sel", base + 0x128, 20, 4, clkout_sels, ARRAY_SIZE(clkout_sels)); + hws[IMX8MM_CLK_CLKOUT2_DIV] = imx_clk_hw_divider("clkout2_div", "clkout2_sel", base + 0x128, 16, 4); + hws[IMX8MM_CLK_CLKOUT2] = imx_clk_hw_gate("clkout2", "clkout2_div", base + 0x128, 24); + np = dev->of_node; base = devm_platform_ioremap_resource(pdev, 0); if (WARN_ON(IS_ERR(base))) diff --git a/drivers/clk/imx/clk-imx8mn.c b/drivers/clk/imx/clk-imx8mn.c index 3c21db942d5b..324c5fd0aa04 100644 --- a/drivers/clk/imx/clk-imx8mn.c +++ b/drivers/clk/imx/clk-imx8mn.c @@ -281,6 +281,11 @@ static const char * const imx8mn_clko2_sels[] = {"osc_24m", "sys_pll2_200m", "sy "sys_pll2_166m", "sys_pll3_out", "audio_pll1_out", "video_pll1_out", "osc_32k", }; +static const char * const clkout_sels[] = {"audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "dummy", "dummy", "gpu_pll_out", "dummy", + "arm_pll_out", "sys_pll1", "sys_pll2", "sys_pll3", + "dummy", "dummy", "osc_24m", "dummy", "osc_32k"}; + static struct clk_hw_onecell_data *clk_hw_data; static struct clk_hw **hws; @@ -405,6 +410,13 @@ static int imx8mn_clocks_probe(struct platform_device *pdev) hws[IMX8MN_SYS_PLL2_500M] = imx_clk_hw_fixed_factor("sys_pll2_500m", "sys_pll2_500m_cg", 1, 2); hws[IMX8MN_SYS_PLL2_1000M] = imx_clk_hw_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1); + hws[IMX8MN_CLK_CLKOUT1_SEL] = imx_clk_hw_mux("clkout1_sel", base + 0x128, 4, 4, clkout_sels, ARRAY_SIZE(clkout_sels)); + hws[IMX8MN_CLK_CLKOUT1_DIV] = imx_clk_hw_divider("clkout1_div", "clkout1_sel", base + 0x128, 0, 4); + hws[IMX8MN_CLK_CLKOUT1] = imx_clk_hw_gate("clkout1", "clkout1_div", base + 0x128, 8); + hws[IMX8MN_CLK_CLKOUT2_SEL] = imx_clk_hw_mux("clkout2_sel", base + 0x128, 20, 4, clkout_sels, ARRAY_SIZE(clkout_sels)); + hws[IMX8MN_CLK_CLKOUT2_DIV] = imx_clk_hw_divider("clkout2_div", "clkout2_sel", base + 0x128, 16, 4); + hws[IMX8MN_CLK_CLKOUT2] = imx_clk_hw_gate("clkout2", "clkout2_div", base + 0x128, 24); + np = dev->of_node; base = devm_platform_ioremap_resource(pdev, 0); if (WARN_ON(IS_ERR(base))) { diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c index 779ea69e639c..4dd4ae9d022b 100644 --- a/drivers/clk/imx/clk-imx8mq.c +++ b/drivers/clk/imx/clk-imx8mq.c @@ -270,6 +270,14 @@ static const char * const imx8mq_clko1_sels[] = {"osc_25m", "sys1_pll_800m", "os static const char * const imx8mq_clko2_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_400m", "sys2_pll_166m", "sys3_pll_out", "audio_pll1_out", "video_pll1_out", "ckil", }; +static const char * const pllout_monitor_sels[] = {"osc_25m", "osc_27m", "dummy", "dummy", "ckil", + "audio_pll1_out_monitor", "audio_pll2_out_monitor", + "video_pll1_out_monitor", "gpu_pll_out_monitor", + "vpu_pll_out_monitor", "arm_pll_out_monitor", + "sys_pll1_out_monitor", "sys_pll2_out_monitor", + "sys_pll3_out_monitor", "dram_pll_out_monitor", + "video_pll2_out_monitor", }; + static struct clk_hw_onecell_data *clk_hw_data; static struct clk_hw **hws; @@ -399,6 +407,20 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) hws[IMX8MQ_SYS2_PLL_500M] = imx_clk_hw_fixed_factor("sys2_pll_500m", "sys2_pll_500m_cg", 1, 2); hws[IMX8MQ_SYS2_PLL_1000M] = imx_clk_hw_fixed_factor("sys2_pll_1000m", "sys2_pll_1000m_cg", 1, 1); + hws[IMX8MQ_CLK_MON_AUDIO_PLL1_DIV] = imx_clk_hw_divider("audio_pll1_out_monitor", "audio_pll1_bypass", base + 0x78, 0, 3); + hws[IMX8MQ_CLK_MON_AUDIO_PLL2_DIV] = imx_clk_hw_divider("audio_pll2_out_monitor", "audio_pll2_bypass", base + 0x78, 4, 3); + hws[IMX8MQ_CLK_MON_VIDEO_PLL1_DIV] = imx_clk_hw_divider("video_pll1_out_monitor", "video_pll1_bypass", base + 0x78, 8, 3); + hws[IMX8MQ_CLK_MON_GPU_PLL_DIV] = imx_clk_hw_divider("gpu_pll_out_monitor", "gpu_pll_bypass", base + 0x78, 12, 3); + hws[IMX8MQ_CLK_MON_VPU_PLL_DIV] = imx_clk_hw_divider("vpu_pll_out_monitor", "vpu_pll_bypass", base + 0x78, 16, 3); + hws[IMX8MQ_CLK_MON_ARM_PLL_DIV] = imx_clk_hw_divider("arm_pll_out_monitor", "arm_pll_bypass", base + 0x78, 20, 3); + hws[IMX8MQ_CLK_MON_SYS_PLL1_DIV] = imx_clk_hw_divider("sys_pll1_out_monitor", "sys1_pll_out", base + 0x7c, 0, 3); + hws[IMX8MQ_CLK_MON_SYS_PLL2_DIV] = imx_clk_hw_divider("sys_pll2_out_monitor", "sys2_pll_out", base + 0x7c, 4, 3); + hws[IMX8MQ_CLK_MON_SYS_PLL3_DIV] = imx_clk_hw_divider("sys_pll3_out_monitor", "sys3_pll_out", base + 0x7c, 8, 3); + hws[IMX8MQ_CLK_MON_DRAM_PLL_DIV] = imx_clk_hw_divider("dram_pll_out_monitor", "dram_pll_out", base + 0x7c, 12, 3); + hws[IMX8MQ_CLK_MON_VIDEO_PLL2_DIV] = imx_clk_hw_divider("video_pll2_out_monitor", "video2_pll_out", base + 0x7c, 16, 3); + hws[IMX8MQ_CLK_MON_SEL] = imx_clk_hw_mux("pllout_monitor_sel", base + 0x74, 0, 4, pllout_monitor_sels, ARRAY_SIZE(pllout_monitor_sels)); + hws[IMX8MQ_CLK_MON_CLK2_OUT] = imx_clk_hw_gate("pllout_monitor_clk2", "pllout_monitor_sel", base + 0x74, 4); + np = dev->of_node; base = devm_platform_ioremap_resource(pdev, 0); if (WARN_ON(IS_ERR(base))) diff --git a/drivers/clk/imx/clk-imx8qxp.c b/drivers/clk/imx/clk-imx8qxp.c index 5b3d4ede7c7c..fbf1170c09ed 100644 --- a/drivers/clk/imx/clk-imx8qxp.c +++ b/drivers/clk/imx/clk-imx8qxp.c @@ -17,6 +17,14 @@ #include #include +static const char *dc0_sels[] = { + "clk_dummy", + "clk_dummy", + "dc0_pll0_clk", + "dc0_pll1_clk", + "dc0_bypass0_clk", +}; + static int imx8qxp_clk_probe(struct platform_device *pdev) { struct device_node *ccm_node = pdev->dev.of_node; @@ -115,12 +123,26 @@ static int imx8qxp_clk_probe(struct platform_device *pdev) clks[IMX_CONN_USB2_LPM_CLK] = imx_clk_scu("usb3_lpm_div", IMX_SC_R_USB_2, IMX_SC_PM_CLK_MISC, clk_cells); /* Display controller SS */ - clks[IMX_DC0_DISP0_CLK] = imx_clk_scu("dc0_disp0_clk", IMX_SC_R_DC_0, IMX_SC_PM_CLK_MISC0, clk_cells); - clks[IMX_DC0_DISP1_CLK] = imx_clk_scu("dc0_disp1_clk", IMX_SC_R_DC_0, IMX_SC_PM_CLK_MISC1, clk_cells); + clks[IMX_DC0_DISP0_CLK] = imx_clk_scu2("dc0_disp0_clk", dc0_sels, ARRAY_SIZE(dc0_sels), IMX_SC_R_DC_0, IMX_SC_PM_CLK_MISC0, clk_cells); + clks[IMX_DC0_DISP1_CLK] = imx_clk_scu2("dc0_disp1_clk", dc0_sels, ARRAY_SIZE(dc0_sels), IMX_SC_R_DC_0, IMX_SC_PM_CLK_MISC1, clk_cells); + clks[IMX_DC0_PLL0_CLK] = imx_clk_scu("dc0_pll0_clk", IMX_SC_R_DC_0_PLL_0, IMX_SC_PM_CLK_PLL, clk_cells); + clks[IMX_DC0_PLL1_CLK] = imx_clk_scu("dc0_pll1_clk", IMX_SC_R_DC_0_PLL_1, IMX_SC_PM_CLK_PLL, clk_cells); + clks[IMX_DC0_BYPASS0_CLK] = imx_clk_scu("dc0_bypass0_clk", IMX_SC_R_DC_0_VIDEO0, IMX_SC_PM_CLK_BYPASS, clk_cells); + clks[IMX_DC0_BYPASS1_CLK] = imx_clk_scu("dc0_bypass1_clk", IMX_SC_R_DC_0_VIDEO1, IMX_SC_PM_CLK_BYPASS, clk_cells); /* MIPI-LVDS SS */ + clks[IMX_MIPI0_LVDS_PIXEL_CLK] = imx_clk_scu("mipi0_lvds_pixel_clk", IMX_SC_R_LVDS_0, IMX_SC_PM_CLK_MISC2, clk_cells); + clks[IMX_MIPI0_LVDS_BYPASS_CLK] = imx_clk_scu("mipi0_lvds_bypass_clk", IMX_SC_R_LVDS_0, IMX_SC_PM_CLK_BYPASS, clk_cells); + clks[IMX_MIPI0_LVDS_PHY_CLK] = imx_clk_scu("mipi0_lvds_phy_clk", IMX_SC_R_LVDS_0, IMX_SC_PM_CLK_MISC3, clk_cells); clks[IMX_MIPI0_I2C0_CLK] = imx_clk_scu("mipi0_i2c0_clk", IMX_SC_R_MIPI_0_I2C_0, IMX_SC_PM_CLK_MISC2, clk_cells); clks[IMX_MIPI0_I2C1_CLK] = imx_clk_scu("mipi0_i2c1_clk", IMX_SC_R_MIPI_0_I2C_1, IMX_SC_PM_CLK_MISC2, clk_cells); + clks[IMX_MIPI0_PWM0_CLK] = imx_clk_scu("mipi0_pwm0_clk", IMX_SC_R_MIPI_0_PWM_0, IMX_SC_PM_CLK_PER, clk_cells); + clks[IMX_MIPI1_LVDS_PIXEL_CLK] = imx_clk_scu("mipi1_lvds_pixel_clk", IMX_SC_R_LVDS_1, IMX_SC_PM_CLK_MISC2, clk_cells); + clks[IMX_MIPI1_LVDS_BYPASS_CLK] = imx_clk_scu("mipi1_lvds_bypass_clk", IMX_SC_R_LVDS_1, IMX_SC_PM_CLK_BYPASS, clk_cells); + clks[IMX_MIPI1_LVDS_PHY_CLK] = imx_clk_scu("mipi1_lvds_phy_clk", IMX_SC_R_LVDS_1, IMX_SC_PM_CLK_MISC3, clk_cells); + clks[IMX_MIPI1_I2C0_CLK] = imx_clk_scu("mipi1_i2c0_clk", IMX_SC_R_MIPI_1_I2C_0, IMX_SC_PM_CLK_MISC2, clk_cells); + clks[IMX_MIPI1_I2C1_CLK] = imx_clk_scu("mipi1_i2c1_clk", IMX_SC_R_MIPI_1_I2C_1, IMX_SC_PM_CLK_MISC2, clk_cells); + clks[IMX_MIPI1_PWM0_CLK] = imx_clk_scu("mipi1_pwm0_clk", IMX_SC_R_MIPI_1_PWM_0, IMX_SC_PM_CLK_PER, clk_cells); /* MIPI CSI SS */ clks[IMX_CSI0_CORE_CLK] = imx_clk_scu("mipi_csi0_core_clk", IMX_SC_R_CSI_0, IMX_SC_PM_CLK_PER, clk_cells); diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index ce8475098b31..886e2d9fced5 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -426,66 +426,77 @@ config COMMON_CLK_MT8183 config COMMON_CLK_MT8183_AUDIOSYS bool "Clock driver for MediaTek MT8183 audiosys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 audiosys clocks. config COMMON_CLK_MT8183_CAMSYS bool "Clock driver for MediaTek MT8183 camsys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 camsys clocks. config COMMON_CLK_MT8183_IMGSYS bool "Clock driver for MediaTek MT8183 imgsys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 imgsys clocks. config COMMON_CLK_MT8183_IPU_CORE0 bool "Clock driver for MediaTek MT8183 ipu_core0" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 ipu_core0 clocks. config COMMON_CLK_MT8183_IPU_CORE1 bool "Clock driver for MediaTek MT8183 ipu_core1" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 ipu_core1 clocks. config COMMON_CLK_MT8183_IPU_ADL bool "Clock driver for MediaTek MT8183 ipu_adl" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 ipu_adl clocks. config COMMON_CLK_MT8183_IPU_CONN bool "Clock driver for MediaTek MT8183 ipu_conn" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 ipu_conn clocks. config COMMON_CLK_MT8183_MFGCFG bool "Clock driver for MediaTek MT8183 mfgcfg" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 mfgcfg clocks. config COMMON_CLK_MT8183_MMSYS bool "Clock driver for MediaTek MT8183 mmsys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 mmsys clocks. config COMMON_CLK_MT8183_VDECSYS bool "Clock driver for MediaTek MT8183 vdecsys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 vdecsys clocks. config COMMON_CLK_MT8183_VENCSYS bool "Clock driver for MediaTek MT8183 vencsys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 vencsys clocks. diff --git a/drivers/clk/mediatek/clk-mux.c b/drivers/clk/mediatek/clk-mux.c index dcc1352bf13c..b0c61709bacc 100644 --- a/drivers/clk/mediatek/clk-mux.c +++ b/drivers/clk/mediatek/clk-mux.c @@ -17,29 +17,36 @@ static inline struct mtk_clk_mux *to_mtk_clk_mux(struct clk_hw *hw) return container_of(hw, struct mtk_clk_mux, hw); } -static int mtk_clk_mux_enable(struct clk_hw *hw) -{ - struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); - u32 mask = BIT(mux->data->gate_shift); - - return regmap_update_bits(mux->regmap, mux->data->mux_ofs, - mask, ~mask); -} - -static void mtk_clk_mux_disable(struct clk_hw *hw) -{ - struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); - u32 mask = BIT(mux->data->gate_shift); - - regmap_update_bits(mux->regmap, mux->data->mux_ofs, mask, mask); -} - static int mtk_clk_mux_enable_setclr(struct clk_hw *hw) { struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); + unsigned long flags = 0; - return regmap_write(mux->regmap, mux->data->clr_ofs, - BIT(mux->data->gate_shift)); + if (mux->lock) + spin_lock_irqsave(mux->lock, flags); + else + __acquire(mux->lock); + + regmap_write(mux->regmap, mux->data->clr_ofs, + BIT(mux->data->gate_shift)); + + /* + * If the parent has been changed when the clock was disabled, it will + * not be effective yet. Set the update bit to ensure the mux gets + * updated. + */ + if (mux->reparent && mux->data->upd_shift >= 0) { + regmap_write(mux->regmap, mux->data->upd_ofs, + BIT(mux->data->upd_shift)); + mux->reparent = false; + } + + if (mux->lock) + spin_unlock_irqrestore(mux->lock, flags); + else + __release(mux->lock); + + return 0; } static void mtk_clk_mux_disable_setclr(struct clk_hw *hw) @@ -72,28 +79,6 @@ static u8 mtk_clk_mux_get_parent(struct clk_hw *hw) return val; } -static int mtk_clk_mux_set_parent_lock(struct clk_hw *hw, u8 index) -{ - struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); - u32 mask = GENMASK(mux->data->mux_width - 1, 0); - unsigned long flags = 0; - - if (mux->lock) - spin_lock_irqsave(mux->lock, flags); - else - __acquire(mux->lock); - - regmap_update_bits(mux->regmap, mux->data->mux_ofs, mask, - index << mux->data->mux_shift); - - if (mux->lock) - spin_unlock_irqrestore(mux->lock, flags); - else - __release(mux->lock); - - return 0; -} - static int mtk_clk_mux_set_parent_setclr_lock(struct clk_hw *hw, u8 index) { struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); @@ -116,9 +101,11 @@ static int mtk_clk_mux_set_parent_setclr_lock(struct clk_hw *hw, u8 index) regmap_write(mux->regmap, mux->data->set_ofs, index << mux->data->mux_shift); - if (mux->data->upd_shift >= 0) + if (mux->data->upd_shift >= 0) { regmap_write(mux->regmap, mux->data->upd_ofs, BIT(mux->data->upd_shift)); + mux->reparent = true; + } } if (mux->lock) @@ -129,25 +116,7 @@ static int mtk_clk_mux_set_parent_setclr_lock(struct clk_hw *hw, u8 index) return 0; } -const struct clk_ops mtk_mux_ops = { - .get_parent = mtk_clk_mux_get_parent, - .set_parent = mtk_clk_mux_set_parent_lock, -}; - -const struct clk_ops mtk_mux_clr_set_upd_ops = { - .get_parent = mtk_clk_mux_get_parent, - .set_parent = mtk_clk_mux_set_parent_setclr_lock, -}; - -const struct clk_ops mtk_mux_gate_ops = { - .enable = mtk_clk_mux_enable, - .disable = mtk_clk_mux_disable, - .is_enabled = mtk_clk_mux_is_enabled, - .get_parent = mtk_clk_mux_get_parent, - .set_parent = mtk_clk_mux_set_parent_lock, -}; - -const struct clk_ops mtk_mux_gate_clr_set_upd_ops = { +static const struct clk_ops mtk_mux_ops = { .enable = mtk_clk_mux_enable_setclr, .disable = mtk_clk_mux_disable_setclr, .is_enabled = mtk_clk_mux_is_enabled, @@ -171,7 +140,7 @@ static struct clk *mtk_clk_register_mux(const struct mtk_mux *mux, init.flags = mux->flags | CLK_SET_RATE_PARENT; init.parent_names = mux->parent_names; init.num_parents = mux->num_parents; - init.ops = mux->ops; + init.ops = &mtk_mux_ops; clk_mux->regmap = regmap; clk_mux->data = mux; diff --git a/drivers/clk/mediatek/clk-mux.h b/drivers/clk/mediatek/clk-mux.h index 8e2f927dd2ff..f1946161ade1 100644 --- a/drivers/clk/mediatek/clk-mux.h +++ b/drivers/clk/mediatek/clk-mux.h @@ -14,6 +14,7 @@ struct mtk_clk_mux { struct regmap *regmap; const struct mtk_mux *data; spinlock_t *lock; + bool reparent; }; struct mtk_mux { @@ -32,19 +33,12 @@ struct mtk_mux { u8 gate_shift; s8 upd_shift; - const struct clk_ops *ops; - signed char num_parents; }; -extern const struct clk_ops mtk_mux_ops; -extern const struct clk_ops mtk_mux_clr_set_upd_ops; -extern const struct clk_ops mtk_mux_gate_ops; -extern const struct clk_ops mtk_mux_gate_clr_set_upd_ops; - #define GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents, _mux_ofs, \ _mux_set_ofs, _mux_clr_ofs, _shift, _width, \ - _gate, _upd_ofs, _upd, _flags, _ops) { \ + _gate, _upd_ofs, _upd, _flags) { \ .id = _id, \ .name = _name, \ .mux_ofs = _mux_ofs, \ @@ -58,7 +52,6 @@ extern const struct clk_ops mtk_mux_gate_clr_set_upd_ops; .parent_names = _parents, \ .num_parents = ARRAY_SIZE(_parents), \ .flags = _flags, \ - .ops = &_ops, \ } #define MUX_GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents, _mux_ofs, \ @@ -66,8 +59,7 @@ extern const struct clk_ops mtk_mux_gate_clr_set_upd_ops; _gate, _upd_ofs, _upd, _flags) \ GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents, _mux_ofs, \ _mux_set_ofs, _mux_clr_ofs, _shift, _width, \ - _gate, _upd_ofs, _upd, _flags, \ - mtk_mux_gate_clr_set_upd_ops) + _gate, _upd_ofs, _upd, _flags) \ #define MUX_GATE_CLR_SET_UPD(_id, _name, _parents, _mux_ofs, \ _mux_set_ofs, _mux_clr_ofs, _shift, _width, \ diff --git a/drivers/clk/meson/axg.c b/drivers/clk/meson/axg.c index 0e44695b8772..2ad3801398dc 100644 --- a/drivers/clk/meson/axg.c +++ b/drivers/clk/meson/axg.c @@ -1879,7 +1879,6 @@ static MESON_GATE(axg_mmc_pclk, HHI_GCLK_MPEG2, 11); static MESON_GATE(axg_vpu_intr, HHI_GCLK_MPEG2, 25); static MESON_GATE(axg_sec_ahb_ahb3_bridge, HHI_GCLK_MPEG2, 26); static MESON_GATE(axg_gic, HHI_GCLK_MPEG2, 30); -static MESON_GATE(axg_mipi_enable, HHI_MIPI_CNTL0, 29); /* Always On (AO) domain gates */ @@ -1974,7 +1973,6 @@ static struct clk_hw_onecell_data axg_hw_onecell_data = { [CLKID_PCIE_REF] = &axg_pcie_ref.hw, [CLKID_PCIE_CML_EN0] = &axg_pcie_cml_en0.hw, [CLKID_PCIE_CML_EN1] = &axg_pcie_cml_en1.hw, - [CLKID_MIPI_ENABLE] = &axg_mipi_enable.hw, [CLKID_GEN_CLK_SEL] = &axg_gen_clk_sel.hw, [CLKID_GEN_CLK_DIV] = &axg_gen_clk_div.hw, [CLKID_GEN_CLK] = &axg_gen_clk.hw, @@ -2115,7 +2113,6 @@ static struct clk_regmap *const axg_clk_regmaps[] = { &axg_pcie_ref, &axg_pcie_cml_en0, &axg_pcie_cml_en1, - &axg_mipi_enable, &axg_gen_clk_sel, &axg_gen_clk_div, &axg_gen_clk, diff --git a/drivers/clk/meson/axg.h b/drivers/clk/meson/axg.h index 481b307ea3cb..23ea87964af2 100644 --- a/drivers/clk/meson/axg.h +++ b/drivers/clk/meson/axg.h @@ -16,7 +16,6 @@ * Register offsets from the data sheet must be multiplied by 4 before * adding them to the base address to get the right value. */ -#define HHI_MIPI_CNTL0 0x00 #define HHI_GP0_PLL_CNTL 0x40 #define HHI_GP0_PLL_CNTL2 0x44 #define HHI_GP0_PLL_CNTL3 0x48 diff --git a/drivers/clk/meson/clk-pll.c b/drivers/clk/meson/clk-pll.c index b17a13e9337c..49f27fe53213 100644 --- a/drivers/clk/meson/clk-pll.c +++ b/drivers/clk/meson/clk-pll.c @@ -365,13 +365,14 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, { struct clk_regmap *clk = to_clk_regmap(hw); struct meson_clk_pll_data *pll = meson_clk_pll_data(clk); - unsigned int enabled, m, n, frac = 0, ret; + unsigned int enabled, m, n, frac = 0; unsigned long old_rate; + int ret; if (parent_rate == 0 || rate == 0) return -EINVAL; - old_rate = rate; + old_rate = clk_hw_get_rate(hw); ret = meson_clk_get_pll_settings(rate, parent_rate, &m, &n, pll); if (ret) @@ -393,7 +394,8 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, if (!enabled) return 0; - if (meson_clk_pll_enable(hw)) { + ret = meson_clk_pll_enable(hw); + if (ret) { pr_warn("%s: pll did not lock, trying to restore old rate %lu\n", __func__, old_rate); /* @@ -405,7 +407,7 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, meson_clk_pll_set_rate(hw, old_rate, parent_rate); } - return 0; + return ret; } /* diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c index 862f0756b50f..a844d35b553a 100644 --- a/drivers/clk/meson/meson8b.c +++ b/drivers/clk/meson/meson8b.c @@ -52,15 +52,6 @@ static const struct pll_params_table sys_pll_params_table[] = { { /* sentinel */ }, }; -static struct clk_fixed_rate meson8b_xtal = { - .fixed_rate = 24000000, - .hw.init = &(struct clk_init_data){ - .name = "xtal", - .num_parents = 0, - .ops = &clk_fixed_rate_ops, - }, -}; - static struct clk_regmap meson8b_fixed_pll_dco = { .data = &(struct meson_clk_pll_data){ .en = { @@ -2715,7 +2706,6 @@ static MESON_GATE(meson8b_ao_iface, HHI_GCLK_AO, 3); static struct clk_hw_onecell_data meson8_hw_onecell_data = { .hws = { - [CLKID_XTAL] = &meson8b_xtal.hw, [CLKID_PLL_FIXED] = &meson8b_fixed_pll.hw, [CLKID_PLL_VID] = &meson8b_vid_pll.hw, [CLKID_PLL_SYS] = &meson8b_sys_pll.hw, @@ -2922,7 +2912,6 @@ static struct clk_hw_onecell_data meson8_hw_onecell_data = { static struct clk_hw_onecell_data meson8b_hw_onecell_data = { .hws = { - [CLKID_XTAL] = &meson8b_xtal.hw, [CLKID_PLL_FIXED] = &meson8b_fixed_pll.hw, [CLKID_PLL_VID] = &meson8b_vid_pll.hw, [CLKID_PLL_SYS] = &meson8b_sys_pll.hw, @@ -3140,7 +3129,6 @@ static struct clk_hw_onecell_data meson8b_hw_onecell_data = { static struct clk_hw_onecell_data meson8m2_hw_onecell_data = { .hws = { - [CLKID_XTAL] = &meson8b_xtal.hw, [CLKID_PLL_FIXED] = &meson8b_fixed_pll.hw, [CLKID_PLL_VID] = &meson8b_vid_pll.hw, [CLKID_PLL_SYS] = &meson8b_sys_pll.hw, @@ -3725,36 +3713,19 @@ static struct meson8b_nb_data meson8b_cpu_nb_data = { .nb.notifier_call = meson8b_cpu_clk_notifier_cb, }; -static const struct regmap_config clkc_regmap_config = { - .reg_bits = 32, - .val_bits = 32, - .reg_stride = 4, -}; - static void __init meson8b_clkc_init_common(struct device_node *np, struct clk_hw_onecell_data *clk_hw_onecell_data) { struct meson8b_clk_reset *rstc; const char *notifier_clk_name; struct clk *notifier_clk; - void __iomem *clk_base; struct regmap *map; int i, ret; map = syscon_node_to_regmap(of_get_parent(np)); if (IS_ERR(map)) { - pr_info("failed to get HHI regmap - Trying obsolete regs\n"); - - /* Generic clocks, PLLs and some of the reset-bits */ - clk_base = of_iomap(np, 1); - if (!clk_base) { - pr_err("%s: Unable to map clk base\n", __func__); - return; - } - - map = regmap_init_mmio(NULL, clk_base, &clkc_regmap_config); - if (IS_ERR(map)) - return; + pr_err("failed to get HHI regmap - Trying obsolete regs\n"); + return; } rstc = kzalloc(sizeof(*rstc), GFP_KERNEL); @@ -3778,16 +3749,10 @@ static void __init meson8b_clkc_init_common(struct device_node *np, meson8b_clk_regmaps[i]->map = map; /* - * always skip CLKID_UNUSED and also skip XTAL if the .dtb provides the - * XTAL clock as input. + * register all clks and start with the first used ID (which is + * CLKID_PLL_FIXED) */ - if (!IS_ERR(of_clk_get_by_name(np, "xtal"))) - i = CLKID_PLL_FIXED; - else - i = CLKID_XTAL; - - /* register all clks */ - for (; i < CLK_NR_CLKS; i++) { + for (i = CLKID_PLL_FIXED; i < CLK_NR_CLKS; i++) { /* array might be sparse */ if (!clk_hw_onecell_data->hws[i]) continue; diff --git a/drivers/clk/mstar/Kconfig b/drivers/clk/mstar/Kconfig new file mode 100644 index 000000000000..de37e1bce2d2 --- /dev/null +++ b/drivers/clk/mstar/Kconfig @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +config MSTAR_MSC313_MPLL + bool "MStar MPLL driver" + depends on ARCH_MSTARV7 || COMPILE_TEST + default ARCH_MSTARV7 + select REGMAP_MMIO + help + Support for the MPLL PLL and dividers block present on + MStar/Sigmastar SoCs. diff --git a/drivers/clk/mstar/Makefile b/drivers/clk/mstar/Makefile new file mode 100644 index 000000000000..f8dcd25ede1d --- /dev/null +++ b/drivers/clk/mstar/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for mstar specific clk +# + +obj-$(CONFIG_MSTAR_MSC313_MPLL) += clk-msc313-mpll.o diff --git a/drivers/clk/mstar/clk-msc313-mpll.c b/drivers/clk/mstar/clk-msc313-mpll.c new file mode 100644 index 000000000000..61beb4e87525 --- /dev/null +++ b/drivers/clk/mstar/clk-msc313-mpll.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MStar MSC313 MPLL driver + * + * Copyright (C) 2020 Daniel Palmer + */ + +#include +#include +#include +#include + +#define REG_CONFIG1 0x8 +#define REG_CONFIG2 0xc + +static const struct regmap_config msc313_mpll_regmap_config = { + .reg_bits = 16, + .val_bits = 16, + .reg_stride = 4, +}; + +static const struct reg_field config1_loop_div_first = REG_FIELD(REG_CONFIG1, 8, 9); +static const struct reg_field config1_input_div_first = REG_FIELD(REG_CONFIG1, 4, 5); +static const struct reg_field config2_output_div_first = REG_FIELD(REG_CONFIG2, 12, 13); +static const struct reg_field config2_loop_div_second = REG_FIELD(REG_CONFIG2, 0, 7); + +static const unsigned int output_dividers[] = { + 2, 3, 4, 5, 6, 7, 10 +}; + +#define NUMOUTPUTS (ARRAY_SIZE(output_dividers) + 1) + +struct msc313_mpll { + struct clk_hw clk_hw; + struct regmap_field *input_div; + struct regmap_field *loop_div_first; + struct regmap_field *loop_div_second; + struct regmap_field *output_div; + struct clk_hw_onecell_data *clk_data; +}; + +#define to_mpll(_hw) container_of(_hw, struct msc313_mpll, clk_hw) + +static unsigned long msc313_mpll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct msc313_mpll *mpll = to_mpll(hw); + unsigned int input_div, output_div, loop_first, loop_second; + unsigned long output_rate; + + regmap_field_read(mpll->input_div, &input_div); + regmap_field_read(mpll->output_div, &output_div); + regmap_field_read(mpll->loop_div_first, &loop_first); + regmap_field_read(mpll->loop_div_second, &loop_second); + + output_rate = parent_rate / (1 << input_div); + output_rate *= (1 << loop_first) * max(loop_second, 1U); + output_rate /= max(output_div, 1U); + + return output_rate; +} + +static const struct clk_ops msc313_mpll_ops = { + .recalc_rate = msc313_mpll_recalc_rate, +}; + +static const struct clk_parent_data mpll_parent = { + .index = 0, +}; + +static int msc313_mpll_probe(struct platform_device *pdev) +{ + void __iomem *base; + struct msc313_mpll *mpll; + struct clk_init_data clk_init = { }; + struct device *dev = &pdev->dev; + struct regmap *regmap; + char *outputname; + struct clk_hw *divhw; + int ret, i; + + mpll = devm_kzalloc(dev, sizeof(*mpll), GFP_KERNEL); + if (!mpll) + return -ENOMEM; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + regmap = devm_regmap_init_mmio(dev, base, &msc313_mpll_regmap_config); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + mpll->input_div = devm_regmap_field_alloc(dev, regmap, config1_input_div_first); + if (IS_ERR(mpll->input_div)) + return PTR_ERR(mpll->input_div); + mpll->output_div = devm_regmap_field_alloc(dev, regmap, config2_output_div_first); + if (IS_ERR(mpll->output_div)) + return PTR_ERR(mpll->output_div); + mpll->loop_div_first = devm_regmap_field_alloc(dev, regmap, config1_loop_div_first); + if (IS_ERR(mpll->loop_div_first)) + return PTR_ERR(mpll->loop_div_first); + mpll->loop_div_second = devm_regmap_field_alloc(dev, regmap, config2_loop_div_second); + if (IS_ERR(mpll->loop_div_second)) + return PTR_ERR(mpll->loop_div_second); + + mpll->clk_data = devm_kzalloc(dev, struct_size(mpll->clk_data, hws, + ARRAY_SIZE(output_dividers)), GFP_KERNEL); + if (!mpll->clk_data) + return -ENOMEM; + + clk_init.name = dev_name(dev); + clk_init.ops = &msc313_mpll_ops; + clk_init.parent_data = &mpll_parent; + clk_init.num_parents = 1; + mpll->clk_hw.init = &clk_init; + + ret = devm_clk_hw_register(dev, &mpll->clk_hw); + if (ret) + return ret; + + mpll->clk_data->num = NUMOUTPUTS; + mpll->clk_data->hws[0] = &mpll->clk_hw; + + for (i = 0; i < ARRAY_SIZE(output_dividers); i++) { + outputname = devm_kasprintf(dev, GFP_KERNEL, "%s_div_%u", + clk_init.name, output_dividers[i]); + if (!outputname) + return -ENOMEM; + divhw = devm_clk_hw_register_fixed_factor(dev, outputname, + clk_init.name, 0, 1, output_dividers[i]); + if (IS_ERR(divhw)) + return PTR_ERR(divhw); + mpll->clk_data->hws[i + 1] = divhw; + } + + platform_set_drvdata(pdev, mpll); + + return devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_onecell_get, + mpll->clk_data); +} + +static const struct of_device_id msc313_mpll_of_match[] = { + { .compatible = "mstar,msc313-mpll", }, + {} +}; + +static struct platform_driver msc313_mpll_driver = { + .driver = { + .name = "mstar-msc313-mpll", + .of_match_table = msc313_mpll_of_match, + }, + .probe = msc313_mpll_probe, +}; +builtin_platform_driver(msc313_mpll_driver); diff --git a/drivers/clk/mvebu/ap-cpu-clk.c b/drivers/clk/mvebu/ap-cpu-clk.c index b4259b60dcfd..08ba59ec3fb1 100644 --- a/drivers/clk/mvebu/ap-cpu-clk.c +++ b/drivers/clk/mvebu/ap-cpu-clk.c @@ -30,7 +30,7 @@ #define APN806_MAX_DIVIDER 32 -/** +/* * struct cpu_dfs_regs: CPU DFS register mapping * @divider_reg: full integer ratio from PLL frequency to CPU clock frequency * @force_reg: request to force new ratio regardless of relation to other clocks diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index d32bb12cd8d0..45646b867cdb 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -28,6 +28,14 @@ config QCOM_A53PLL Say Y if you want to support higher CPU frequencies on MSM8916 devices. +config QCOM_A7PLL + tristate "SDX55 A7 PLL" + help + Support for the A7 PLL on SDX55 devices. It provides the CPU with + frequencies above 1GHz. + Say Y if you want to support higher CPU frequencies on SDX55 + devices. + config QCOM_CLK_APCS_MSM8916 tristate "MSM8916 APCS Clock Controller" depends on QCOM_APCS_IPC || COMPILE_TEST @@ -46,6 +54,15 @@ config QCOM_CLK_APCC_MSM8996 Say Y if you want to support CPU clock scaling using CPUfreq drivers for dynamic power management. +config QCOM_CLK_APCS_SDX55 + tristate "SDX55 APCS Clock Controller" + depends on QCOM_APCS_IPC || COMPILE_TEST + help + Support for the APCS Clock Controller on SDX55 platform. The + APCS is managing the mux and divider which feeds the CPUs. + Say Y if you want to support CPU frequency scaling on devices + such as SDX55. + config QCOM_CLK_RPM tristate "RPM based Clock Controller" depends on MFD_QCOM_RPM @@ -317,6 +334,24 @@ config SC_GCC_7180 Say Y if you want to use peripheral devices such as UART, SPI, I2C, USB, UFS, SDCC, etc. +config SC_GCC_7280 + tristate "SC7280 Global Clock Controller" + select QCOM_GDSC + depends on COMMON_CLK_QCOM + help + Support for the global clock controller on SC7280 devices. + Say Y if you want to use peripheral devices such as UART, SPI, + I2C, USB, UFS, SDCC, PCIe etc. + +config SC_GCC_8180X + tristate "SC8180X Global Clock Controller" + select QCOM_GDSC + depends on COMMON_CLK_QCOM + help + Support for the global clock controller on SC8180X devices. + Say Y if you want to use peripheral devices such as UART, SPI, + I2C, USB, UFS, SDCC, etc. + config SC_LPASS_CORECC_7180 tristate "SC7180 LPASS Core Clock Controller" select SC_GCC_7180 @@ -366,6 +401,24 @@ config SDM_GCC_660 Say Y if you want to use peripheral devices such as UART, SPI, i2C, USB, UFS, SDDC, PCIe, etc. +config SDM_MMCC_660 + tristate "SDM660 Multimedia Clock Controller" + select SDM_GCC_660 + select QCOM_GDSC + help + Support for the multimedia clock controller on SDM660 devices. + Say Y if you want to support multimedia devices such as display, + graphics, video encode/decode, camera, etc. + +config SDM_GPUCC_660 + tristate "SDM660 Graphics Clock Controller" + select SDM_GCC_660 + select QCOM_GDSC + help + Support for the graphics clock controller on SDM630/636/660 devices. + Say Y if you want to support graphics controller devices and + functionality such as 3D graphics + config QCS_TURING_404 tristate "QCS404 Turing Clock Controller" help @@ -454,6 +507,14 @@ config SM_GCC_8250 Say Y if you want to use peripheral devices such as UART, SPI, I2C, USB, SD/UFS, PCIe etc. +config SM_GCC_8350 + tristate "SM8350 Global Clock Controller" + select QCOM_GDSC + help + Support for the global clock controller on SM8350 devices. + Say Y if you want to use peripheral devices such as UART, + SPI, I2C, USB, SD/UFS, PCIe etc. + config SM_GPUCC_8150 tristate "SM8150 Graphics Clock Controller" select SM_GCC_8150 diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile index 9e5e0e3cb7b4..c8291312e723 100644 --- a/drivers/clk/qcom/Makefile +++ b/drivers/clk/qcom/Makefile @@ -44,8 +44,10 @@ obj-$(CONFIG_MSM_MMCC_8974) += mmcc-msm8974.o obj-$(CONFIG_MSM_MMCC_8996) += mmcc-msm8996.o obj-$(CONFIG_MSM_MMCC_8998) += mmcc-msm8998.o obj-$(CONFIG_QCOM_A53PLL) += a53-pll.o +obj-$(CONFIG_QCOM_A7PLL) += a7-pll.o obj-$(CONFIG_QCOM_CLK_APCS_MSM8916) += apcs-msm8916.o obj-$(CONFIG_QCOM_CLK_APCC_MSM8996) += clk-cpu-8996.o +obj-$(CONFIG_QCOM_CLK_APCS_SDX55) += apcs-sdx55.o obj-$(CONFIG_QCOM_CLK_RPM) += clk-rpm.o obj-$(CONFIG_QCOM_CLK_RPMH) += clk-rpmh.o obj-$(CONFIG_QCOM_CLK_SMD_RPM) += clk-smd-rpm.o @@ -55,6 +57,8 @@ obj-$(CONFIG_QCS_TURING_404) += turingcc-qcs404.o obj-$(CONFIG_SC_CAMCC_7180) += camcc-sc7180.o obj-$(CONFIG_SC_DISPCC_7180) += dispcc-sc7180.o obj-$(CONFIG_SC_GCC_7180) += gcc-sc7180.o +obj-$(CONFIG_SC_GCC_7280) += gcc-sc7280.o +obj-$(CONFIG_SC_GCC_8180X) += gcc-sc8180x.o obj-$(CONFIG_SC_GPUCC_7180) += gpucc-sc7180.o obj-$(CONFIG_SC_LPASS_CORECC_7180) += lpasscorecc-sc7180.o obj-$(CONFIG_SC_MSS_7180) += mss-sc7180.o @@ -62,6 +66,8 @@ obj-$(CONFIG_SC_VIDEOCC_7180) += videocc-sc7180.o obj-$(CONFIG_SDM_CAMCC_845) += camcc-sdm845.o obj-$(CONFIG_SDM_DISPCC_845) += dispcc-sdm845.o obj-$(CONFIG_SDM_GCC_660) += gcc-sdm660.o +obj-$(CONFIG_SDM_MMCC_660) += mmcc-sdm660.o +obj-$(CONFIG_SDM_GPUCC_660) += gpucc-sdm660.o obj-$(CONFIG_SDM_GCC_845) += gcc-sdm845.o obj-$(CONFIG_SDM_GPUCC_845) += gpucc-sdm845.o obj-$(CONFIG_SDM_LPASSCC_845) += lpasscc-sdm845.o @@ -70,6 +76,7 @@ obj-$(CONFIG_SDX_GCC_55) += gcc-sdx55.o obj-$(CONFIG_SM_DISPCC_8250) += dispcc-sm8250.o obj-$(CONFIG_SM_GCC_8150) += gcc-sm8150.o obj-$(CONFIG_SM_GCC_8250) += gcc-sm8250.o +obj-$(CONFIG_SM_GCC_8350) += gcc-sm8350.o obj-$(CONFIG_SM_GPUCC_8150) += gpucc-sm8150.o obj-$(CONFIG_SM_GPUCC_8250) += gpucc-sm8250.o obj-$(CONFIG_SM_VIDEOCC_8150) += videocc-sm8150.o diff --git a/drivers/clk/qcom/a7-pll.c b/drivers/clk/qcom/a7-pll.c new file mode 100644 index 000000000000..e171d3caf2cf --- /dev/null +++ b/drivers/clk/qcom/a7-pll.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm A7 PLL driver + * + * Copyright (c) 2020, Linaro Limited + * Author: Manivannan Sadhasivam + */ + +#include +#include +#include +#include + +#include "clk-alpha-pll.h" + +#define LUCID_PLL_OFF_L_VAL 0x04 + +static const struct pll_vco lucid_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static struct clk_alpha_pll a7pll = { + .offset = 0x100, + .vco_table = lucid_vco, + .num_vco = ARRAY_SIZE(lucid_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "a7pll", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_ops, + }, + }, +}; + +static const struct alpha_pll_config a7pll_config = { + .l = 0x39, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x2261, + .config_ctl_hi1_val = 0x029A699C, + .user_ctl_val = 0x1, + .user_ctl_hi_val = 0x805, +}; + +static const struct regmap_config a7pll_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x1000, + .fast_io = true, +}; + +static int qcom_a7pll_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct regmap *regmap; + void __iomem *base; + u32 l_val; + int ret; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + regmap = devm_regmap_init_mmio(dev, base, &a7pll_regmap_config); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* Configure PLL only if the l_val is zero */ + regmap_read(regmap, a7pll.offset + LUCID_PLL_OFF_L_VAL, &l_val); + if (!l_val) + clk_lucid_pll_configure(&a7pll, regmap, &a7pll_config); + + ret = devm_clk_register_regmap(dev, &a7pll.clkr); + if (ret) + return ret; + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, + &a7pll.clkr.hw); +} + +static const struct of_device_id qcom_a7pll_match_table[] = { + { .compatible = "qcom,sdx55-a7pll" }, + { } +}; + +static struct platform_driver qcom_a7pll_driver = { + .probe = qcom_a7pll_probe, + .driver = { + .name = "qcom-a7pll", + .of_match_table = qcom_a7pll_match_table, + }, +}; +module_platform_driver(qcom_a7pll_driver); + +MODULE_DESCRIPTION("Qualcomm A7 PLL Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/apcs-sdx55.c b/drivers/clk/qcom/apcs-sdx55.c new file mode 100644 index 000000000000..d0edabebf9c2 --- /dev/null +++ b/drivers/clk/qcom/apcs-sdx55.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm SDX55 APCS clock controller driver + * + * Copyright (c) 2020, Linaro Limited + * Author: Manivannan Sadhasivam + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clk-regmap.h" +#include "clk-regmap-mux-div.h" + +static const u32 apcs_mux_clk_parent_map[] = { 0, 1, 5 }; + +static const struct clk_parent_data pdata[] = { + { .fw_name = "ref" }, + { .fw_name = "aux" }, + { .fw_name = "pll" }, +}; + +/* + * We use the notifier function for switching to a temporary safe configuration + * (mux and divider), while the A7 PLL is reconfigured. + */ +static int a7cc_notifier_cb(struct notifier_block *nb, unsigned long event, + void *data) +{ + int ret = 0; + struct clk_regmap_mux_div *md = container_of(nb, + struct clk_regmap_mux_div, + clk_nb); + if (event == PRE_RATE_CHANGE) + /* set the mux and divider to safe frequency (400mhz) */ + ret = mux_div_set_src_div(md, 1, 2); + + return notifier_from_errno(ret); +} + +static int qcom_apcs_sdx55_clk_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device *parent = dev->parent; + struct device *cpu_dev; + struct clk_regmap_mux_div *a7cc; + struct regmap *regmap; + struct clk_init_data init = { }; + int ret; + + regmap = dev_get_regmap(parent, NULL); + if (!regmap) { + dev_err_probe(dev, -ENODEV, "Failed to get parent regmap\n"); + return -ENODEV; + } + + a7cc = devm_kzalloc(dev, sizeof(*a7cc), GFP_KERNEL); + if (!a7cc) + return -ENOMEM; + + init.name = "a7mux"; + init.parent_data = pdata; + init.num_parents = ARRAY_SIZE(pdata); + init.ops = &clk_regmap_mux_div_ops; + + a7cc->clkr.hw.init = &init; + a7cc->clkr.regmap = regmap; + a7cc->reg_offset = 0x8; + a7cc->hid_width = 5; + a7cc->hid_shift = 0; + a7cc->src_width = 3; + a7cc->src_shift = 8; + a7cc->parent_map = apcs_mux_clk_parent_map; + + a7cc->pclk = devm_clk_get(parent, "pll"); + if (IS_ERR(a7cc->pclk)) { + ret = PTR_ERR(a7cc->pclk); + if (ret != -EPROBE_DEFER) + dev_err_probe(dev, ret, "Failed to get PLL clk\n"); + return ret; + } + + a7cc->clk_nb.notifier_call = a7cc_notifier_cb; + ret = clk_notifier_register(a7cc->pclk, &a7cc->clk_nb); + if (ret) { + dev_err_probe(dev, ret, "Failed to register clock notifier\n"); + return ret; + } + + ret = devm_clk_register_regmap(dev, &a7cc->clkr); + if (ret) { + dev_err_probe(dev, ret, "Failed to register regmap clock\n"); + goto err; + } + + ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, + &a7cc->clkr.hw); + if (ret) { + dev_err_probe(dev, ret, "Failed to add clock provider\n"); + goto err; + } + + platform_set_drvdata(pdev, a7cc); + + /* + * Attach the power domain to cpudev. Since there is no dedicated driver + * for CPUs and the SDX55 platform lacks hardware specific CPUFreq + * driver, there seems to be no better place to do this. So do it here! + */ + cpu_dev = get_cpu_device(0); + dev_pm_domain_attach(cpu_dev, true); + + return 0; + +err: + clk_notifier_unregister(a7cc->pclk, &a7cc->clk_nb); + return ret; +} + +static int qcom_apcs_sdx55_clk_remove(struct platform_device *pdev) +{ + struct device *cpu_dev = get_cpu_device(0); + struct clk_regmap_mux_div *a7cc = platform_get_drvdata(pdev); + + clk_notifier_unregister(a7cc->pclk, &a7cc->clk_nb); + dev_pm_domain_detach(cpu_dev, true); + + return 0; +} + +static struct platform_driver qcom_apcs_sdx55_clk_driver = { + .probe = qcom_apcs_sdx55_clk_probe, + .remove = qcom_apcs_sdx55_clk_remove, + .driver = { + .name = "qcom-sdx55-acps-clk", + }, +}; +module_platform_driver(qcom_apcs_sdx55_clk_driver); + +MODULE_AUTHOR("Manivannan Sadhasivam "); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Qualcomm SDX55 APCS clock driver"); diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 21c357c26ec4..c6eb99169ddc 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -156,6 +156,12 @@ EXPORT_SYMBOL_GPL(clk_alpha_pll_regs); /* LUCID PLL specific settings and offsets */ #define LUCID_PCAL_DONE BIT(27) +/* LUCID 5LPE PLL specific settings and offsets */ +#define LUCID_5LPE_PCAL_DONE BIT(11) +#define LUCID_5LPE_ALPHA_PLL_ACK_LATCH BIT(13) +#define LUCID_5LPE_PLL_LATCH_INPUT BIT(14) +#define LUCID_5LPE_ENABLE_VOTE_RUN BIT(21) + #define pll_alpha_width(p) \ ((PLL_ALPHA_VAL_U(p) - PLL_ALPHA_VAL(p) == 4) ? \ ALPHA_REG_BITWIDTH : ALPHA_REG_16BIT_WIDTH) @@ -777,15 +783,15 @@ static long alpha_pll_huayra_round_rate(struct clk_hw *hw, unsigned long rate, static int trion_pll_is_enabled(struct clk_alpha_pll *pll, struct regmap *regmap) { - u32 mode_regval, opmode_regval; + u32 mode_val, opmode_val; int ret; - ret = regmap_read(regmap, PLL_MODE(pll), &mode_regval); - ret |= regmap_read(regmap, PLL_OPMODE(pll), &opmode_regval); + ret = regmap_read(regmap, PLL_MODE(pll), &mode_val); + ret |= regmap_read(regmap, PLL_OPMODE(pll), &opmode_val); if (ret) return 0; - return ((opmode_regval & PLL_RUN) && (mode_regval & PLL_OUTCTRL)); + return ((opmode_val & PLL_RUN) && (mode_val & PLL_OUTCTRL)); } static int clk_trion_pll_is_enabled(struct clk_hw *hw) @@ -1445,12 +1451,12 @@ EXPORT_SYMBOL_GPL(clk_trion_pll_configure); static int __alpha_pll_trion_prepare(struct clk_hw *hw, u32 pcal_done) { struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); - u32 regval; + u32 val; int ret; /* Return early if calibration is not needed. */ - regmap_read(pll->clkr.regmap, PLL_STATUS(pll), ®val); - if (regval & pcal_done) + regmap_read(pll->clkr.regmap, PLL_STATUS(pll), &val); + if (val & pcal_done) return 0; /* On/off to calibrate */ @@ -1471,12 +1477,12 @@ static int alpha_pll_lucid_prepare(struct clk_hw *hw) return __alpha_pll_trion_prepare(hw, LUCID_PCAL_DONE); } -static int alpha_pll_trion_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long prate) +static int __alpha_pll_trion_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long prate, u32 latch_bit, u32 latch_ack) { struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); unsigned long rrate; - u32 regval, l, alpha_width = pll_alpha_width(pll); + u32 val, l, alpha_width = pll_alpha_width(pll); u64 a; int ret; @@ -1490,22 +1496,20 @@ static int alpha_pll_trion_set_rate(struct clk_hw *hw, unsigned long rate, regmap_write(pll->clkr.regmap, PLL_ALPHA_VAL(pll), a); /* Latch the PLL input */ - ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), - PLL_UPDATE, PLL_UPDATE); + ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), latch_bit, latch_bit); if (ret) return ret; /* Wait for 2 reference cycles before checking the ACK bit. */ udelay(1); - regmap_read(pll->clkr.regmap, PLL_MODE(pll), ®val); - if (!(regval & ALPHA_PLL_ACK_LATCH)) { + regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val); + if (!(val & latch_ack)) { pr_err("Lucid PLL latch failed. Output may be unstable!\n"); return -EINVAL; } /* Return the latch input to 0 */ - ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), - PLL_UPDATE, 0); + ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), latch_bit, 0); if (ret) return ret; @@ -1520,6 +1524,12 @@ static int alpha_pll_trion_set_rate(struct clk_hw *hw, unsigned long rate, return 0; } +static int alpha_pll_trion_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long prate) +{ + return __alpha_pll_trion_set_rate(hw, rate, prate, PLL_UPDATE, ALPHA_PLL_ACK_LATCH); +} + const struct clk_ops clk_alpha_pll_trion_ops = { .prepare = alpha_pll_trion_prepare, .enable = clk_trion_pll_enable, @@ -1600,3 +1610,170 @@ const struct clk_ops clk_alpha_pll_agera_ops = { .set_rate = clk_alpha_pll_agera_set_rate, }; EXPORT_SYMBOL_GPL(clk_alpha_pll_agera_ops); + +static int alpha_pll_lucid_5lpe_enable(struct clk_hw *hw) +{ + struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); + u32 val; + int ret; + + ret = regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &val); + if (ret) + return ret; + + /* If in FSM mode, just vote for it */ + if (val & LUCID_5LPE_ENABLE_VOTE_RUN) { + ret = clk_enable_regmap(hw); + if (ret) + return ret; + return wait_for_pll_enable_lock(pll); + } + + /* Check if PLL is already enabled, return if enabled */ + ret = trion_pll_is_enabled(pll, pll->clkr.regmap); + if (ret < 0) + return ret; + + ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), PLL_RESET_N, PLL_RESET_N); + if (ret) + return ret; + + regmap_write(pll->clkr.regmap, PLL_OPMODE(pll), PLL_RUN); + + ret = wait_for_pll_enable_lock(pll); + if (ret) + return ret; + + /* Enable the PLL outputs */ + ret = regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll), PLL_OUT_MASK, PLL_OUT_MASK); + if (ret) + return ret; + + /* Enable the global PLL outputs */ + return regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), PLL_OUTCTRL, PLL_OUTCTRL); +} + +static void alpha_pll_lucid_5lpe_disable(struct clk_hw *hw) +{ + struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); + u32 val; + int ret; + + ret = regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &val); + if (ret) + return; + + /* If in FSM mode, just unvote it */ + if (val & LUCID_5LPE_ENABLE_VOTE_RUN) { + clk_disable_regmap(hw); + return; + } + + /* Disable the global PLL output */ + ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), PLL_OUTCTRL, 0); + if (ret) + return; + + /* Disable the PLL outputs */ + ret = regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll), PLL_OUT_MASK, 0); + if (ret) + return; + + /* Place the PLL mode in STANDBY */ + regmap_write(pll->clkr.regmap, PLL_OPMODE(pll), PLL_STANDBY); +} + +/* + * The Lucid 5LPE PLL requires a power-on self-calibration which happens + * when the PLL comes out of reset. Calibrate in case it is not completed. + */ +static int alpha_pll_lucid_5lpe_prepare(struct clk_hw *hw) +{ + struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); + struct clk_hw *p; + u32 val = 0; + int ret; + + /* Return early if calibration is not needed. */ + regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val); + if (val & LUCID_5LPE_PCAL_DONE) + return 0; + + p = clk_hw_get_parent(hw); + if (!p) + return -EINVAL; + + ret = alpha_pll_lucid_5lpe_enable(hw); + if (ret) + return ret; + + alpha_pll_lucid_5lpe_disable(hw); + + return 0; +} + +static int alpha_pll_lucid_5lpe_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long prate) +{ + return __alpha_pll_trion_set_rate(hw, rate, prate, + LUCID_5LPE_PLL_LATCH_INPUT, + LUCID_5LPE_ALPHA_PLL_ACK_LATCH); +} + +static int clk_lucid_5lpe_pll_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); + int i, val = 0, div, ret; + u32 mask; + + /* + * If the PLL is in FSM mode, then treat set_rate callback as a + * no-operation. + */ + ret = regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &val); + if (ret) + return ret; + + if (val & LUCID_5LPE_ENABLE_VOTE_RUN) + return 0; + + div = DIV_ROUND_UP_ULL((u64)parent_rate, rate); + for (i = 0; i < pll->num_post_div; i++) { + if (pll->post_div_table[i].div == div) { + val = pll->post_div_table[i].val; + break; + } + } + + mask = GENMASK(pll->width + pll->post_div_shift - 1, pll->post_div_shift); + return regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll), + mask, val << pll->post_div_shift); +} + +const struct clk_ops clk_alpha_pll_lucid_5lpe_ops = { + .prepare = alpha_pll_lucid_5lpe_prepare, + .enable = alpha_pll_lucid_5lpe_enable, + .disable = alpha_pll_lucid_5lpe_disable, + .is_enabled = clk_trion_pll_is_enabled, + .recalc_rate = clk_trion_pll_recalc_rate, + .round_rate = clk_alpha_pll_round_rate, + .set_rate = alpha_pll_lucid_5lpe_set_rate, +}; +EXPORT_SYMBOL(clk_alpha_pll_lucid_5lpe_ops); + +const struct clk_ops clk_alpha_pll_fixed_lucid_5lpe_ops = { + .enable = alpha_pll_lucid_5lpe_enable, + .disable = alpha_pll_lucid_5lpe_disable, + .is_enabled = clk_trion_pll_is_enabled, + .recalc_rate = clk_trion_pll_recalc_rate, + .round_rate = clk_alpha_pll_round_rate, +}; +EXPORT_SYMBOL(clk_alpha_pll_fixed_lucid_5lpe_ops); + +const struct clk_ops clk_alpha_pll_postdiv_lucid_5lpe_ops = { + .recalc_rate = clk_alpha_pll_postdiv_fabia_recalc_rate, + .round_rate = clk_alpha_pll_postdiv_fabia_round_rate, + .set_rate = clk_lucid_5lpe_pll_postdiv_set_rate, +}; +EXPORT_SYMBOL(clk_alpha_pll_postdiv_lucid_5lpe_ops); diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h index 0ea30d2f3da1..6943e933be0f 100644 --- a/drivers/clk/qcom/clk-alpha-pll.h +++ b/drivers/clk/qcom/clk-alpha-pll.h @@ -144,6 +144,10 @@ extern const struct clk_ops clk_alpha_pll_lucid_ops; extern const struct clk_ops clk_alpha_pll_postdiv_lucid_ops; extern const struct clk_ops clk_alpha_pll_agera_ops; +extern const struct clk_ops clk_alpha_pll_lucid_5lpe_ops; +extern const struct clk_ops clk_alpha_pll_fixed_lucid_5lpe_ops; +extern const struct clk_ops clk_alpha_pll_postdiv_lucid_5lpe_ops; + void clk_alpha_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap, const struct alpha_pll_config *config); void clk_fabia_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap, diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h index 86d2b8b90173..99efcc7f8d88 100644 --- a/drivers/clk/qcom/clk-rcg.h +++ b/drivers/clk/qcom/clk-rcg.h @@ -153,6 +153,15 @@ struct clk_rcg2 { #define to_clk_rcg2(_hw) container_of(to_clk_regmap(_hw), struct clk_rcg2, clkr) +struct clk_rcg2_gfx3d { + u8 div; + struct clk_rcg2 rcg; + struct clk_hw **hws; +}; + +#define to_clk_rcg2_gfx3d(_hw) \ + container_of(to_clk_rcg2(_hw), struct clk_rcg2_gfx3d, rcg) + extern const struct clk_ops clk_rcg2_ops; extern const struct clk_ops clk_rcg2_floor_ops; extern const struct clk_ops clk_edp_pixel_ops; diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index 59a5a0f261f3..42f13a2d1cc1 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -728,40 +728,51 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct clk_rate_request parent_req = { }; - struct clk_hw *p2, *p8, *p9, *xo; - unsigned long p9_rate; + struct clk_rcg2_gfx3d *cgfx = to_clk_rcg2_gfx3d(hw); + struct clk_hw *xo, *p0, *p1, *p2; + unsigned long request, p0_rate; int ret; + p0 = cgfx->hws[0]; + p1 = cgfx->hws[1]; + p2 = cgfx->hws[2]; + /* + * This function does ping-pong the RCG between PLLs: if we don't + * have at least one fixed PLL and two variable ones, + * then it's not going to work correctly. + */ + if (WARN_ON(!p0 || !p1 || !p2)) + return -EINVAL; + xo = clk_hw_get_parent_by_index(hw, 0); if (req->rate == clk_hw_get_rate(xo)) { req->best_parent_hw = xo; return 0; } - p9 = clk_hw_get_parent_by_index(hw, 2); - p2 = clk_hw_get_parent_by_index(hw, 3); - p8 = clk_hw_get_parent_by_index(hw, 4); + request = req->rate; + if (cgfx->div > 1) + parent_req.rate = request = request * cgfx->div; - /* PLL9 is a fixed rate PLL */ - p9_rate = clk_hw_get_rate(p9); + /* This has to be a fixed rate PLL */ + p0_rate = clk_hw_get_rate(p0); - parent_req.rate = req->rate = min(req->rate, p9_rate); - if (req->rate == p9_rate) { - req->rate = req->best_parent_rate = p9_rate; - req->best_parent_hw = p9; + if (request == p0_rate) { + req->rate = req->best_parent_rate = p0_rate; + req->best_parent_hw = p0; return 0; } - if (req->best_parent_hw == p9) { + if (req->best_parent_hw == p0) { /* Are we going back to a previously used rate? */ - if (clk_hw_get_rate(p8) == req->rate) - req->best_parent_hw = p8; - else + if (clk_hw_get_rate(p2) == request) req->best_parent_hw = p2; - } else if (req->best_parent_hw == p8) { - req->best_parent_hw = p2; + else + req->best_parent_hw = p1; + } else if (req->best_parent_hw == p2) { + req->best_parent_hw = p1; } else { - req->best_parent_hw = p8; + req->best_parent_hw = p2; } ret = __clk_determine_rate(req->best_parent_hw, &parent_req); @@ -769,6 +780,8 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, return ret; req->rate = req->best_parent_rate = parent_req.rate; + if (cgfx->div > 1) + req->rate /= cgfx->div; return 0; } @@ -776,12 +789,16 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, static int clk_gfx3d_set_rate_and_parent(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate, u8 index) { - struct clk_rcg2 *rcg = to_clk_rcg2(hw); + struct clk_rcg2_gfx3d *cgfx = to_clk_rcg2_gfx3d(hw); + struct clk_rcg2 *rcg = &cgfx->rcg; u32 cfg; int ret; - /* Just mux it, we don't use the division or m/n hardware */ cfg = rcg->parent_map[index].cfg << CFG_SRC_SEL_SHIFT; + /* On some targets, the GFX3D RCG may need to divide PLL frequency */ + if (cgfx->div > 1) + cfg |= ((2 * cgfx->div) - 1) << CFG_SRC_DIV_SHIFT; + ret = regmap_write(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, cfg); if (ret) return ret; diff --git a/drivers/clk/qcom/clk-regmap.c b/drivers/clk/qcom/clk-regmap.c index ce80db27ccf2..92ac4e0d7dbe 100644 --- a/drivers/clk/qcom/clk-regmap.c +++ b/drivers/clk/qcom/clk-regmap.c @@ -87,6 +87,7 @@ EXPORT_SYMBOL_GPL(clk_disable_regmap); /** * devm_clk_register_regmap - register a clk_regmap clock * + * @dev: reference to the caller's device * @rclk: clk to operate on * * Clocks that use regmap for their register I/O should register their diff --git a/drivers/clk/qcom/clk-rpm.c b/drivers/clk/qcom/clk-rpm.c index f71d228fd6bd..a18811c38018 100644 --- a/drivers/clk/qcom/clk-rpm.c +++ b/drivers/clk/qcom/clk-rpm.c @@ -73,62 +73,6 @@ }, \ } -#define DEFINE_CLK_RPM_PXO_BRANCH(_platform, _name, _active, r_id, r) \ - static struct clk_rpm _platform##_##_active; \ - static struct clk_rpm _platform##_##_name = { \ - .rpm_clk_id = (r_id), \ - .active_only = true, \ - .peer = &_platform##_##_active, \ - .rate = (r), \ - .branch = true, \ - .hw.init = &(struct clk_init_data){ \ - .ops = &clk_rpm_branch_ops, \ - .name = #_name, \ - .parent_names = (const char *[]){ "pxo_board" }, \ - .num_parents = 1, \ - }, \ - }; \ - static struct clk_rpm _platform##_##_active = { \ - .rpm_clk_id = (r_id), \ - .peer = &_platform##_##_name, \ - .rate = (r), \ - .branch = true, \ - .hw.init = &(struct clk_init_data){ \ - .ops = &clk_rpm_branch_ops, \ - .name = #_active, \ - .parent_names = (const char *[]){ "pxo_board" }, \ - .num_parents = 1, \ - }, \ - } - -#define DEFINE_CLK_RPM_CXO_BRANCH(_platform, _name, _active, r_id, r) \ - static struct clk_rpm _platform##_##_active; \ - static struct clk_rpm _platform##_##_name = { \ - .rpm_clk_id = (r_id), \ - .peer = &_platform##_##_active, \ - .rate = (r), \ - .branch = true, \ - .hw.init = &(struct clk_init_data){ \ - .ops = &clk_rpm_branch_ops, \ - .name = #_name, \ - .parent_names = (const char *[]){ "cxo_board" }, \ - .num_parents = 1, \ - }, \ - }; \ - static struct clk_rpm _platform##_##_active = { \ - .rpm_clk_id = (r_id), \ - .active_only = true, \ - .peer = &_platform##_##_name, \ - .rate = (r), \ - .branch = true, \ - .hw.init = &(struct clk_init_data){ \ - .ops = &clk_rpm_branch_ops, \ - .name = #_active, \ - .parent_names = (const char *[]){ "cxo_board" }, \ - .num_parents = 1, \ - }, \ - } - #define to_clk_rpm(_hw) container_of(_hw, struct clk_rpm, hw) struct rpm_cc; @@ -450,13 +394,6 @@ static const struct clk_ops clk_rpm_ops = { .recalc_rate = clk_rpm_recalc_rate, }; -static const struct clk_ops clk_rpm_branch_ops = { - .prepare = clk_rpm_prepare, - .unprepare = clk_rpm_unprepare, - .round_rate = clk_rpm_round_rate, - .recalc_rate = clk_rpm_recalc_rate, -}; - /* MSM8660/APQ8060 */ DEFINE_CLK_RPM(msm8660, afab_clk, afab_a_clk, QCOM_RPM_APPS_FABRIC_CLK); DEFINE_CLK_RPM(msm8660, sfab_clk, sfab_a_clk, QCOM_RPM_SYS_FABRIC_CLK); diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c index 6a2a13c5058e..91dc390a583b 100644 --- a/drivers/clk/qcom/clk-rpmh.c +++ b/drivers/clk/qcom/clk-rpmh.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. */ #include @@ -348,6 +348,10 @@ DEFINE_CLK_RPMH_VRM(sdm845, rf_clk1, rf_clk1_ao, "rfclka1", 1); DEFINE_CLK_RPMH_VRM(sdm845, rf_clk2, rf_clk2_ao, "rfclka2", 1); DEFINE_CLK_RPMH_VRM(sdm845, rf_clk3, rf_clk3_ao, "rfclka3", 1); DEFINE_CLK_RPMH_VRM(sm8150, rf_clk3, rf_clk3_ao, "rfclka3", 1); +DEFINE_CLK_RPMH_VRM(sc8180x, rf_clk1, rf_clk1_ao, "rfclkd1", 1); +DEFINE_CLK_RPMH_VRM(sc8180x, rf_clk2, rf_clk2_ao, "rfclkd2", 1); +DEFINE_CLK_RPMH_VRM(sc8180x, rf_clk3, rf_clk3_ao, "rfclkd3", 1); +DEFINE_CLK_RPMH_VRM(sc8180x, rf_clk4, rf_clk4_ao, "rfclkd4", 1); DEFINE_CLK_RPMH_BCM(sdm845, ipa, "IP0"); DEFINE_CLK_RPMH_BCM(sdm845, ce, "CE0"); @@ -431,6 +435,26 @@ static const struct clk_rpmh_desc clk_rpmh_sc7180 = { .num_clks = ARRAY_SIZE(sc7180_rpmh_clocks), }; +static struct clk_hw *sc8180x_rpmh_clocks[] = { + [RPMH_CXO_CLK] = &sdm845_bi_tcxo.hw, + [RPMH_CXO_CLK_A] = &sdm845_bi_tcxo_ao.hw, + [RPMH_LN_BB_CLK2] = &sdm845_ln_bb_clk2.hw, + [RPMH_LN_BB_CLK2_A] = &sdm845_ln_bb_clk2_ao.hw, + [RPMH_LN_BB_CLK3] = &sdm845_ln_bb_clk3.hw, + [RPMH_LN_BB_CLK3_A] = &sdm845_ln_bb_clk3_ao.hw, + [RPMH_RF_CLK1] = &sc8180x_rf_clk1.hw, + [RPMH_RF_CLK1_A] = &sc8180x_rf_clk1_ao.hw, + [RPMH_RF_CLK2] = &sc8180x_rf_clk2.hw, + [RPMH_RF_CLK2_A] = &sc8180x_rf_clk2_ao.hw, + [RPMH_RF_CLK3] = &sc8180x_rf_clk3.hw, + [RPMH_RF_CLK3_A] = &sc8180x_rf_clk3_ao.hw, +}; + +static const struct clk_rpmh_desc clk_rpmh_sc8180x = { + .clks = sc8180x_rpmh_clocks, + .num_clks = ARRAY_SIZE(sc8180x_rpmh_clocks), +}; + DEFINE_CLK_RPMH_VRM(sm8250, ln_bb_clk1, ln_bb_clk1_ao, "lnbclka1", 2); static struct clk_hw *sm8250_rpmh_clocks[] = { @@ -486,6 +510,27 @@ static const struct clk_rpmh_desc clk_rpmh_sm8350 = { .num_clks = ARRAY_SIZE(sm8350_rpmh_clocks), }; +static struct clk_hw *sc7280_rpmh_clocks[] = { + [RPMH_CXO_CLK] = &sdm845_bi_tcxo.hw, + [RPMH_CXO_CLK_A] = &sdm845_bi_tcxo_ao.hw, + [RPMH_LN_BB_CLK2] = &sdm845_ln_bb_clk2.hw, + [RPMH_LN_BB_CLK2_A] = &sdm845_ln_bb_clk2_ao.hw, + [RPMH_RF_CLK1] = &sdm845_rf_clk1.hw, + [RPMH_RF_CLK1_A] = &sdm845_rf_clk1_ao.hw, + [RPMH_RF_CLK3] = &sdm845_rf_clk3.hw, + [RPMH_RF_CLK3_A] = &sdm845_rf_clk3_ao.hw, + [RPMH_RF_CLK4] = &sm8350_rf_clk4.hw, + [RPMH_RF_CLK4_A] = &sm8350_rf_clk4_ao.hw, + [RPMH_IPA_CLK] = &sdm845_ipa.hw, + [RPMH_PKA_CLK] = &sm8350_pka.hw, + [RPMH_HWKM_CLK] = &sm8350_hwkm.hw, +}; + +static const struct clk_rpmh_desc clk_rpmh_sc7280 = { + .clks = sc7280_rpmh_clocks, + .num_clks = ARRAY_SIZE(sc7280_rpmh_clocks), +}; + static struct clk_hw *of_clk_rpmh_hw_get(struct of_phandle_args *clkspec, void *data) { @@ -570,11 +615,13 @@ static int clk_rpmh_probe(struct platform_device *pdev) static const struct of_device_id clk_rpmh_match_table[] = { { .compatible = "qcom,sc7180-rpmh-clk", .data = &clk_rpmh_sc7180}, + { .compatible = "qcom,sc8180x-rpmh-clk", .data = &clk_rpmh_sc8180x}, { .compatible = "qcom,sdm845-rpmh-clk", .data = &clk_rpmh_sdm845}, { .compatible = "qcom,sdx55-rpmh-clk", .data = &clk_rpmh_sdx55}, { .compatible = "qcom,sm8150-rpmh-clk", .data = &clk_rpmh_sm8150}, { .compatible = "qcom,sm8250-rpmh-clk", .data = &clk_rpmh_sm8250}, { .compatible = "qcom,sm8350-rpmh-clk", .data = &clk_rpmh_sm8350}, + { .compatible = "qcom,sc7280-rpmh-clk", .data = &clk_rpmh_sc7280}, { } }; MODULE_DEVICE_TABLE(of, clk_rpmh_match_table); diff --git a/drivers/clk/qcom/gcc-ipq4019.c b/drivers/clk/qcom/gcc-ipq4019.c index ef5137fd50f3..8abad4032de7 100644 --- a/drivers/clk/qcom/gcc-ipq4019.c +++ b/drivers/clk/qcom/gcc-ipq4019.c @@ -1276,16 +1276,15 @@ static int clk_cpu_div_set_rate(struct clk_hw *hw, unsigned long rate, struct clk_fepll *pll = to_clk_fepll(hw); const struct freq_tbl *f; u32 mask; - int ret; f = qcom_find_freq(pll->freq_tbl, rate); if (!f) return -EINVAL; mask = (BIT(pll->cdiv.width) - 1) << pll->cdiv.shift; - ret = regmap_update_bits(pll->cdiv.clkr.regmap, - pll->cdiv.reg, mask, - f->pre_div << pll->cdiv.shift); + regmap_update_bits(pll->cdiv.clkr.regmap, + pll->cdiv.reg, mask, + f->pre_div << pll->cdiv.shift); /* * There is no status bit which can be checked for successful CPU * divider update operation so using delay for the same. diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c index 9d7016bcd680..050c91af888e 100644 --- a/drivers/clk/qcom/gcc-msm8998.c +++ b/drivers/clk/qcom/gcc-msm8998.c @@ -135,7 +135,7 @@ static struct pll_vco fabia_vco[] = { static struct clk_alpha_pll gpll0 = { .offset = 0x0, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .vco_table = fabia_vco, .num_vco = ARRAY_SIZE(fabia_vco), .clkr = { @@ -145,58 +145,58 @@ static struct clk_alpha_pll gpll0 = { .name = "gpll0", .parent_names = (const char *[]){ "xo" }, .num_parents = 1, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_fabia_ops, } }, }; static struct clk_alpha_pll_postdiv gpll0_out_even = { .offset = 0x0, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_even", .parent_names = (const char *[]){ "gpll0" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll0_out_main = { .offset = 0x0, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_main", .parent_names = (const char *[]){ "gpll0" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll0_out_odd = { .offset = 0x0, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_odd", .parent_names = (const char *[]){ "gpll0" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll0_out_test = { .offset = 0x0, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_test", .parent_names = (const char *[]){ "gpll0" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll gpll1 = { .offset = 0x1000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .vco_table = fabia_vco, .num_vco = ARRAY_SIZE(fabia_vco), .clkr = { @@ -206,58 +206,58 @@ static struct clk_alpha_pll gpll1 = { .name = "gpll1", .parent_names = (const char *[]){ "xo" }, .num_parents = 1, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_fabia_ops, } }, }; static struct clk_alpha_pll_postdiv gpll1_out_even = { .offset = 0x1000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll1_out_even", .parent_names = (const char *[]){ "gpll1" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll1_out_main = { .offset = 0x1000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll1_out_main", .parent_names = (const char *[]){ "gpll1" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll1_out_odd = { .offset = 0x1000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll1_out_odd", .parent_names = (const char *[]){ "gpll1" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll1_out_test = { .offset = 0x1000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll1_out_test", .parent_names = (const char *[]){ "gpll1" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll gpll2 = { .offset = 0x2000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .vco_table = fabia_vco, .num_vco = ARRAY_SIZE(fabia_vco), .clkr = { @@ -267,58 +267,58 @@ static struct clk_alpha_pll gpll2 = { .name = "gpll2", .parent_names = (const char *[]){ "xo" }, .num_parents = 1, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_fabia_ops, } }, }; static struct clk_alpha_pll_postdiv gpll2_out_even = { .offset = 0x2000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll2_out_even", .parent_names = (const char *[]){ "gpll2" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll2_out_main = { .offset = 0x2000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll2_out_main", .parent_names = (const char *[]){ "gpll2" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll2_out_odd = { .offset = 0x2000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll2_out_odd", .parent_names = (const char *[]){ "gpll2" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll2_out_test = { .offset = 0x2000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll2_out_test", .parent_names = (const char *[]){ "gpll2" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll gpll3 = { .offset = 0x3000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .vco_table = fabia_vco, .num_vco = ARRAY_SIZE(fabia_vco), .clkr = { @@ -328,58 +328,58 @@ static struct clk_alpha_pll gpll3 = { .name = "gpll3", .parent_names = (const char *[]){ "xo" }, .num_parents = 1, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_fabia_ops, } }, }; static struct clk_alpha_pll_postdiv gpll3_out_even = { .offset = 0x3000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll3_out_even", .parent_names = (const char *[]){ "gpll3" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll3_out_main = { .offset = 0x3000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll3_out_main", .parent_names = (const char *[]){ "gpll3" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll3_out_odd = { .offset = 0x3000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll3_out_odd", .parent_names = (const char *[]){ "gpll3" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll3_out_test = { .offset = 0x3000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll3_out_test", .parent_names = (const char *[]){ "gpll3" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll gpll4 = { .offset = 0x77000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .vco_table = fabia_vco, .num_vco = ARRAY_SIZE(fabia_vco), .clkr = { @@ -389,52 +389,52 @@ static struct clk_alpha_pll gpll4 = { .name = "gpll4", .parent_names = (const char *[]){ "xo" }, .num_parents = 1, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_fabia_ops, } }, }; static struct clk_alpha_pll_postdiv gpll4_out_even = { .offset = 0x77000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll4_out_even", .parent_names = (const char *[]){ "gpll4" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll4_out_main = { .offset = 0x77000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll4_out_main", .parent_names = (const char *[]){ "gpll4" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll4_out_odd = { .offset = 0x77000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll4_out_odd", .parent_names = (const char *[]){ "gpll4" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll4_out_test = { .offset = 0x77000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll4_out_test", .parent_names = (const char *[]){ "gpll4" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; @@ -1341,6 +1341,22 @@ static struct clk_branch gcc_boot_rom_ahb_clk = { }, }; +static struct clk_branch gcc_mmss_gpll0_clk = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mmss_gpll0_clk", + .parent_names = (const char *[]){ + "gpll0_out_main", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_mss_gpll0_div_clk_src = { .halt_check = BRANCH_HALT_DELAY, .clkr = { @@ -2065,6 +2081,12 @@ static struct clk_branch gcc_gpu_cfg_ahb_clk = { .hw.init = &(struct clk_init_data){ .name = "gcc_gpu_cfg_ahb_clk", .ops = &clk_branch2_ops, + /* + * The GPU IOMMU depends on this clock and hypervisor + * will crash the SoC if this clock goes down, due to + * secure contexts protection. + */ + .flags = CLK_IS_CRITICAL, }, }, }; @@ -2144,6 +2166,25 @@ static struct clk_branch gcc_hmss_trig_clk = { }, }; +static struct freq_tbl ftbl_hmss_gpll0_clk_src[] = { + F( 300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + F( 600000000, P_GPLL0_OUT_MAIN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 hmss_gpll0_clk_src = { + .cmd_rcgr = 0x4805c, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_hmss_gpll0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "hmss_gpll0_clk_src", + .parent_names = gcc_parent_names_1, + .num_parents = ARRAY_SIZE(gcc_parent_names_1), + .ops = &clk_rcg2_ops, + }, +}; + static struct clk_branch gcc_mmss_noc_cfg_ahb_clk = { .halt_reg = 0x9004, .halt_check = BRANCH_HALT, @@ -2944,6 +2985,8 @@ static struct clk_regmap *gcc_msm8998_clocks[] = { [GCC_MSS_GPLL0_DIV_CLK_SRC] = &gcc_mss_gpll0_div_clk_src.clkr, [GCC_MSS_SNOC_AXI_CLK] = &gcc_mss_snoc_axi_clk.clkr, [GCC_MSS_MNOC_BIMC_AXI_CLK] = &gcc_mss_mnoc_bimc_axi_clk.clkr, + [GCC_MMSS_GPLL0_CLK] = &gcc_mmss_gpll0_clk.clkr, + [HMSS_GPLL0_CLK_SRC] = &hmss_gpll0_clk_src.clkr, }; static struct gdsc *gcc_msm8998_gdscs[] = { diff --git a/drivers/clk/qcom/gcc-sc7180.c b/drivers/clk/qcom/gcc-sc7180.c index b05901b24917..88e896abb663 100644 --- a/drivers/clk/qcom/gcc-sc7180.c +++ b/drivers/clk/qcom/gcc-sc7180.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */ #include @@ -919,19 +919,6 @@ static struct clk_branch gcc_camera_throttle_hf_axi_clk = { }, }; -static struct clk_branch gcc_camera_xo_clk = { - .halt_reg = 0xb02c, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0xb02c, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gcc_camera_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_ce1_ahb_clk = { .halt_reg = 0x4100c, .halt_check = BRANCH_HALT_VOTED, @@ -1096,19 +1083,6 @@ static struct clk_branch gcc_disp_throttle_hf_axi_clk = { }, }; -static struct clk_branch gcc_disp_xo_clk = { - .halt_reg = 0xb030, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0xb030, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gcc_disp_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_gp1_clk = { .halt_reg = 0x64000, .halt_check = BRANCH_HALT, @@ -2159,19 +2133,6 @@ static struct clk_branch gcc_video_throttle_axi_clk = { }, }; -static struct clk_branch gcc_video_xo_clk = { - .halt_reg = 0xb028, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0xb028, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gcc_video_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_mss_cfg_ahb_clk = { .halt_reg = 0x8a000, .halt_check = BRANCH_HALT, @@ -2304,7 +2265,6 @@ static struct clk_regmap *gcc_sc7180_clocks[] = { [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, [GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr, [GCC_CAMERA_THROTTLE_HF_AXI_CLK] = &gcc_camera_throttle_hf_axi_clk.clkr, - [GCC_CAMERA_XO_CLK] = &gcc_camera_xo_clk.clkr, [GCC_CE1_AHB_CLK] = &gcc_ce1_ahb_clk.clkr, [GCC_CE1_AXI_CLK] = &gcc_ce1_axi_clk.clkr, [GCC_CE1_CLK] = &gcc_ce1_clk.clkr, @@ -2317,7 +2277,6 @@ static struct clk_regmap *gcc_sc7180_clocks[] = { [GCC_DISP_GPLL0_DIV_CLK_SRC] = &gcc_disp_gpll0_div_clk_src.clkr, [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, [GCC_DISP_THROTTLE_HF_AXI_CLK] = &gcc_disp_throttle_hf_axi_clk.clkr, - [GCC_DISP_XO_CLK] = &gcc_disp_xo_clk.clkr, [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, @@ -2413,7 +2372,6 @@ static struct clk_regmap *gcc_sc7180_clocks[] = { [GCC_VIDEO_AXI_CLK] = &gcc_video_axi_clk.clkr, [GCC_VIDEO_GPLL0_DIV_CLK_SRC] = &gcc_video_gpll0_div_clk_src.clkr, [GCC_VIDEO_THROTTLE_AXI_CLK] = &gcc_video_throttle_axi_clk.clkr, - [GCC_VIDEO_XO_CLK] = &gcc_video_xo_clk.clkr, [GPLL0] = &gpll0.clkr, [GPLL0_OUT_EVEN] = &gpll0_out_even.clkr, [GPLL6] = &gpll6.clkr, @@ -2510,6 +2468,9 @@ static int gcc_sc7180_probe(struct platform_device *pdev) regmap_update_bits(regmap, 0x0b004, BIT(0), BIT(0)); regmap_update_bits(regmap, 0x0b008, BIT(0), BIT(0)); regmap_update_bits(regmap, 0x0b00c, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x0b02c, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x0b028, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x0b030, BIT(0), BIT(0)); regmap_update_bits(regmap, 0x71004, BIT(0), BIT(0)); ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, diff --git a/drivers/clk/qcom/gcc-sc7280.c b/drivers/clk/qcom/gcc-sc7280.c new file mode 100644 index 000000000000..22736c16ed16 --- /dev/null +++ b/drivers/clk/qcom/gcc-sc7280.c @@ -0,0 +1,3603 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "common.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_BI_TCXO, + P_GCC_GPLL0_OUT_EVEN, + P_GCC_GPLL0_OUT_MAIN, + P_GCC_GPLL0_OUT_ODD, + P_GCC_GPLL10_OUT_MAIN, + P_GCC_GPLL4_OUT_MAIN, + P_GCC_GPLL9_OUT_MAIN, + P_PCIE_0_PIPE_CLK, + P_PCIE_1_PIPE_CLK, + P_SLEEP_CLK, + P_UFS_PHY_RX_SYMBOL_0_CLK, + P_UFS_PHY_RX_SYMBOL_1_CLK, + P_UFS_PHY_TX_SYMBOL_0_CLK, + P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, + P_GCC_MSS_GPLL0_MAIN_DIV_CLK, +}; + +static struct clk_alpha_pll gcc_gpll0 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gcc_gpll0_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gcc_gpll0_out_even = { + .offset = 0x0, + .post_div_shift = 8, + .post_div_table = post_div_table_gcc_gpll0_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0_out_even", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_ops, + }, +}; + +static const struct clk_div_table post_div_table_gcc_gpll0_out_odd[] = { + { 0x3, 3 }, + { } +}; + +static struct clk_alpha_pll_postdiv gcc_gpll0_out_odd = { + .offset = 0x0, + .post_div_shift = 12, + .post_div_table = post_div_table_gcc_gpll0_out_odd, + .num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_odd), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0_out_odd", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_ops, + }, +}; + +static struct clk_alpha_pll gcc_gpll1 = { + .offset = 0x1000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll1", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll10 = { + .offset = 0x1e000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll10", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll4 = { + .offset = 0x76000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll9 = { + .offset = 0x1c000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll9", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_gpll0_main_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_gpll0_main_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0_out_even.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct parent_map gcc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_0[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct clk_parent_data gcc_parent_data_0_ao[] = { + { .fw_name = "bi_tcxo_ao" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_ODD, 3 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_1[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_odd.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_SLEEP_CLK, 5 }, +}; + +static const struct clk_parent_data gcc_parent_data_2[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "sleep_clk" }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data gcc_parent_data_3[] = { + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_ODD, 3 }, + { P_SLEEP_CLK, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_4[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_odd.clkr.hw }, + { .fw_name = "sleep_clk" }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_5[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_PCIE_0_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_6[] = { + { .fw_name = "pcie_0_pipe_clk", .name = "pcie_0_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_7[] = { + { P_PCIE_1_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_7[] = { + { .fw_name = "pcie_1_pipe_clk", .name = "pcie_1_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_8[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_ODD, 3 }, + { P_GCC_GPLL10_OUT_MAIN, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_8[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_odd.clkr.hw }, + { .hw = &gcc_gpll10.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_9[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL9_OUT_MAIN, 2 }, + { P_GCC_GPLL0_OUT_ODD, 3 }, + { P_GCC_GPLL4_OUT_MAIN, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_9[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll9.clkr.hw }, + { .hw = &gcc_gpll0_out_odd.clkr.hw }, + { .hw = &gcc_gpll4.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_10[] = { + { P_UFS_PHY_RX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_10[] = { + { .fw_name = "ufs_phy_rx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_11[] = { + { P_UFS_PHY_RX_SYMBOL_1_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_11[] = { + { .fw_name = "ufs_phy_rx_symbol_1_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_12[] = { + { P_UFS_PHY_TX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_12[] = { + { .fw_name = "ufs_phy_tx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_13[] = { + { P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_13[] = { + { .fw_name = "usb3_phy_wrapper_gcc_usb30_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_14[] = { + { P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_14[] = { + { .fw_name = "usb3_phy_wrapper_gcc_usb30_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_15[] = { + { P_BI_TCXO, 0 }, + { P_GCC_MSS_GPLL0_MAIN_DIV_CLK, 1 }, +}; + +static const struct clk_parent_data gcc_parent_data_15[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_mss_gpll0_main_div_clk_src.clkr.hw }, +}; + +static struct clk_regmap_mux gcc_pcie_0_pipe_clk_src = { + .reg = 0x6b054, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_6, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = ARRAY_SIZE(gcc_parent_data_6), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_pcie_1_pipe_clk_src = { + .reg = 0x8d054, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_7, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = ARRAY_SIZE(gcc_parent_data_7), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_0_clk_src = { + .reg = 0x77058, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_10, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk_src", + .parent_data = gcc_parent_data_10, + .num_parents = ARRAY_SIZE(gcc_parent_data_10), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_1_clk_src = { + .reg = 0x770c8, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_11, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk_src", + .parent_data = gcc_parent_data_11, + .num_parents = ARRAY_SIZE(gcc_parent_data_11), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_tx_symbol_0_clk_src = { + .reg = 0x77048, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_12, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk_src", + .parent_data = gcc_parent_data_12, + .num_parents = ARRAY_SIZE(gcc_parent_data_12), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_prim_phy_pipe_clk_src = { + .reg = 0xf060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_13, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk_src", + .parent_data = gcc_parent_data_13, + .num_parents = ARRAY_SIZE(gcc_parent_data_13), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_sec_phy_pipe_clk_src = { + .reg = 0x9e060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_14, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_pipe_clk_src", + .parent_data = gcc_parent_data_14, + .num_parents = ARRAY_SIZE(gcc_parent_data_14), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; +static const struct freq_tbl ftbl_gcc_cpuss_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_cpuss_ahb_clk_src = { + .cmd_rcgr = 0x4800c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_cpuss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_ahb_clk_src", + .parent_data = gcc_parent_data_0_ao, + .num_parents = ARRAY_SIZE(gcc_parent_data_0_ao), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_ODD, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_gp1_clk_src = { + .cmd_rcgr = 0x64004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp2_clk_src = { + .cmd_rcgr = 0x65004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp3_clk_src = { + .cmd_rcgr = 0x66004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_aux_clk_src = { + .cmd_rcgr = 0x6b058, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_phy_rchng_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_phy_rchng_clk_src = { + .cmd_rcgr = 0x6b03c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_aux_clk_src = { + .cmd_rcgr = 0x8d058, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_phy_rchng_clk_src = { + .cmd_rcgr = 0x8d03c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = { + F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pdm2_clk_src = { + .cmd_rcgr = 0x33010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pdm2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qspi_core_clk_src[] = { + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_MAIN, 4, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_qspi_core_clk_src = { + .cmd_rcgr = 0x4b00c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qspi_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GCC_GPLL0_OUT_EVEN, 2.5, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = { + .name = "gcc_qupv3_wrap0_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { + .cmd_rcgr = 0x17010, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = { + .name = "gcc_qupv3_wrap0_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { + .cmd_rcgr = 0x17140, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s1_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s2_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = { + .name = "gcc_qupv3_wrap0_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { + .cmd_rcgr = 0x17270, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = { + .name = "gcc_qupv3_wrap0_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { + .cmd_rcgr = 0x173a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = { + .name = "gcc_qupv3_wrap0_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { + .cmd_rcgr = 0x174d0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = { + .name = "gcc_qupv3_wrap0_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { + .cmd_rcgr = 0x17600, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s6_clk_src_init = { + .name = "gcc_qupv3_wrap0_s6_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s6_clk_src = { + .cmd_rcgr = 0x17730, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s6_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s7_clk_src_init = { + .name = "gcc_qupv3_wrap0_s7_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s7_clk_src = { + .cmd_rcgr = 0x17860, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s7_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = { + .name = "gcc_qupv3_wrap1_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = { + .cmd_rcgr = 0x18010, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = { + .name = "gcc_qupv3_wrap1_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = { + .cmd_rcgr = 0x18140, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s2_clk_src_init = { + .name = "gcc_qupv3_wrap1_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s2_clk_src = { + .cmd_rcgr = 0x18270, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = { + .name = "gcc_qupv3_wrap1_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = { + .cmd_rcgr = 0x183a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = { + .name = "gcc_qupv3_wrap1_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = { + .cmd_rcgr = 0x184d0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = { + .name = "gcc_qupv3_wrap1_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = { + .cmd_rcgr = 0x18600, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s6_clk_src_init = { + .name = "gcc_qupv3_wrap1_s6_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s6_clk_src = { + .cmd_rcgr = 0x18730, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s6_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s7_clk_src_init = { + .name = "gcc_qupv3_wrap1_s7_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s7_clk_src = { + .cmd_rcgr = 0x18860, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s7_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_sdcc1_apps_clk_src[] = { + F(144000, P_BI_TCXO, 16, 3, 25), + F(400000, P_BI_TCXO, 12, 1, 4), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(20000000, P_GCC_GPLL0_OUT_EVEN, 5, 1, 3), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(192000000, P_GCC_GPLL10_OUT_MAIN, 2, 0, 0), + F(384000000, P_GCC_GPLL10_OUT_MAIN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { + .cmd_rcgr = 0x7500c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_gcc_sdcc1_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk_src", + .parent_data = gcc_parent_data_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_8), + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc1_ice_core_clk_src[] = { + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { + .cmd_rcgr = 0x7502c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_sdcc1_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ice_core_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(202000000, P_GCC_GPLL9_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { + .cmd_rcgr = 0x1400c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_9, + .freq_tbl = ftbl_gcc_sdcc2_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk_src", + .parent_data = gcc_parent_data_9, + .num_parents = ARRAY_SIZE(gcc_parent_data_9), + .flags = CLK_OPS_PARENT_ENABLE, + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { + .cmd_rcgr = 0x1600c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_sdcc4_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = { + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = { + .cmd_rcgr = 0x77024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_ice_core_clk_src[] = { + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = { + .cmd_rcgr = 0x7706c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = { + .cmd_rcgr = 0x770a0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = { + .cmd_rcgr = 0x77084, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = { + F(66666667, P_GCC_GPLL0_OUT_EVEN, 4.5, 0, 0), + F(133333333, P_GCC_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_ODD, 1, 0, 0), + F(240000000, P_GCC_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_prim_master_clk_src = { + .cmd_rcgr = 0xf020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_usb30_prim_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_prim_mock_utmi_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = { + .cmd_rcgr = 0xf038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_sec_master_clk_src[] = { + F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0), + F(120000000, P_GCC_GPLL0_OUT_EVEN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_sec_master_clk_src = { + .cmd_rcgr = 0x9e020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_5, + .freq_tbl = ftbl_gcc_usb30_sec_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = ARRAY_SIZE(gcc_parent_data_5), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_sec_mock_utmi_clk_src = { + .cmd_rcgr = 0x9e038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = { + .cmd_rcgr = 0xf064, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_sec_phy_aux_clk_src = { + .cmd_rcgr = 0x9e064, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sec_ctrl_clk_src[] = { + F(4800000, P_BI_TCXO, 4, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sec_ctrl_clk_src = { + .cmd_rcgr = 0x3d02c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_sec_ctrl_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sec_ctrl_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div gcc_cpuss_ahb_postdiv_clk_src = { + .reg = 0x48024, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_cpuss_ahb_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_cpuss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = { + .reg = 0xf050, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_sec_mock_utmi_postdiv_clk_src = { + .reg = 0x9e050, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_sec_mock_utmi_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch gcc_pcie_clkref_en = { + .halt_reg = 0x8c004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_edp_clkref_en = { + .halt_reg = 0x8c008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_edp_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_0_axi_clk = { + .halt_reg = 0x6b080, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x6b080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_0_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_1_axi_clk = { + .halt_reg = 0x8d084, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x8d084, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_1_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_tbu_clk = { + .halt_reg = 0x90010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x90010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_center_sf_axi_clk = { + .halt_reg = 0x8d088, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d088, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_center_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_clk = { + .halt_reg = 0x770cc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x770cc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770cc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_prim_axi_clk = { + .halt_reg = 0xf080, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xf080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_sec_axi_clk = { + .halt_reg = 0x9e080, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9e080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x9e080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_sec_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_hf_axi_clk = { + .halt_reg = 0x26010, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_sf_axi_clk = { + .halt_reg = 0x2601c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x2601c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2601c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = { + .halt_reg = 0xf07c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xf07c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf07c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_sec_axi_clk = { + .halt_reg = 0x9e07c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9e07c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x9e07c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_sec_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* For CPUSS functionality the AHB clock needs to be left enabled */ +static struct clk_branch gcc_cpuss_ahb_clk = { + .halt_reg = 0x48000, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x48000, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_cpuss_ahb_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ddrss_gpu_axi_clk = { + .halt_reg = 0x71154, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x71154, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x71154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_gpu_axi_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_ddrss_pcie_sf_clk = { + .halt_reg = 0x8d080, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x8d080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_pcie_sf_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_gpll0_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_hf_axi_clk = { + .halt_reg = 0x2700c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x2700c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2700c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_sf_axi_clk = { + .halt_reg = 0x27014, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x27014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x27014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x64000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x64000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x65000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x65000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x66000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x66000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0_out_even.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_iref_en = { + .halt_reg = 0x8c014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_iref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_memnoc_gfx_clk = { + .halt_reg = 0x7100c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7100c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7100c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_memnoc_gfx_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = { + .halt_reg = 0x71018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x71018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_snoc_dvm_gfx_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_phy_rchng_clk = { + .halt_reg = 0x6b038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_phy_rchng_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie1_phy_rchng_clk = { + .halt_reg = 0x8d038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie1_phy_rchng_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_aux_clk = { + .halt_reg = 0x6b028, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_cfg_ahb_clk = { + .halt_reg = 0x6b024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_mstr_axi_clk = { + .halt_reg = 0x6b01c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_pipe_clk = { + .halt_reg = 0x6b030, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_axi_clk = { + .halt_reg = 0x6b014, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = { + .halt_reg = 0x6b010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_aux_clk = { + .halt_reg = 0x8d028, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(29), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_cfg_ahb_clk = { + .halt_reg = 0x8d024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_mstr_axi_clk = { + .halt_reg = 0x8d01c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_pipe_clk = { + .halt_reg = 0x8d030, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(30), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_axi_clk = { + .halt_reg = 0x8d014, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_q2a_axi_clk = { + .halt_reg = 0x8d010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_throttle_core_clk = { + .halt_reg = 0x90018, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x90018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x3300c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3300c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pdm2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x33004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x33004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x33004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_xo4_clk = { + .halt_reg = 0x33008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_xo4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = { + .halt_reg = 0x26008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x26008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_nrt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_rt_ahb_clk = { + .halt_reg = 0x2600c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2600c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_rt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_disp_ahb_clk = { + .halt_reg = 0x27008, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x27008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_disp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = { + .halt_reg = 0x28008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x28008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x28008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_vcodec_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_cnoc_periph_ahb_clk = { + .halt_reg = 0x4b004, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x4b004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x4b004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_cnoc_periph_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_core_clk = { + .halt_reg = 0x4b008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4b008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qspi_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_2x_clk = { + .halt_reg = 0x23008, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_clk = { + .halt_reg = 0x23000, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s0_clk = { + .halt_reg = 0x1700c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s1_clk = { + .halt_reg = 0x1713c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s2_clk = { + .halt_reg = 0x1726c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s3_clk = { + .halt_reg = 0x1739c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s4_clk = { + .halt_reg = 0x174cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s5_clk = { + .halt_reg = 0x175fc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s6_clk = { + .halt_reg = 0x1772c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s6_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s7_clk = { + .halt_reg = 0x1785c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s7_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s7_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_2x_clk = { + .halt_reg = 0x23140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_clk = { + .halt_reg = 0x23138, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s0_clk = { + .halt_reg = 0x1800c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s1_clk = { + .halt_reg = 0x1813c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s2_clk = { + .halt_reg = 0x1826c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(24), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s3_clk = { + .halt_reg = 0x1839c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s4_clk = { + .halt_reg = 0x184cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s5_clk = { + .halt_reg = 0x185fc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s6_clk = { + .halt_reg = 0x1872c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s6_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s7_clk = { + .halt_reg = 0x1885c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s7_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s7_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = { + .halt_reg = 0x17004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = { + .halt_reg = 0x17008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_m_ahb_clk = { + .halt_reg = 0x18004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x18004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = { + .halt_reg = 0x18008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x18008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ahb_clk = { + .halt_reg = 0x75004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x75004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_apps_clk = { + .halt_reg = 0x75008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x75008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ice_core_clk = { + .halt_reg = 0x75024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc1_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x14008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x14004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_ahb_clk = { + .halt_reg = 0x16008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_apps_clk = { + .halt_reg = 0x16004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc4_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* For CPUSS functionality the AHB clock needs to be left enabled */ +static struct clk_branch gcc_sys_noc_cpuss_ahb_clk = { + .halt_reg = 0x48178, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x48178, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_cpuss_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_cpuss_ahb_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_throttle_pcie_ahb_clk = { + .halt_reg = 0x9001c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9001c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_throttle_pcie_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_titan_nrt_throttle_core_clk = { + .halt_reg = 0x26024, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_titan_nrt_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_titan_rt_throttle_core_clk = { + .halt_reg = 0x26018, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_titan_rt_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_1_clkref_en = { + .halt_reg = 0x8c000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_1_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ahb_clk = { + .halt_reg = 0x77018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_clk = { + .halt_reg = 0x77010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_clk = { + .halt_reg = 0x77064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_clk = { + .halt_reg = 0x7709c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7709c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7709c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { + .halt_reg = 0x77020, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x77020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = { + .halt_reg = 0x770b8, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x770b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = { + .halt_reg = 0x7701c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x7701c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_clk = { + .halt_reg = 0x7705c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7705c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7705c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk = { + .halt_reg = 0xf010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_mock_utmi_clk = { + .halt_reg = 0xf01c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = + &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_sleep_clk = { + .halt_reg = 0xf018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_master_clk = { + .halt_reg = 0x9e010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9e010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_mock_utmi_clk = { + .halt_reg = 0x9e01c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9e01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = + &gcc_usb30_sec_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_sleep_clk = { + .halt_reg = 0x9e018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9e018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_aux_clk = { + .halt_reg = 0xf054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = { + .halt_reg = 0xf058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_com_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { + .halt_reg = 0xf05c, + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0xf05c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf05c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_lpass_clk = { + .halt_reg = 0x47020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x47020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_lpass_clk", + .ops = &clk_branch2_ops, + }, + }, +}; +static struct clk_branch gcc_mss_cfg_ahb_clk = { + .halt_reg = 0x8a000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8a000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_offline_axi_clk = { + .halt_reg = 0x8a004, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x8a004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_offline_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_snoc_axi_clk = { + .halt_reg = 0x8a154, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x8a154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_snoc_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_q6_memnoc_axi_clk = { + .halt_reg = 0x8a158, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8a158, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_q6_memnoc_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_mss_q6ss_boot_clk_src = { + .reg = 0x8a2a4, + .shift = 0, + .width = 1, + .parent_map = gcc_parent_map_15, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_q6ss_boot_clk_src", + .parent_data = gcc_parent_data_15, + .num_parents = ARRAY_SIZE(gcc_parent_data_15), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_aux_clk = { + .halt_reg = 0x9e054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9e054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_com_aux_clk = { + .halt_reg = 0x9e058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9e058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_com_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_pipe_clk = { + .halt_reg = 0x9e05c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x9e05c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x9e05c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi0_clk = { + .halt_reg = 0x2800c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x2800c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2800c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_mvp_throttle_core_clk = { + .halt_reg = 0x28010, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x28010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x28010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_mvp_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_wpss_ahb_clk = { + .halt_reg = 0x9d154, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9d154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_wpss_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_wpss_ahb_bdg_mst_clk = { + .halt_reg = 0x9d158, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9d158, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_wpss_ahb_bdg_mst_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_wpss_rscp_clk = { + .halt_reg = 0x9d16c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9d16c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_wpss_rscp_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc gcc_pcie_0_gdsc = { + .gdscr = 0x6b004, + .pd = { + .name = "gcc_pcie_0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc gcc_pcie_1_gdsc = { + .gdscr = 0x8d004, + .pd = { + .name = "gcc_pcie_1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc gcc_ufs_phy_gdsc = { + .gdscr = 0x77004, + .pd = { + .name = "gcc_ufs_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc gcc_usb30_prim_gdsc = { + .gdscr = 0xf004, + .pd = { + .name = "gcc_usb30_prim_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc gcc_usb30_sec_gdsc = { + .gdscr = 0x9e004, + .pd = { + .name = "gcc_usb30_sec_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc = { + .gdscr = 0x7d050, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc = { + .gdscr = 0x7d058, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc = { + .gdscr = 0x7d054, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_turing_mmu_tbu0_gdsc = { + .gdscr = 0x7d05c, + .pd = { + .name = "hlos1_vote_turing_mmu_tbu0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_turing_mmu_tbu1_gdsc = { + .gdscr = 0x7d060, + .pd = { + .name = "hlos1_vote_turing_mmu_tbu1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct clk_regmap *gcc_sc7280_clocks[] = { + [GCC_AGGRE_NOC_PCIE_0_AXI_CLK] = &gcc_aggre_noc_pcie_0_axi_clk.clkr, + [GCC_AGGRE_NOC_PCIE_1_AXI_CLK] = &gcc_aggre_noc_pcie_1_axi_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr, + [GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr, + [GCC_AGGRE_USB3_SEC_AXI_CLK] = &gcc_aggre_usb3_sec_axi_clk.clkr, + [GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr, + [GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr, + [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, + [GCC_CFG_NOC_USB3_SEC_AXI_CLK] = &gcc_cfg_noc_usb3_sec_axi_clk.clkr, + [GCC_CPUSS_AHB_CLK] = &gcc_cpuss_ahb_clk.clkr, + [GCC_CPUSS_AHB_CLK_SRC] = &gcc_cpuss_ahb_clk_src.clkr, + [GCC_CPUSS_AHB_POSTDIV_CLK_SRC] = &gcc_cpuss_ahb_postdiv_clk_src.clkr, + [GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr, + [GCC_DDRSS_PCIE_SF_CLK] = &gcc_ddrss_pcie_sf_clk.clkr, + [GCC_DISP_GPLL0_CLK_SRC] = &gcc_disp_gpll0_clk_src.clkr, + [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, + [GCC_DISP_SF_AXI_CLK] = &gcc_disp_sf_axi_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_GPLL0] = &gcc_gpll0.clkr, + [GCC_GPLL0_OUT_EVEN] = &gcc_gpll0_out_even.clkr, + [GCC_GPLL0_OUT_ODD] = &gcc_gpll0_out_odd.clkr, + [GCC_GPLL1] = &gcc_gpll1.clkr, + [GCC_GPLL10] = &gcc_gpll10.clkr, + [GCC_GPLL4] = &gcc_gpll4.clkr, + [GCC_GPLL9] = &gcc_gpll9.clkr, + [GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr, + [GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr, + [GCC_GPU_IREF_EN] = &gcc_gpu_iref_en.clkr, + [GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr, + [GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr, + [GCC_PCIE0_PHY_RCHNG_CLK] = &gcc_pcie0_phy_rchng_clk.clkr, + [GCC_PCIE1_PHY_RCHNG_CLK] = &gcc_pcie1_phy_rchng_clk.clkr, + [GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr, + [GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr, + [GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr, + [GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr, + [GCC_PCIE_0_PHY_RCHNG_CLK_SRC] = &gcc_pcie_0_phy_rchng_clk_src.clkr, + [GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr, + [GCC_PCIE_0_PIPE_CLK_SRC] = &gcc_pcie_0_pipe_clk_src.clkr, + [GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr, + [GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr, + [GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr, + [GCC_PCIE_1_AUX_CLK_SRC] = &gcc_pcie_1_aux_clk_src.clkr, + [GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr, + [GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr, + [GCC_PCIE_1_PHY_RCHNG_CLK_SRC] = &gcc_pcie_1_phy_rchng_clk_src.clkr, + [GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr, + [GCC_PCIE_1_PIPE_CLK_SRC] = &gcc_pcie_1_pipe_clk_src.clkr, + [GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr, + [GCC_PCIE_1_SLV_Q2A_AXI_CLK] = &gcc_pcie_1_slv_q2a_axi_clk.clkr, + [GCC_PCIE_THROTTLE_CORE_CLK] = &gcc_pcie_throttle_core_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr, + [GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr, + [GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr, + [GCC_QSPI_CNOC_PERIPH_AHB_CLK] = &gcc_qspi_cnoc_periph_ahb_clk.clkr, + [GCC_QSPI_CORE_CLK] = &gcc_qspi_core_clk.clkr, + [GCC_QSPI_CORE_CLK_SRC] = &gcc_qspi_core_clk_src.clkr, + [GCC_QUPV3_WRAP0_CORE_2X_CLK] = &gcc_qupv3_wrap0_core_2x_clk.clkr, + [GCC_QUPV3_WRAP0_CORE_CLK] = &gcc_qupv3_wrap0_core_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr, + [GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr, + [GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr, + [GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr, + [GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr, + [GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr, + [GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr, + [GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr, + [GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr, + [GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr, + [GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr, + [GCC_QUPV3_WRAP0_S6_CLK] = &gcc_qupv3_wrap0_s6_clk.clkr, + [GCC_QUPV3_WRAP0_S6_CLK_SRC] = &gcc_qupv3_wrap0_s6_clk_src.clkr, + [GCC_QUPV3_WRAP0_S7_CLK] = &gcc_qupv3_wrap0_s7_clk.clkr, + [GCC_QUPV3_WRAP0_S7_CLK_SRC] = &gcc_qupv3_wrap0_s7_clk_src.clkr, + [GCC_QUPV3_WRAP1_CORE_2X_CLK] = &gcc_qupv3_wrap1_core_2x_clk.clkr, + [GCC_QUPV3_WRAP1_CORE_CLK] = &gcc_qupv3_wrap1_core_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr, + [GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr, + [GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr, + [GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr, + [GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr, + [GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr, + [GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr, + [GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr, + [GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr, + [GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr, + [GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr, + [GCC_QUPV3_WRAP1_S6_CLK] = &gcc_qupv3_wrap1_s6_clk.clkr, + [GCC_QUPV3_WRAP1_S6_CLK_SRC] = &gcc_qupv3_wrap1_s6_clk_src.clkr, + [GCC_QUPV3_WRAP1_S7_CLK] = &gcc_qupv3_wrap1_s7_clk.clkr, + [GCC_QUPV3_WRAP1_S7_CLK_SRC] = &gcc_qupv3_wrap1_s7_clk_src.clkr, + [GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr, + [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, + [GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr, + [GCC_SDCC1_APPS_CLK_SRC] = &gcc_sdcc1_apps_clk_src.clkr, + [GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr, + [GCC_SDCC1_ICE_CORE_CLK_SRC] = &gcc_sdcc1_ice_core_clk_src.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, + [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, + [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr, + [GCC_SYS_NOC_CPUSS_AHB_CLK] = &gcc_sys_noc_cpuss_ahb_clk.clkr, + [GCC_THROTTLE_PCIE_AHB_CLK] = &gcc_throttle_pcie_ahb_clk.clkr, + [GCC_TITAN_NRT_THROTTLE_CORE_CLK] = + &gcc_titan_nrt_throttle_core_clk.clkr, + [GCC_TITAN_RT_THROTTLE_CORE_CLK] = &gcc_titan_rt_throttle_core_clk.clkr, + [GCC_UFS_1_CLKREF_EN] = &gcc_ufs_1_clkref_en.clkr, + [GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr, + [GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr, + [GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC] = + &gcc_ufs_phy_rx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC] = + &gcc_ufs_phy_rx_symbol_1_clk_src.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC] = + &gcc_ufs_phy_tx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = + &gcc_ufs_phy_unipro_core_clk_src.clkr, + [GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = + &gcc_usb30_prim_mock_utmi_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = + &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK] = &gcc_usb30_sec_master_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK_SRC] = &gcc_usb30_sec_master_clk_src.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK] = &gcc_usb30_sec_mock_utmi_clk.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK_SRC] = + &gcc_usb30_sec_mock_utmi_clk_src.clkr, + [GCC_USB30_SEC_MOCK_UTMI_POSTDIV_CLK_SRC] = + &gcc_usb30_sec_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_SEC_SLEEP_CLK] = &gcc_usb30_sec_sleep_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr, + [GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb3_prim_phy_pipe_clk_src.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK] = &gcc_usb3_sec_phy_aux_clk.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK_SRC] = &gcc_usb3_sec_phy_aux_clk_src.clkr, + [GCC_USB3_SEC_PHY_COM_AUX_CLK] = &gcc_usb3_sec_phy_com_aux_clk.clkr, + [GCC_USB3_SEC_PHY_PIPE_CLK] = &gcc_usb3_sec_phy_pipe_clk.clkr, + [GCC_USB3_SEC_PHY_PIPE_CLK_SRC] = &gcc_usb3_sec_phy_pipe_clk_src.clkr, + [GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr, + [GCC_VIDEO_MVP_THROTTLE_CORE_CLK] = + &gcc_video_mvp_throttle_core_clk.clkr, + [GCC_CFG_NOC_LPASS_CLK] = &gcc_cfg_noc_lpass_clk.clkr, + [GCC_MSS_GPLL0_MAIN_DIV_CLK_SRC] = &gcc_mss_gpll0_main_div_clk_src.clkr, + [GCC_MSS_CFG_AHB_CLK] = &gcc_mss_cfg_ahb_clk.clkr, + [GCC_MSS_OFFLINE_AXI_CLK] = &gcc_mss_offline_axi_clk.clkr, + [GCC_MSS_SNOC_AXI_CLK] = &gcc_mss_snoc_axi_clk.clkr, + [GCC_MSS_Q6_MEMNOC_AXI_CLK] = &gcc_mss_q6_memnoc_axi_clk.clkr, + [GCC_MSS_Q6SS_BOOT_CLK_SRC] = &gcc_mss_q6ss_boot_clk_src.clkr, + [GCC_AGGRE_NOC_PCIE_TBU_CLK] = &gcc_aggre_noc_pcie_tbu_clk.clkr, + [GCC_AGGRE_NOC_PCIE_CENTER_SF_AXI_CLK] = + &gcc_aggre_noc_pcie_center_sf_axi_clk.clkr, + [GCC_PCIE_CLKREF_EN] = &gcc_pcie_clkref_en.clkr, + [GCC_EDP_CLKREF_EN] = &gcc_edp_clkref_en.clkr, + [GCC_SEC_CTRL_CLK_SRC] = &gcc_sec_ctrl_clk_src.clkr, + [GCC_WPSS_AHB_CLK] = &gcc_wpss_ahb_clk.clkr, + [GCC_WPSS_AHB_BDG_MST_CLK] = &gcc_wpss_ahb_bdg_mst_clk.clkr, + [GCC_WPSS_RSCP_CLK] = &gcc_wpss_rscp_clk.clkr, +}; + +static struct gdsc *gcc_sc7280_gdscs[] = { + [GCC_PCIE_0_GDSC] = &gcc_pcie_0_gdsc, + [GCC_PCIE_1_GDSC] = &gcc_pcie_1_gdsc, + [GCC_UFS_PHY_GDSC] = &gcc_ufs_phy_gdsc, + [GCC_USB30_PRIM_GDSC] = &gcc_usb30_prim_gdsc, + [GCC_USB30_SEC_GDSC] = &gcc_usb30_sec_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_SF0_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc, + [HLOS1_VOTE_TURING_MMU_TBU0_GDSC] = &hlos1_vote_turing_mmu_tbu0_gdsc, + [HLOS1_VOTE_TURING_MMU_TBU1_GDSC] = &hlos1_vote_turing_mmu_tbu1_gdsc, +}; + +static const struct qcom_reset_map gcc_sc7280_resets[] = { + [GCC_PCIE_0_BCR] = { 0x6b000 }, + [GCC_PCIE_0_PHY_BCR] = { 0x6c01c }, + [GCC_PCIE_1_BCR] = { 0x8d000 }, + [GCC_PCIE_1_PHY_BCR] = { 0x8e01c }, + [GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 }, + [GCC_QUSB2PHY_SEC_BCR] = { 0x12004 }, + [GCC_SDCC1_BCR] = { 0x75000 }, + [GCC_SDCC2_BCR] = { 0x14000 }, + [GCC_SDCC4_BCR] = { 0x16000 }, + [GCC_UFS_PHY_BCR] = { 0x77000 }, + [GCC_USB30_PRIM_BCR] = { 0xf000 }, + [GCC_USB30_SEC_BCR] = { 0x9e000 }, + [GCC_USB3_DP_PHY_PRIM_BCR] = { 0x50008 }, + [GCC_USB3_PHY_PRIM_BCR] = { 0x50000 }, + [GCC_USB3PHY_PHY_PRIM_BCR] = { 0x50004 }, + [GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x6a000 }, +}; + +static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = { + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s6_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s7_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s6_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s7_clk_src), +}; + +static const struct regmap_config gcc_sc7280_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x9f128, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_sc7280_desc = { + .config = &gcc_sc7280_regmap_config, + .clks = gcc_sc7280_clocks, + .num_clks = ARRAY_SIZE(gcc_sc7280_clocks), + .resets = gcc_sc7280_resets, + .num_resets = ARRAY_SIZE(gcc_sc7280_resets), + .gdscs = gcc_sc7280_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_sc7280_gdscs), +}; + +static const struct of_device_id gcc_sc7280_match_table[] = { + { .compatible = "qcom,gcc-sc7280" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_sc7280_match_table); + +static int gcc_sc7280_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + int ret; + + regmap = qcom_cc_map(pdev, &gcc_sc7280_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* + * Keep the clocks always-ON + * GCC_CAMERA_AHB_CLK/XO_CLK, GCC_DISP_AHB_CLK/XO_CLK + * GCC_VIDEO_AHB_CLK/XO_CLK, GCC_GPU_CFG_AHB_CLK + */ + regmap_update_bits(regmap, 0x26004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x26028, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x27004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x2701C, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x28004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x28014, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x71004, BIT(0), BIT(0)); + + ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, + ARRAY_SIZE(gcc_dfs_clocks)); + if (ret) + return ret; + + return qcom_cc_really_probe(pdev, &gcc_sc7280_desc, regmap); +} + +static struct platform_driver gcc_sc7280_driver = { + .probe = gcc_sc7280_probe, + .driver = { + .name = "gcc-sc7280", + .of_match_table = gcc_sc7280_match_table, + }, +}; + +static int __init gcc_sc7280_init(void) +{ + return platform_driver_register(&gcc_sc7280_driver); +} +subsys_initcall(gcc_sc7280_init); + +static void __exit gcc_sc7280_exit(void) +{ + platform_driver_unregister(&gcc_sc7280_driver); +} +module_exit(gcc_sc7280_exit); + +MODULE_DESCRIPTION("QTI GCC SC7280 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/gcc-sc8180x.c b/drivers/clk/qcom/gcc-sc8180x.c new file mode 100644 index 000000000000..90525ae1bb3a --- /dev/null +++ b/drivers/clk/qcom/gcc-sc8180x.c @@ -0,0 +1,4629 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2021, Linaro Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common.h" +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_AUD_REF_CLK, + P_BI_TCXO, + P_GPLL0_OUT_EVEN, + P_GPLL0_OUT_MAIN, + P_GPLL1_OUT_MAIN, + P_GPLL2_OUT_MAIN, + P_GPLL4_OUT_MAIN, + P_GPLL5_OUT_MAIN, + P_GPLL7_OUT_MAIN, + P_GPLL9_OUT_MAIN, + P_SLEEP_CLK, +}; + +static struct pll_vco trion_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static struct clk_alpha_pll gpll0 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .vco_table = trion_vco, + .num_vco = ARRAY_SIZE(trion_vco), + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_trion_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_trion_even[] = { + { 0x0, 1 }, + { 0x1, 2 }, + { 0x3, 4 }, + { 0x7, 8 }, + { } +}; + +static struct clk_alpha_pll_postdiv gpll0_out_even = { + .offset = 0x0, + .post_div_shift = 8, + .post_div_table = post_div_table_trion_even, + .num_post_div = ARRAY_SIZE(post_div_table_trion_even), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .width = 4, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll0_out_even", + .parent_hws = (const struct clk_hw *[]){ &gpll0.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_trion_ops, + }, +}; + +static struct clk_alpha_pll gpll1 = { + .offset = 0x1000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .vco_table = trion_vco, + .num_vco = ARRAY_SIZE(trion_vco), + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gpll1", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_trion_ops, + }, + }, +}; + +static struct clk_alpha_pll gpll4 = { + .offset = 0x76000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .vco_table = trion_vco, + .num_vco = ARRAY_SIZE(trion_vco), + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_trion_ops, + }, + }, +}; + +static struct clk_alpha_pll gpll7 = { + .offset = 0x1a000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .vco_table = trion_vco, + .num_vco = ARRAY_SIZE(trion_vco), + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gpll7", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_trion_ops, + }, + }, +}; + +static const struct parent_map gcc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_0[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_SLEEP_CLK, 5 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_1[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, + { .fw_name = "sleep_clk", }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_SLEEP_CLK, 5 }, +}; + +static const struct clk_parent_data gcc_parents_2[] = { + { .fw_name = "bi_tcxo", }, + { .fw_name = "sleep_clk", }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL2_OUT_MAIN, 2 }, + { P_GPLL5_OUT_MAIN, 3 }, + { P_GPLL1_OUT_MAIN, 4 }, + { P_GPLL4_OUT_MAIN, 5 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_3[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, + { .name = "gpll2" }, + { .name = "gpll5" }, + { .hw = &gpll1.clkr.hw }, + { .hw = &gpll4.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data gcc_parents_4[] = { + { .fw_name = "bi_tcxo", }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data gcc_parents_5[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL7_OUT_MAIN, 3 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_6[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll7.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_7[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL9_OUT_MAIN, 2 }, + { P_GPLL4_OUT_MAIN, 5 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_7[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, + { .name = "gppl9" }, + { .hw = &gpll4.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_8[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_AUD_REF_CLK, 2 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_8[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, + { .name = "aud_ref_clk" }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct freq_tbl ftbl_gcc_cpuss_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_cpuss_ahb_clk_src = { + .cmd_rcgr = 0x48014, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_cpuss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_ahb_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_emac_ptp_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(125000000, P_GPLL7_OUT_MAIN, 4, 0, 0), + F(250000000, P_GPLL7_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_emac_ptp_clk_src = { + .cmd_rcgr = 0x6038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_emac_ptp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_emac_ptp_clk_src", + .parent_data = gcc_parents_6, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_emac_rgmii_clk_src[] = { + F(2500000, P_BI_TCXO, 1, 25, 192), + F(5000000, P_BI_TCXO, 1, 25, 96), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(125000000, P_GPLL7_OUT_MAIN, 4, 0, 0), + F(250000000, P_GPLL7_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_emac_rgmii_clk_src = { + .cmd_rcgr = 0x601c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_emac_rgmii_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_emac_rgmii_clk_src", + .parent_data = gcc_parents_6, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_gp1_clk_src = { + .cmd_rcgr = 0x64004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk_src", + .parent_data = gcc_parents_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp2_clk_src = { + .cmd_rcgr = 0x65004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk_src", + .parent_data = gcc_parents_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp3_clk_src = { + .cmd_rcgr = 0x66004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk_src", + .parent_data = gcc_parents_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp4_clk_src = { + .cmd_rcgr = 0xbe004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp4_clk_src", + .parent_data = gcc_parents_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp5_clk_src = { + .cmd_rcgr = 0xbf004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp5_clk_src", + .parent_data = gcc_parents_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_npu_axi_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(60000000, P_GPLL0_OUT_EVEN, 5, 0, 0), + F(150000000, P_GPLL0_OUT_EVEN, 2, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + F(403000000, P_GPLL4_OUT_MAIN, 2, 0, 0), + F(533000000, P_GPLL1_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_npu_axi_clk_src = { + .cmd_rcgr = 0x4d014, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_npu_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_npu_axi_clk_src", + .parent_data = gcc_parents_3, + .num_parents = 7, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_aux_clk_src = { + .cmd_rcgr = 0x6b02c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_aux_clk_src = { + .cmd_rcgr = 0x8d02c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_2_aux_clk_src = { + .cmd_rcgr = 0x9d02c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_3_aux_clk_src = { + .cmd_rcgr = 0xa302c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_phy_refgen_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_phy_refgen_clk_src = { + .cmd_rcgr = 0x6f014, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_phy_refgen_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_phy_refgen_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pdm2_clk_src = { + .cmd_rcgr = 0x33010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pdm2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qspi_1_core_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_qspi_1_core_clk_src = { + .cmd_rcgr = 0x4a00c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qspi_1_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_1_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qspi_core_clk_src = { + .cmd_rcgr = 0x4b008, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qspi_1_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = { + F(7372800, P_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GPLL0_OUT_EVEN, 1, 4, 25), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(64000000, P_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(102400000, P_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GPLL0_OUT_EVEN, 2.5, 0, 0), + F(128000000, P_GPLL0_OUT_MAIN, 1, 16, 75), + { } +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { + .cmd_rcgr = 0x17148, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s0_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { + .cmd_rcgr = 0x17278, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s1_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { + .cmd_rcgr = 0x173a8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s2_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { + .cmd_rcgr = 0x174d8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s3_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { + .cmd_rcgr = 0x17608, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s4_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { + .cmd_rcgr = 0x17738, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s5_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s6_clk_src = { + .cmd_rcgr = 0x17868, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s6_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s7_clk_src = { + .cmd_rcgr = 0x17998, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s7_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = { + .cmd_rcgr = 0x18148, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s0_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = { + .cmd_rcgr = 0x18278, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s1_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s2_clk_src = { + .cmd_rcgr = 0x183a8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s2_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = { + .cmd_rcgr = 0x184d8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s3_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = { + .cmd_rcgr = 0x18608, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s4_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = { + .cmd_rcgr = 0x18738, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s5_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s0_clk_src = { + .cmd_rcgr = 0x1e148, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s0_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s1_clk_src = { + .cmd_rcgr = 0x1e278, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s1_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s2_clk_src = { + .cmd_rcgr = 0x1e3a8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s2_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s3_clk_src = { + .cmd_rcgr = 0x1e4d8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s3_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s4_clk_src = { + .cmd_rcgr = 0x1e608, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s4_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s5_clk_src = { + .cmd_rcgr = 0x1e738, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s5_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GPLL0_OUT_MAIN, 12, 1, 2), + F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { + .cmd_rcgr = 0x1400c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_7, + .freq_tbl = ftbl_gcc_sdcc2_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk_src", + .parent_data = gcc_parents_7, + .num_parents = 5, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(37500000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), + F(75000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { + .cmd_rcgr = 0x1600c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_5, + .freq_tbl = ftbl_gcc_sdcc4_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk_src", + .parent_data = gcc_parents_5, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_tsif_ref_clk_src[] = { + F(105495, P_BI_TCXO, 2, 1, 91), + { } +}; + +static struct clk_rcg2 gcc_tsif_ref_clk_src = { + .cmd_rcgr = 0x36010, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_gcc_tsif_ref_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_ref_clk_src", + .parent_data = gcc_parents_8, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_2_axi_clk_src[] = { + F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_2_axi_clk_src = { + .cmd_rcgr = 0xa2020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_2_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_axi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_card_2_ice_core_clk_src = { + .cmd_rcgr = 0xa2060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_2_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_ice_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_2_phy_aux_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_2_phy_aux_clk_src = { + .cmd_rcgr = 0xa2094, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_ufs_card_2_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_phy_aux_clk_src", + .parent_data = gcc_parents_4, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_card_2_unipro_core_clk_src = { + .cmd_rcgr = 0xa2078, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_2_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_unipro_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_axi_clk_src[] = { + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_axi_clk_src = { + .cmd_rcgr = 0x75020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_ice_core_clk_src[] = { + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_ice_core_clk_src = { + .cmd_rcgr = 0x75060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_card_phy_aux_clk_src = { + .cmd_rcgr = 0x75094, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_ufs_card_2_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_clk_src", + .parent_data = gcc_parents_4, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_unipro_core_clk_src[] = { + F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0), + F(75000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_unipro_core_clk_src = { + .cmd_rcgr = 0x75078, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_unipro_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = { + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = { + .cmd_rcgr = 0x77020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = { + .cmd_rcgr = 0x77060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_2_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = { + .cmd_rcgr = 0x77094, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk_src", + .parent_data = gcc_parents_4, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = { + .cmd_rcgr = 0x77078, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_2_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_mp_master_clk_src[] = { + F(33333333, P_GPLL0_OUT_EVEN, 9, 0, 0), + F(66666667, P_GPLL0_OUT_EVEN, 4.5, 0, 0), + F(133333333, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_mp_master_clk_src = { + .cmd_rcgr = 0xa601c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mp_master_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_mp_mock_utmi_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(20000000, P_GPLL0_OUT_EVEN, 15, 0, 0), + F(40000000, P_GPLL0_OUT_EVEN, 7.5, 0, 0), + F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_mp_mock_utmi_clk_src = { + .cmd_rcgr = 0xa6034, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mp_mock_utmi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_prim_master_clk_src = { + .cmd_rcgr = 0xf01c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = { + .cmd_rcgr = 0xf034, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_sec_master_clk_src = { + .cmd_rcgr = 0x1001c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_sec_mock_utmi_clk_src = { + .cmd_rcgr = 0x10034, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_mp_phy_aux_clk_src = { + .cmd_rcgr = 0xa6068, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_card_2_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_mp_phy_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = { + .cmd_rcgr = 0xf060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_card_2_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_sec_phy_aux_clk_src = { + .cmd_rcgr = 0x10060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_card_2_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_tbu_clk = { + .halt_reg = 0x90018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x90018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_card_axi_clk = { + .halt_reg = 0x750c0, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x750c0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x750c0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_card_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_card_axi_hw_ctl_clk = { + .halt_reg = 0x750c0, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x750c0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x750c0, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_card_axi_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_aggre_ufs_card_axi_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_clk = { + .halt_reg = 0x770c0, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x770c0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770c0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_hw_ctl_clk = { + .halt_reg = 0x770c0, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x770c0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770c0, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_aggre_ufs_phy_axi_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_mp_axi_clk = { + .halt_reg = 0xa6084, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_mp_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_mp_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_prim_axi_clk = { + .halt_reg = 0xf07c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf07c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_prim_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_prim_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_sec_axi_clk = { + .halt_reg = 0x1007c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1007c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_sec_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_sec_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x38004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x38004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_boot_rom_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_hf_axi_clk = { + .halt_reg = 0xb030, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_sf_axi_clk = { + .halt_reg = 0xb034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_mp_axi_clk = { + .halt_reg = 0xa609c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa609c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_mp_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_mp_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = { + .halt_reg = 0xf078, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf078, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_prim_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_prim_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_sec_axi_clk = { + .halt_reg = 0x10078, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10078, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_sec_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_sec_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* For CPUSS functionality the AHB clock needs to be left enabled */ +static struct clk_branch gcc_cpuss_ahb_clk = { + .halt_reg = 0x48000, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_cpuss_ahb_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cpuss_rbcpr_clk = { + .halt_reg = 0x48008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x48008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_rbcpr_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ddrss_gpu_axi_clk = { + .halt_reg = 0x71154, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x71154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_gpu_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_hf_axi_clk = { + .halt_reg = 0xb038, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_sf_axi_clk = { + .halt_reg = 0xb03c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb03c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_emac_axi_clk = { + .halt_reg = 0x6010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_emac_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_emac_ptp_clk = { + .halt_reg = 0x6034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_emac_ptp_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_emac_ptp_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_emac_rgmii_clk = { + .halt_reg = 0x6018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_emac_rgmii_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_emac_rgmii_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_emac_slv_ahb_clk = { + .halt_reg = 0x6014, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x6014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_emac_slv_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x64000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x64000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_gp1_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x65000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x65000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_gp2_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x66000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x66000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_gp3_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp4_clk = { + .halt_reg = 0xbe000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xbe000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp4_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_gp4_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp5_clk = { + .halt_reg = 0xbf000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xbf000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp5_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_gp5_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_clk_src", + .parent_hws = (const struct clk_hw *[]){ &gpll0.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &gpll0_out_even.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_memnoc_gfx_clk = { + .halt_reg = 0x7100c, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x7100c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_memnoc_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = { + .halt_reg = 0x71018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x71018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_snoc_dvm_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_npu_at_clk = { + .halt_reg = 0x4d010, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x4d010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_npu_at_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_npu_axi_clk = { + .halt_reg = 0x4d008, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x4d008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_npu_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_npu_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_npu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_npu_gpll0_clk_src", + .parent_hws = (const struct clk_hw *[]){ &gpll0.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_npu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_npu_gpll0_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &gpll0_out_even.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_npu_trig_clk = { + .halt_reg = 0x4d00c, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x4d00c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_npu_trig_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_phy_refgen_clk = { + .halt_reg = 0x6f02c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6f02c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_phy_refgen_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_phy_refgen_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie1_phy_refgen_clk = { + .halt_reg = 0x6f030, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6f030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie1_phy_refgen_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_phy_refgen_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie2_phy_refgen_clk = { + .halt_reg = 0x6f034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6f034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie2_phy_refgen_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_phy_refgen_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie3_phy_refgen_clk = { + .halt_reg = 0x6f038, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6f038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie3_phy_refgen_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_phy_refgen_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_aux_clk = { + .halt_reg = 0x6b020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_0_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_cfg_ahb_clk = { + .halt_reg = 0x6b01c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_clkref_clk = { + .halt_reg = 0x8c00c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c00c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_mstr_axi_clk = { + .halt_reg = 0x6b018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_pipe_clk = { + .halt_reg = 0x6b024, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_axi_clk = { + .halt_reg = 0x6b014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = { + .halt_reg = 0x6b010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_aux_clk = { + .halt_reg = 0x8d020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(29), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_1_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_cfg_ahb_clk = { + .halt_reg = 0x8d01c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_clkref_clk = { + .halt_reg = 0x8c02c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c02c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_mstr_axi_clk = { + .halt_reg = 0x8d018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_pipe_clk = { + .halt_reg = 0x8d024, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(30), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_axi_clk = { + .halt_reg = 0x8d014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_q2a_axi_clk = { + .halt_reg = 0x8d010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_aux_clk = { + .halt_reg = 0x9d020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_2_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_cfg_ahb_clk = { + .halt_reg = 0x9d01c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9d01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_clkref_clk = { + .halt_reg = 0x8c014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_mstr_axi_clk = { + .halt_reg = 0x9d018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_pipe_clk = { + .halt_reg = 0x9d024, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_slv_axi_clk = { + .halt_reg = 0x9d014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9d014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_slv_q2a_axi_clk = { + .halt_reg = 0x9d010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_aux_clk = { + .halt_reg = 0xa3020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_3_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_cfg_ahb_clk = { + .halt_reg = 0xa301c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xa301c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_clkref_clk = { + .halt_reg = 0x8c018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_mstr_axi_clk = { + .halt_reg = 0xa3018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_pipe_clk = { + .halt_reg = 0xa3024, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_slv_axi_clk = { + .halt_reg = 0xa3014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xa3014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_slv_q2a_axi_clk = { + .halt_reg = 0xa3010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_phy_aux_clk = { + .halt_reg = 0x6f004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6f004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_0_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x3300c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3300c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pdm2_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x33004, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x33004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x33004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_xo4_clk = { + .halt_reg = 0x33008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_xo4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_prng_ahb_clk = { + .halt_reg = 0x34004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_prng_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = { + .halt_reg = 0xb018, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xb018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xb018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_nrt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_rt_ahb_clk = { + .halt_reg = 0xb01c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xb01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xb01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_rt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_disp_ahb_clk = { + .halt_reg = 0xb020, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xb020, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xb020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_disp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_cvp_ahb_clk = { + .halt_reg = 0xb010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xb010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xb010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_cvp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = { + .halt_reg = 0xb014, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xb014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xb014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_vcodec_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_1_cnoc_periph_ahb_clk = { + .halt_reg = 0x4a004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4a004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_1_cnoc_periph_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_1_core_clk = { + .halt_reg = 0x4a008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4a008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_1_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qspi_1_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_cnoc_periph_ahb_clk = { + .halt_reg = 0x4b000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4b000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_cnoc_periph_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_core_clk = { + .halt_reg = 0x4b004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4b004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qspi_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s0_clk = { + .halt_reg = 0x17144, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s0_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s0_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s1_clk = { + .halt_reg = 0x17274, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s1_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s1_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s2_clk = { + .halt_reg = 0x173a4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s2_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s3_clk = { + .halt_reg = 0x174d4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s3_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s3_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s4_clk = { + .halt_reg = 0x17604, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s4_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s4_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s5_clk = { + .halt_reg = 0x17734, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s5_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s5_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s6_clk = { + .halt_reg = 0x17864, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s6_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s6_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s7_clk = { + .halt_reg = 0x17994, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s7_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s7_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s0_clk = { + .halt_reg = 0x18144, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s0_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s0_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s1_clk = { + .halt_reg = 0x18274, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s1_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s1_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s2_clk = { + .halt_reg = 0x183a4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(24), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s2_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s3_clk = { + .halt_reg = 0x184d4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s3_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s3_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s4_clk = { + .halt_reg = 0x18604, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s4_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s4_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s5_clk = { + .halt_reg = 0x18734, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s5_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s5_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s0_clk = { + .halt_reg = 0x1e144, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s0_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s0_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s1_clk = { + .halt_reg = 0x1e274, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s1_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s1_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s2_clk = { + .halt_reg = 0x1e3a4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s2_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s3_clk = { + .halt_reg = 0x1e4d4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s3_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s3_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s4_clk = { + .halt_reg = 0x1e604, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s4_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s4_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s5_clk = { + .halt_reg = 0x1e734, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s5_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s5_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = { + .halt_reg = 0x17004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = { + .halt_reg = 0x17008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_m_ahb_clk = { + .halt_reg = 0x18004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = { + .halt_reg = 0x18008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x18008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_m_ahb_clk = { + .halt_reg = 0x1e004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_2_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_s_ahb_clk = { + .halt_reg = 0x1e008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1e008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_2_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x14008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x14004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_sdcc2_apps_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_ahb_clk = { + .halt_reg = 0x16008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_apps_clk = { + .halt_reg = 0x16004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_sdcc4_apps_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* For CPUSS functionality the SYS NOC clock needs to be left enabled */ +static struct clk_branch gcc_sys_noc_cpuss_ahb_clk = { + .halt_reg = 0x4819c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_cpuss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_cpuss_ahb_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_tsif_ahb_clk = { + .halt_reg = 0x36004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_tsif_inactivity_timers_clk = { + .halt_reg = 0x3600c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_inactivity_timers_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_tsif_ref_clk = { + .halt_reg = 0x36008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_ref_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_tsif_ref_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_ahb_clk = { + .halt_reg = 0xa2014, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xa2014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xa2014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_axi_clk = { + .halt_reg = 0xa2010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xa2010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xa2010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_2_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_ice_core_clk = { + .halt_reg = 0xa205c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xa205c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xa205c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_ice_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_2_ice_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_phy_aux_clk = { + .halt_reg = 0xa2090, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xa2090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xa2090, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_2_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_rx_symbol_0_clk = { + .halt_reg = 0xa201c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa201c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_rx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_rx_symbol_1_clk = { + .halt_reg = 0xa20ac, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa20ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_rx_symbol_1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_tx_symbol_0_clk = { + .halt_reg = 0xa2018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa2018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_tx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_unipro_core_clk = { + .halt_reg = 0xa2058, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xa2058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xa2058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_unipro_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_2_unipro_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ahb_clk = { + .halt_reg = 0x75014, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_axi_clk = { + .halt_reg = 0x75010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_axi_hw_ctl_clk = { + .halt_reg = 0x75010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_axi_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ice_core_clk = { + .halt_reg = 0x7505c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x7505c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7505c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_ice_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ice_core_hw_ctl_clk = { + .halt_reg = 0x7505c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x7505c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7505c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_ice_core_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_phy_aux_clk = { + .halt_reg = 0x75090, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75090, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_phy_aux_hw_ctl_clk = { + .halt_reg = 0x75090, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75090, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_phy_aux_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_rx_symbol_0_clk = { + .halt_reg = 0x7501c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x7501c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_rx_symbol_1_clk = { + .halt_reg = 0x750ac, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x750ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_tx_symbol_0_clk = { + .halt_reg = 0x75018, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x75018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_tx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_unipro_core_clk = { + .halt_reg = 0x75058, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_unipro_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_unipro_core_hw_ctl_clk = { + .halt_reg = 0x75058, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75058, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_unipro_core_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ahb_clk = { + .halt_reg = 0x77014, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_clk = { + .halt_reg = 0x77010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_hw_ctl_clk = { + .halt_reg = 0x77010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_axi_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_clk = { + .halt_reg = 0x7705c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x7705c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7705c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_ice_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_hw_ctl_clk = { + .halt_reg = 0x7705c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x7705c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7705c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_ice_core_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_clk = { + .halt_reg = 0x77090, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77090, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_hw_ctl_clk = { + .halt_reg = 0x77090, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77090, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_phy_aux_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { + .halt_reg = 0x7701c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x7701c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = { + .halt_reg = 0x770ac, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x770ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = { + .halt_reg = 0x77018, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x77018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_clk = { + .halt_reg = 0x77058, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_unipro_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_hw_ctl_clk = { + .halt_reg = 0x77058, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77058, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_unipro_core_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_mp_master_clk = { + .halt_reg = 0xa6010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mp_master_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_mp_master_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_mp_mock_utmi_clk = { + .halt_reg = 0xa6018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mp_mock_utmi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_mp_mock_utmi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_mp_sleep_clk = { + .halt_reg = 0xa6014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mp_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk = { + .halt_reg = 0xf010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_prim_master_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_mock_utmi_clk = { + .halt_reg = 0xf018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_prim_mock_utmi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_sleep_clk = { + .halt_reg = 0xf014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_master_clk = { + .halt_reg = 0x10010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_sec_master_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_mock_utmi_clk = { + .halt_reg = 0x10018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_sec_mock_utmi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_sleep_clk = { + .halt_reg = 0x10014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_mp_phy_aux_clk = { + .halt_reg = 0xa6050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_mp_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_mp_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_mp_phy_com_aux_clk = { + .halt_reg = 0xa6054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_mp_phy_com_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_mp_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_mp_phy_pipe_0_clk = { + .halt_reg = 0xa6058, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0xa6058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_mp_phy_pipe_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_mp_phy_pipe_1_clk = { + .halt_reg = 0xa605c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0xa605c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_mp_phy_pipe_1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_clkref_clk = { + .halt_reg = 0x8c008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_aux_clk = { + .halt_reg = 0xf050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_prim_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = { + .halt_reg = 0xf054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_com_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_prim_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { + .halt_reg = 0xf058, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0xf058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_clkref_clk = { + .halt_reg = 0x8c028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_aux_clk = { + .halt_reg = 0x10050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_sec_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_com_aux_clk = { + .halt_reg = 0x10054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_com_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_sec_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_pipe_clk = { + .halt_reg = 0x10058, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x10058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi0_clk = { + .halt_reg = 0xb024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi1_clk = { + .halt_reg = 0xb028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axic_clk = { + .halt_reg = 0xb02c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb02c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axic_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc usb30_sec_gdsc = { + .gdscr = 0x10004, + .pd = { + .name = "usb30_sec_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc emac_gdsc = { + .gdscr = 0x6004, + .pd = { + .name = "emac_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc usb30_prim_gdsc = { + .gdscr = 0xf004, + .pd = { + .name = "usb30_prim_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc pcie_0_gdsc = { + .gdscr = 0x6b004, + .pd = { + .name = "pcie_0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc ufs_card_gdsc = { + .gdscr = 0x75004, + .pd = { + .name = "ufs_card_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc ufs_phy_gdsc = { + .gdscr = 0x77004, + .pd = { + .name = "ufs_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc pcie_1_gdsc = { + .gdscr = 0x8d004, + .pd = { + .name = "pcie_1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc pcie_2_gdsc = { + .gdscr = 0x9d004, + .pd = { + .name = "pcie_2_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc ufs_card_2_gdsc = { + .gdscr = 0xa2004, + .pd = { + .name = "ufs_card_2_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc pcie_3_gdsc = { + .gdscr = 0xa3004, + .pd = { + .name = "pcie_3_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc usb30_mp_gdsc = { + .gdscr = 0xa6004, + .pd = { + .name = "usb30_mp_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct clk_regmap *gcc_sc8180x_clocks[] = { + [GCC_AGGRE_NOC_PCIE_TBU_CLK] = &gcc_aggre_noc_pcie_tbu_clk.clkr, + [GCC_AGGRE_UFS_CARD_AXI_CLK] = &gcc_aggre_ufs_card_axi_clk.clkr, + [GCC_AGGRE_UFS_CARD_AXI_HW_CTL_CLK] = &gcc_aggre_ufs_card_axi_hw_ctl_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK] = &gcc_aggre_ufs_phy_axi_hw_ctl_clk.clkr, + [GCC_AGGRE_USB3_MP_AXI_CLK] = &gcc_aggre_usb3_mp_axi_clk.clkr, + [GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr, + [GCC_AGGRE_USB3_SEC_AXI_CLK] = &gcc_aggre_usb3_sec_axi_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr, + [GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr, + [GCC_CFG_NOC_USB3_MP_AXI_CLK] = &gcc_cfg_noc_usb3_mp_axi_clk.clkr, + [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, + [GCC_CFG_NOC_USB3_SEC_AXI_CLK] = &gcc_cfg_noc_usb3_sec_axi_clk.clkr, + [GCC_CPUSS_AHB_CLK] = &gcc_cpuss_ahb_clk.clkr, + [GCC_CPUSS_AHB_CLK_SRC] = &gcc_cpuss_ahb_clk_src.clkr, + [GCC_CPUSS_RBCPR_CLK] = &gcc_cpuss_rbcpr_clk.clkr, + [GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr, + [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, + [GCC_DISP_SF_AXI_CLK] = &gcc_disp_sf_axi_clk.clkr, + [GCC_EMAC_AXI_CLK] = &gcc_emac_axi_clk.clkr, + [GCC_EMAC_PTP_CLK] = &gcc_emac_ptp_clk.clkr, + [GCC_EMAC_PTP_CLK_SRC] = &gcc_emac_ptp_clk_src.clkr, + [GCC_EMAC_RGMII_CLK] = &gcc_emac_rgmii_clk.clkr, + [GCC_EMAC_RGMII_CLK_SRC] = &gcc_emac_rgmii_clk_src.clkr, + [GCC_EMAC_SLV_AHB_CLK] = &gcc_emac_slv_ahb_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_GP4_CLK] = &gcc_gp4_clk.clkr, + [GCC_GP4_CLK_SRC] = &gcc_gp4_clk_src.clkr, + [GCC_GP5_CLK] = &gcc_gp5_clk.clkr, + [GCC_GP5_CLK_SRC] = &gcc_gp5_clk_src.clkr, + [GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr, + [GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr, + [GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr, + [GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr, + [GCC_NPU_AT_CLK] = &gcc_npu_at_clk.clkr, + [GCC_NPU_AXI_CLK] = &gcc_npu_axi_clk.clkr, + [GCC_NPU_AXI_CLK_SRC] = &gcc_npu_axi_clk_src.clkr, + [GCC_NPU_GPLL0_CLK_SRC] = &gcc_npu_gpll0_clk_src.clkr, + [GCC_NPU_GPLL0_DIV_CLK_SRC] = &gcc_npu_gpll0_div_clk_src.clkr, + [GCC_NPU_TRIG_CLK] = &gcc_npu_trig_clk.clkr, + [GCC_PCIE0_PHY_REFGEN_CLK] = &gcc_pcie0_phy_refgen_clk.clkr, + [GCC_PCIE1_PHY_REFGEN_CLK] = &gcc_pcie1_phy_refgen_clk.clkr, + [GCC_PCIE2_PHY_REFGEN_CLK] = &gcc_pcie2_phy_refgen_clk.clkr, + [GCC_PCIE3_PHY_REFGEN_CLK] = &gcc_pcie3_phy_refgen_clk.clkr, + [GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr, + [GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr, + [GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr, + [GCC_PCIE_0_CLKREF_CLK] = &gcc_pcie_0_clkref_clk.clkr, + [GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr, + [GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr, + [GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr, + [GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr, + [GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr, + [GCC_PCIE_1_AUX_CLK_SRC] = &gcc_pcie_1_aux_clk_src.clkr, + [GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr, + [GCC_PCIE_1_CLKREF_CLK] = &gcc_pcie_1_clkref_clk.clkr, + [GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr, + [GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr, + [GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr, + [GCC_PCIE_1_SLV_Q2A_AXI_CLK] = &gcc_pcie_1_slv_q2a_axi_clk.clkr, + [GCC_PCIE_2_AUX_CLK] = &gcc_pcie_2_aux_clk.clkr, + [GCC_PCIE_2_AUX_CLK_SRC] = &gcc_pcie_2_aux_clk_src.clkr, + [GCC_PCIE_2_CFG_AHB_CLK] = &gcc_pcie_2_cfg_ahb_clk.clkr, + [GCC_PCIE_2_CLKREF_CLK] = &gcc_pcie_2_clkref_clk.clkr, + [GCC_PCIE_2_MSTR_AXI_CLK] = &gcc_pcie_2_mstr_axi_clk.clkr, + [GCC_PCIE_2_PIPE_CLK] = &gcc_pcie_2_pipe_clk.clkr, + [GCC_PCIE_2_SLV_AXI_CLK] = &gcc_pcie_2_slv_axi_clk.clkr, + [GCC_PCIE_2_SLV_Q2A_AXI_CLK] = &gcc_pcie_2_slv_q2a_axi_clk.clkr, + [GCC_PCIE_3_AUX_CLK] = &gcc_pcie_3_aux_clk.clkr, + [GCC_PCIE_3_AUX_CLK_SRC] = &gcc_pcie_3_aux_clk_src.clkr, + [GCC_PCIE_3_CFG_AHB_CLK] = &gcc_pcie_3_cfg_ahb_clk.clkr, + [GCC_PCIE_3_CLKREF_CLK] = &gcc_pcie_3_clkref_clk.clkr, + [GCC_PCIE_3_MSTR_AXI_CLK] = &gcc_pcie_3_mstr_axi_clk.clkr, + [GCC_PCIE_3_PIPE_CLK] = &gcc_pcie_3_pipe_clk.clkr, + [GCC_PCIE_3_SLV_AXI_CLK] = &gcc_pcie_3_slv_axi_clk.clkr, + [GCC_PCIE_3_SLV_Q2A_AXI_CLK] = &gcc_pcie_3_slv_q2a_axi_clk.clkr, + [GCC_PCIE_PHY_AUX_CLK] = &gcc_pcie_phy_aux_clk.clkr, + [GCC_PCIE_PHY_REFGEN_CLK_SRC] = &gcc_pcie_phy_refgen_clk_src.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr, + [GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr, + [GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr, + [GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_CVP_AHB_CLK] = &gcc_qmip_video_cvp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr, + [GCC_QSPI_1_CNOC_PERIPH_AHB_CLK] = &gcc_qspi_1_cnoc_periph_ahb_clk.clkr, + [GCC_QSPI_1_CORE_CLK] = &gcc_qspi_1_core_clk.clkr, + [GCC_QSPI_1_CORE_CLK_SRC] = &gcc_qspi_1_core_clk_src.clkr, + [GCC_QSPI_CNOC_PERIPH_AHB_CLK] = &gcc_qspi_cnoc_periph_ahb_clk.clkr, + [GCC_QSPI_CORE_CLK] = &gcc_qspi_core_clk.clkr, + [GCC_QSPI_CORE_CLK_SRC] = &gcc_qspi_core_clk_src.clkr, + [GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr, + [GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr, + [GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr, + [GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr, + [GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr, + [GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr, + [GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr, + [GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr, + [GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr, + [GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr, + [GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr, + [GCC_QUPV3_WRAP0_S6_CLK] = &gcc_qupv3_wrap0_s6_clk.clkr, + [GCC_QUPV3_WRAP0_S6_CLK_SRC] = &gcc_qupv3_wrap0_s6_clk_src.clkr, + [GCC_QUPV3_WRAP0_S7_CLK] = &gcc_qupv3_wrap0_s7_clk.clkr, + [GCC_QUPV3_WRAP0_S7_CLK_SRC] = &gcc_qupv3_wrap0_s7_clk_src.clkr, + [GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr, + [GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr, + [GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr, + [GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr, + [GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr, + [GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr, + [GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr, + [GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr, + [GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr, + [GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr, + [GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr, + [GCC_QUPV3_WRAP2_S0_CLK] = &gcc_qupv3_wrap2_s0_clk.clkr, + [GCC_QUPV3_WRAP2_S0_CLK_SRC] = &gcc_qupv3_wrap2_s0_clk_src.clkr, + [GCC_QUPV3_WRAP2_S1_CLK] = &gcc_qupv3_wrap2_s1_clk.clkr, + [GCC_QUPV3_WRAP2_S1_CLK_SRC] = &gcc_qupv3_wrap2_s1_clk_src.clkr, + [GCC_QUPV3_WRAP2_S2_CLK] = &gcc_qupv3_wrap2_s2_clk.clkr, + [GCC_QUPV3_WRAP2_S2_CLK_SRC] = &gcc_qupv3_wrap2_s2_clk_src.clkr, + [GCC_QUPV3_WRAP2_S3_CLK] = &gcc_qupv3_wrap2_s3_clk.clkr, + [GCC_QUPV3_WRAP2_S3_CLK_SRC] = &gcc_qupv3_wrap2_s3_clk_src.clkr, + [GCC_QUPV3_WRAP2_S4_CLK] = &gcc_qupv3_wrap2_s4_clk.clkr, + [GCC_QUPV3_WRAP2_S4_CLK_SRC] = &gcc_qupv3_wrap2_s4_clk_src.clkr, + [GCC_QUPV3_WRAP2_S5_CLK] = &gcc_qupv3_wrap2_s5_clk.clkr, + [GCC_QUPV3_WRAP2_S5_CLK_SRC] = &gcc_qupv3_wrap2_s5_clk_src.clkr, + [GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_M_AHB_CLK] = &gcc_qupv3_wrap_2_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_S_AHB_CLK] = &gcc_qupv3_wrap_2_s_ahb_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, + [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, + [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr, + [GCC_SYS_NOC_CPUSS_AHB_CLK] = &gcc_sys_noc_cpuss_ahb_clk.clkr, + [GCC_TSIF_AHB_CLK] = &gcc_tsif_ahb_clk.clkr, + [GCC_TSIF_INACTIVITY_TIMERS_CLK] = &gcc_tsif_inactivity_timers_clk.clkr, + [GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr, + [GCC_TSIF_REF_CLK_SRC] = &gcc_tsif_ref_clk_src.clkr, + [GCC_UFS_CARD_2_AHB_CLK] = &gcc_ufs_card_2_ahb_clk.clkr, + [GCC_UFS_CARD_2_AXI_CLK] = &gcc_ufs_card_2_axi_clk.clkr, + [GCC_UFS_CARD_2_AXI_CLK_SRC] = &gcc_ufs_card_2_axi_clk_src.clkr, + [GCC_UFS_CARD_2_ICE_CORE_CLK] = &gcc_ufs_card_2_ice_core_clk.clkr, + [GCC_UFS_CARD_2_ICE_CORE_CLK_SRC] = &gcc_ufs_card_2_ice_core_clk_src.clkr, + [GCC_UFS_CARD_2_PHY_AUX_CLK] = &gcc_ufs_card_2_phy_aux_clk.clkr, + [GCC_UFS_CARD_2_PHY_AUX_CLK_SRC] = &gcc_ufs_card_2_phy_aux_clk_src.clkr, + [GCC_UFS_CARD_2_RX_SYMBOL_0_CLK] = &gcc_ufs_card_2_rx_symbol_0_clk.clkr, + [GCC_UFS_CARD_2_RX_SYMBOL_1_CLK] = &gcc_ufs_card_2_rx_symbol_1_clk.clkr, + [GCC_UFS_CARD_2_TX_SYMBOL_0_CLK] = &gcc_ufs_card_2_tx_symbol_0_clk.clkr, + [GCC_UFS_CARD_2_UNIPRO_CORE_CLK] = &gcc_ufs_card_2_unipro_core_clk.clkr, + [GCC_UFS_CARD_2_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_card_2_unipro_core_clk_src.clkr, + [GCC_UFS_CARD_AHB_CLK] = &gcc_ufs_card_ahb_clk.clkr, + [GCC_UFS_CARD_AXI_CLK] = &gcc_ufs_card_axi_clk.clkr, + [GCC_UFS_CARD_AXI_CLK_SRC] = &gcc_ufs_card_axi_clk_src.clkr, + [GCC_UFS_CARD_AXI_HW_CTL_CLK] = &gcc_ufs_card_axi_hw_ctl_clk.clkr, + [GCC_UFS_CARD_ICE_CORE_CLK] = &gcc_ufs_card_ice_core_clk.clkr, + [GCC_UFS_CARD_ICE_CORE_CLK_SRC] = &gcc_ufs_card_ice_core_clk_src.clkr, + [GCC_UFS_CARD_ICE_CORE_HW_CTL_CLK] = &gcc_ufs_card_ice_core_hw_ctl_clk.clkr, + [GCC_UFS_CARD_PHY_AUX_CLK] = &gcc_ufs_card_phy_aux_clk.clkr, + [GCC_UFS_CARD_PHY_AUX_CLK_SRC] = &gcc_ufs_card_phy_aux_clk_src.clkr, + [GCC_UFS_CARD_PHY_AUX_HW_CTL_CLK] = &gcc_ufs_card_phy_aux_hw_ctl_clk.clkr, + [GCC_UFS_CARD_RX_SYMBOL_0_CLK] = &gcc_ufs_card_rx_symbol_0_clk.clkr, + [GCC_UFS_CARD_RX_SYMBOL_1_CLK] = &gcc_ufs_card_rx_symbol_1_clk.clkr, + [GCC_UFS_CARD_TX_SYMBOL_0_CLK] = &gcc_ufs_card_tx_symbol_0_clk.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_CLK] = &gcc_ufs_card_unipro_core_clk.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_card_unipro_core_clk_src.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_HW_CTL_CLK] = &gcc_ufs_card_unipro_core_hw_ctl_clk.clkr, + [GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr, + [GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr, + [GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr, + [GCC_UFS_PHY_AXI_HW_CTL_CLK] = &gcc_ufs_phy_axi_hw_ctl_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr, + [GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK] = &gcc_ufs_phy_ice_core_hw_ctl_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr, + [GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK] = &gcc_ufs_phy_phy_aux_hw_ctl_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_phy_unipro_core_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK] = &gcc_ufs_phy_unipro_core_hw_ctl_clk.clkr, + [GCC_USB30_MP_MASTER_CLK] = &gcc_usb30_mp_master_clk.clkr, + [GCC_USB30_MP_MASTER_CLK_SRC] = &gcc_usb30_mp_master_clk_src.clkr, + [GCC_USB30_MP_MOCK_UTMI_CLK] = &gcc_usb30_mp_mock_utmi_clk.clkr, + [GCC_USB30_MP_MOCK_UTMI_CLK_SRC] = &gcc_usb30_mp_mock_utmi_clk_src.clkr, + [GCC_USB30_MP_SLEEP_CLK] = &gcc_usb30_mp_sleep_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr, + [GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK] = &gcc_usb30_sec_master_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK_SRC] = &gcc_usb30_sec_master_clk_src.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK] = &gcc_usb30_sec_mock_utmi_clk.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK_SRC] = &gcc_usb30_sec_mock_utmi_clk_src.clkr, + [GCC_USB30_SEC_SLEEP_CLK] = &gcc_usb30_sec_sleep_clk.clkr, + [GCC_USB3_MP_PHY_AUX_CLK] = &gcc_usb3_mp_phy_aux_clk.clkr, + [GCC_USB3_MP_PHY_AUX_CLK_SRC] = &gcc_usb3_mp_phy_aux_clk_src.clkr, + [GCC_USB3_MP_PHY_COM_AUX_CLK] = &gcc_usb3_mp_phy_com_aux_clk.clkr, + [GCC_USB3_MP_PHY_PIPE_0_CLK] = &gcc_usb3_mp_phy_pipe_0_clk.clkr, + [GCC_USB3_MP_PHY_PIPE_1_CLK] = &gcc_usb3_mp_phy_pipe_1_clk.clkr, + [GCC_USB3_PRIM_CLKREF_CLK] = &gcc_usb3_prim_clkref_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr, + [GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr, + [GCC_USB3_SEC_CLKREF_CLK] = &gcc_usb3_sec_clkref_clk.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK] = &gcc_usb3_sec_phy_aux_clk.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK_SRC] = &gcc_usb3_sec_phy_aux_clk_src.clkr, + [GCC_USB3_SEC_PHY_COM_AUX_CLK] = &gcc_usb3_sec_phy_com_aux_clk.clkr, + [GCC_USB3_SEC_PHY_PIPE_CLK] = &gcc_usb3_sec_phy_pipe_clk.clkr, + [GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr, + [GCC_VIDEO_AXI1_CLK] = &gcc_video_axi1_clk.clkr, + [GCC_VIDEO_AXIC_CLK] = &gcc_video_axic_clk.clkr, + [GPLL0] = &gpll0.clkr, + [GPLL0_OUT_EVEN] = &gpll0_out_even.clkr, + [GPLL1] = &gpll1.clkr, + [GPLL4] = &gpll4.clkr, + [GPLL7] = &gpll7.clkr, +}; + +static const struct qcom_reset_map gcc_sc8180x_resets[] = { + [GCC_EMAC_BCR] = { 0x6000 }, + [GCC_GPU_BCR] = { 0x71000 }, + [GCC_MMSS_BCR] = { 0xb000 }, + [GCC_NPU_BCR] = { 0x4d000 }, + [GCC_PCIE_0_BCR] = { 0x6b000 }, + [GCC_PCIE_0_PHY_BCR] = { 0x6c01c }, + [GCC_PCIE_1_BCR] = { 0x8d000 }, + [GCC_PCIE_1_PHY_BCR] = { 0x8e01c }, + [GCC_PCIE_2_BCR] = { 0x9d000 }, + [GCC_PCIE_2_PHY_BCR] = { 0xa701c }, + [GCC_PCIE_3_BCR] = { 0xa3000 }, + [GCC_PCIE_3_PHY_BCR] = { 0xa801c }, + [GCC_PCIE_PHY_BCR] = { 0x6f000 }, + [GCC_PDM_BCR] = { 0x33000 }, + [GCC_PRNG_BCR] = { 0x34000 }, + [GCC_QSPI_1_BCR] = { 0x4a000 }, + [GCC_QSPI_BCR] = { 0x24008 }, + [GCC_QUPV3_WRAPPER_0_BCR] = { 0x17000 }, + [GCC_QUPV3_WRAPPER_1_BCR] = { 0x18000 }, + [GCC_QUPV3_WRAPPER_2_BCR] = { 0x1e000 }, + [GCC_QUSB2PHY_5_BCR] = { 0x12010 }, + [GCC_QUSB2PHY_MP0_BCR] = { 0x12008 }, + [GCC_QUSB2PHY_MP1_BCR] = { 0x1200c }, + [GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 }, + [GCC_QUSB2PHY_SEC_BCR] = { 0x12004 }, + [GCC_USB3_PHY_PRIM_SP0_BCR] = { 0x50000 }, + [GCC_USB3_PHY_PRIM_SP1_BCR] = { 0x50004 }, + [GCC_USB3_DP_PHY_PRIM_SP0_BCR] = { 0x50010 }, + [GCC_USB3_DP_PHY_PRIM_SP1_BCR] = { 0x50014 }, + [GCC_USB3_PHY_SEC_BCR] = { 0x50018 }, + [GCC_USB3PHY_PHY_SEC_BCR] = { 0x5001c }, + [GCC_USB3_DP_PHY_SEC_BCR] = { 0x50020 }, + [GCC_SDCC2_BCR] = { 0x14000 }, + [GCC_SDCC4_BCR] = { 0x16000 }, + [GCC_TSIF_BCR] = { 0x36000 }, + [GCC_UFS_CARD_2_BCR] = { 0xa2000 }, + [GCC_UFS_CARD_BCR] = { 0x75000 }, + [GCC_UFS_PHY_BCR] = { 0x77000 }, + [GCC_USB30_MP_BCR] = { 0xa6000 }, + [GCC_USB30_PRIM_BCR] = { 0xf000 }, + [GCC_USB30_SEC_BCR] = { 0x10000 }, + [GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x6a000 }, + [GCC_VIDEO_AXIC_CLK_BCR] = { 0xb02c, 2 }, + [GCC_VIDEO_AXI0_CLK_BCR] = { 0xb024, 2 }, + [GCC_VIDEO_AXI1_CLK_BCR] = { 0xb028, 2 }, +}; + +static struct gdsc *gcc_sc8180x_gdscs[] = { + [EMAC_GDSC] = &emac_gdsc, + [PCIE_0_GDSC] = &pcie_0_gdsc, + [PCIE_1_GDSC] = &pcie_1_gdsc, + [PCIE_2_GDSC] = &pcie_2_gdsc, + [PCIE_3_GDSC] = &pcie_3_gdsc, + [UFS_CARD_GDSC] = &ufs_card_gdsc, + [UFS_CARD_2_GDSC] = &ufs_card_2_gdsc, + [UFS_PHY_GDSC] = &ufs_phy_gdsc, + [USB30_MP_GDSC] = &usb30_mp_gdsc, + [USB30_PRIM_GDSC] = &usb30_prim_gdsc, + [USB30_SEC_GDSC] = &usb30_sec_gdsc, +}; + +static const struct regmap_config gcc_sc8180x_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0xc0004, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_sc8180x_desc = { + .config = &gcc_sc8180x_regmap_config, + .clks = gcc_sc8180x_clocks, + .num_clks = ARRAY_SIZE(gcc_sc8180x_clocks), + .resets = gcc_sc8180x_resets, + .num_resets = ARRAY_SIZE(gcc_sc8180x_resets), + .gdscs = gcc_sc8180x_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_sc8180x_gdscs), +}; + +static const struct of_device_id gcc_sc8180x_match_table[] = { + { .compatible = "qcom,gcc-sc8180x" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_sc8180x_match_table); + +static int gcc_sc8180x_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + + regmap = qcom_cc_map(pdev, &gcc_sc8180x_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* + * Enable the following always-on clocks: + * GCC_VIDEO_AHB_CLK, GCC_CAMERA_AHB_CLK, GCC_DISP_AHB_CLK, + * GCC_VIDEO_XO_CLK, GCC_CAMERA_XO_CLK, GCC_DISP_XO_CLK, + * GCC_CPUSS_GNOC_CLK, GCC_CPUSS_DVM_BUS_CLK, GCC_NPU_CFG_AHB_CLK and + * GCC_GPU_CFG_AHB_CLK + */ + regmap_update_bits(regmap, 0xb004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xb008, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xb00c, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xb040, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xb044, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xb048, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x48004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x48190, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x4d004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x71004, BIT(0), BIT(0)); + + /* Disable the GPLL0 active input to NPU and GPU via MISC registers */ + regmap_update_bits(regmap, 0x4d110, 0x3, 0x3); + regmap_update_bits(regmap, 0x71028, 0x3, 0x3); + + return qcom_cc_really_probe(pdev, &gcc_sc8180x_desc, regmap); +} + +static struct platform_driver gcc_sc8180x_driver = { + .probe = gcc_sc8180x_probe, + .driver = { + .name = "gcc-sc8180x", + .of_match_table = gcc_sc8180x_match_table, + }, +}; + +static int __init gcc_sc8180x_init(void) +{ + return platform_driver_register(&gcc_sc8180x_driver); +} +core_initcall(gcc_sc8180x_init); + +static void __exit gcc_sc8180x_exit(void) +{ + platform_driver_unregister(&gcc_sc8180x_driver); +} +module_exit(gcc_sc8180x_exit); + +MODULE_DESCRIPTION("QTI GCC SC8180x driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/gcc-sdm660.c b/drivers/clk/qcom/gcc-sdm660.c index 31258795e7b8..6394257ca8c0 100644 --- a/drivers/clk/qcom/gcc-sdm660.c +++ b/drivers/clk/qcom/gcc-sdm660.c @@ -1571,6 +1571,7 @@ static struct clk_branch gcc_gpu_cfg_ahb_clk = { .hw.init = &(struct clk_init_data){ .name = "gcc_gpu_cfg_ahb_clk", .ops = &clk_branch2_ops, + .flags = CLK_IS_CRITICAL, }, }, }; @@ -1684,6 +1685,12 @@ static struct clk_branch gcc_mmss_noc_cfg_ahb_clk = { .hw.init = &(struct clk_init_data){ .name = "gcc_mmss_noc_cfg_ahb_clk", .ops = &clk_branch2_ops, + /* + * Any access to mmss depends on this clock. + * Gating this clock has been shown to crash the system + * when mmssnoc_axi_rpm_clk is inited in rpmcc. + */ + .flags = CLK_IS_CRITICAL, }, }, }; diff --git a/drivers/clk/qcom/gcc-sm8350.c b/drivers/clk/qcom/gcc-sm8350.c new file mode 100644 index 000000000000..1c23b9f84900 --- /dev/null +++ b/drivers/clk/qcom/gcc-sm8350.c @@ -0,0 +1,3890 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2021, Linaro Limited + */ + +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_BI_TCXO, + P_CORE_BI_PLL_TEST_SE, + P_GCC_GPLL0_OUT_EVEN, + P_GCC_GPLL0_OUT_MAIN, + P_GCC_GPLL4_OUT_MAIN, + P_GCC_GPLL9_OUT_MAIN, + P_PCIE_0_PIPE_CLK, + P_PCIE_1_PIPE_CLK, + P_SLEEP_CLK, + P_UFS_CARD_RX_SYMBOL_0_CLK, + P_UFS_CARD_RX_SYMBOL_1_CLK, + P_UFS_CARD_TX_SYMBOL_0_CLK, + P_UFS_PHY_RX_SYMBOL_0_CLK, + P_UFS_PHY_RX_SYMBOL_1_CLK, + P_UFS_PHY_TX_SYMBOL_0_CLK, + P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, + P_USB3_UNI_PHY_SEC_GCC_USB30_PIPE_CLK, +}; + +static struct clk_alpha_pll gcc_gpll0 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_5lpe_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gcc_gpll0_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gcc_gpll0_out_even = { + .offset = 0x0, + .post_div_shift = 8, + .post_div_table = post_div_table_gcc_gpll0_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0_out_even", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_5lpe_ops, + }, +}; + +static struct clk_alpha_pll gcc_gpll4 = { + .offset = 0x76000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + .name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_5lpe_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll9 = { + .offset = 0x1c000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll9", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + .name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_5lpe_ops, + }, + }, +}; + +static const struct parent_map gcc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const struct clk_parent_data gcc_parent_data_0[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, + { .fw_name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map gcc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_SLEEP_CLK, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const struct clk_parent_data gcc_parent_data_1[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .fw_name = "sleep_clk" }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, + { .fw_name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_SLEEP_CLK, 5 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const struct clk_parent_data gcc_parent_data_2[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "sleep_clk" }, + { .fw_name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const struct clk_parent_data gcc_parent_data_3[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_PCIE_0_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_4[] = { + { .fw_name = "pcie_0_pipe_clk", }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_PCIE_1_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_5[] = { + { .fw_name = "pcie_1_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL9_OUT_MAIN, 2 }, + { P_GCC_GPLL4_OUT_MAIN, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const struct clk_parent_data gcc_parent_data_6[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll9.clkr.hw }, + { .hw = &gcc_gpll4.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, + { .fw_name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map gcc_parent_map_7[] = { + { P_UFS_CARD_RX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_7[] = { + { .fw_name = "ufs_card_rx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_8[] = { + { P_UFS_CARD_RX_SYMBOL_1_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_8[] = { + { .fw_name = "ufs_card_rx_symbol_1_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_9[] = { + { P_UFS_CARD_TX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_9[] = { + { .fw_name = "ufs_card_tx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_10[] = { + { P_UFS_PHY_RX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_10[] = { + { .fw_name = "ufs_phy_rx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_11[] = { + { P_UFS_PHY_RX_SYMBOL_1_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_11[] = { + { .fw_name = "ufs_phy_rx_symbol_1_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_12[] = { + { P_UFS_PHY_TX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_12[] = { + { .fw_name = "ufs_phy_tx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_13[] = { + { P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 }, + { P_CORE_BI_PLL_TEST_SE, 1 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_13[] = { + { .fw_name = "usb3_phy_wrapper_gcc_usb30_pipe_clk" }, + { .fw_name = "core_bi_pll_test_se" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_14[] = { + { P_USB3_UNI_PHY_SEC_GCC_USB30_PIPE_CLK, 0 }, + { P_CORE_BI_PLL_TEST_SE, 1 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_14[] = { + { .fw_name = "usb3_uni_phy_sec_gcc_usb30_pipe_clk" }, + { .fw_name = "core_bi_pll_test_se" }, + { .fw_name = "bi_tcxo" }, +}; + +static struct clk_regmap_mux gcc_pcie_0_pipe_clk_src = { + .reg = 0x6b054, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_pcie_1_pipe_clk_src = { + .reg = 0x8d054, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_5, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_card_rx_symbol_0_clk_src = { + .reg = 0x75058, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_7, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_0_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_card_rx_symbol_1_clk_src = { + .reg = 0x750c8, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_8, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_1_clk_src", + .parent_data = gcc_parent_data_8, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_card_tx_symbol_0_clk_src = { + .reg = 0x75048, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_9, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_tx_symbol_0_clk_src", + .parent_data = gcc_parent_data_9, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_0_clk_src = { + .reg = 0x77058, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_10, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk_src", + .parent_data = gcc_parent_data_10, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_1_clk_src = { + .reg = 0x770c8, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_11, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk_src", + .parent_data = gcc_parent_data_11, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_tx_symbol_0_clk_src = { + .reg = 0x77048, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_12, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk_src", + .parent_data = gcc_parent_data_12, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_prim_phy_pipe_clk_src = { + .reg = 0xf060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_13, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk_src", + .parent_data = gcc_parent_data_13, + .num_parents = 3, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_sec_phy_pipe_clk_src = { + .reg = 0x10060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_14, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_pipe_clk_src", + .parent_data = gcc_parent_data_14, + .num_parents = 3, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_gp1_clk_src = { + .cmd_rcgr = 0x64004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = 5, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp2_clk_src = { + .cmd_rcgr = 0x65004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = 5, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp3_clk_src = { + .cmd_rcgr = 0x66004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = 5, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_aux_clk_src = { + .cmd_rcgr = 0x6b058, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_phy_rchng_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_phy_rchng_clk_src = { + .cmd_rcgr = 0x6b03c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_aux_clk_src = { + .cmd_rcgr = 0x8d058, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_phy_rchng_clk_src = { + .cmd_rcgr = 0x8d03c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = { + F(60000000, P_GCC_GPLL0_OUT_MAIN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pdm2_clk_src = { + .cmd_rcgr = 0x33010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pdm2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = { + .name = "gcc_qupv3_wrap0_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { + .cmd_rcgr = 0x17010, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = { + .name = "gcc_qupv3_wrap0_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { + .cmd_rcgr = 0x17140, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = { + .name = "gcc_qupv3_wrap0_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { + .cmd_rcgr = 0x17270, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = { + .name = "gcc_qupv3_wrap0_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { + .cmd_rcgr = 0x173a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = { + .name = "gcc_qupv3_wrap0_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { + .cmd_rcgr = 0x174d0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = { + .name = "gcc_qupv3_wrap0_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { + .cmd_rcgr = 0x17600, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s6_clk_src_init = { + .name = "gcc_qupv3_wrap0_s6_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s6_clk_src = { + .cmd_rcgr = 0x17730, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s6_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s7_clk_src_init = { + .name = "gcc_qupv3_wrap0_s7_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s7_clk_src = { + .cmd_rcgr = 0x17860, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s7_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s0_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = { + .name = "gcc_qupv3_wrap1_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = { + .cmd_rcgr = 0x18010, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = { + .name = "gcc_qupv3_wrap1_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = { + .cmd_rcgr = 0x18140, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s2_clk_src_init = { + .name = "gcc_qupv3_wrap1_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s2_clk_src = { + .cmd_rcgr = 0x18270, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = { + .name = "gcc_qupv3_wrap1_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = { + .cmd_rcgr = 0x183a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = { + .name = "gcc_qupv3_wrap1_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = { + .cmd_rcgr = 0x184d0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = { + .name = "gcc_qupv3_wrap1_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = { + .cmd_rcgr = 0x18600, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s0_clk_src_init = { + .name = "gcc_qupv3_wrap2_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s0_clk_src = { + .cmd_rcgr = 0x1e010, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s1_clk_src_init = { + .name = "gcc_qupv3_wrap2_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s1_clk_src = { + .cmd_rcgr = 0x1e140, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s2_clk_src_init = { + .name = "gcc_qupv3_wrap2_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s2_clk_src = { + .cmd_rcgr = 0x1e270, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s3_clk_src_init = { + .name = "gcc_qupv3_wrap2_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s3_clk_src = { + .cmd_rcgr = 0x1e3a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s4_clk_src_init = { + .name = "gcc_qupv3_wrap2_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s4_clk_src = { + .cmd_rcgr = 0x1e4d0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s5_clk_src_init = { + .name = "gcc_qupv3_wrap2_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s5_clk_src = { + .cmd_rcgr = 0x1e600, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s5_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(202000000, P_GCC_GPLL9_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { + .cmd_rcgr = 0x1400c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_sdcc2_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = 6, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { + .cmd_rcgr = 0x1600c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_sdcc4_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_axi_clk_src[] = { + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_axi_clk_src = { + .cmd_rcgr = 0x75024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_ice_core_clk_src[] = { + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_ice_core_clk_src = { + .cmd_rcgr = 0x7506c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_phy_aux_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_phy_aux_clk_src = { + .cmd_rcgr = 0x750a0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_ufs_card_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_card_unipro_core_clk_src = { + .cmd_rcgr = 0x75084, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = { + .cmd_rcgr = 0x77024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = { + .cmd_rcgr = 0x7706c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = { + .cmd_rcgr = 0x770a0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = { + .cmd_rcgr = 0x77084, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = { + F(66666667, P_GCC_GPLL0_OUT_EVEN, 4.5, 0, 0), + F(133333333, P_GCC_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0), + F(240000000, P_GCC_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_prim_master_clk_src = { + .cmd_rcgr = 0xf020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_prim_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = { + .cmd_rcgr = 0xf038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_sec_master_clk_src = { + .cmd_rcgr = 0x10020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_prim_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_sec_mock_utmi_clk_src = { + .cmd_rcgr = 0x10038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = { + .cmd_rcgr = 0xf064, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_card_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_sec_phy_aux_clk_src = { + .cmd_rcgr = 0x10064, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_card_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = { + .reg = 0xf050, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_sec_mock_utmi_postdiv_clk_src = { + .reg = 0x10050, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_sec_mock_utmi_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_aggre_noc_pcie_0_axi_clk = { + .halt_reg = 0x6b080, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_0_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_aggre_noc_pcie_1_axi_clk = { + .halt_reg = 0x8d084, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_1_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_tbu_clk = { + .halt_reg = 0x9000c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9000c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_card_axi_clk = { + .halt_reg = 0x750cc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x750cc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x750cc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_card_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_card_axi_hw_ctl_clk = { + .halt_reg = 0x750cc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x750cc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x750cc, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_card_axi_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_clk = { + .halt_reg = 0x770cc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x770cc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770cc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_hw_ctl_clk = { + .halt_reg = 0x770cc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x770cc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770cc, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_prim_axi_clk = { + .halt_reg = 0xf080, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xf080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_sec_axi_clk = { + .halt_reg = 0x10080, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x10080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x10080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_sec_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x38004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x38004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_boot_rom_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_camera_hf_axi_clk = { + .halt_reg = 0x26010, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_camera_sf_axi_clk = { + .halt_reg = 0x26014, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = { + .halt_reg = 0xf07c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xf07c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf07c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_sec_axi_clk = { + .halt_reg = 0x1007c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1007c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1007c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_sec_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_ddrss_gpu_axi_clk = { + .halt_reg = 0x71154, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x71154, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x71154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_gpu_axi_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_ddrss_pcie_sf_tbu_clk = { + .halt_reg = 0x8d080, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x8d080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_pcie_sf_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_disp_hf_axi_clk = { + .halt_reg = 0x2700c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x2700c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2700c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_disp_sf_axi_clk = { + .halt_reg = 0x27014, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x27014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x27014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x64000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x64000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x65000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x65000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x66000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x66000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_gpu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_gpu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0_out_even.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_iref_en = { + .halt_reg = 0x8c014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_iref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_memnoc_gfx_clk = { + .halt_reg = 0x7100c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7100c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7100c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_memnoc_gfx_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = { + .halt_reg = 0x71018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x71018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_snoc_dvm_gfx_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_phy_rchng_clk = { + .halt_reg = 0x6b038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_phy_rchng_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie1_phy_rchng_clk = { + .halt_reg = 0x8d038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie1_phy_rchng_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_aux_clk = { + .halt_reg = 0x6b028, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_cfg_ahb_clk = { + .halt_reg = 0x6b024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_clkref_en = { + .halt_reg = 0x8c004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_pcie_0_mstr_axi_clk = { + .halt_reg = 0x6b01c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x6b01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_pcie_0_pipe_clk = { + .halt_reg = 0x6b030, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_axi_clk = { + .halt_reg = 0x6b014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = { + .halt_reg = 0x6b010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_aux_clk = { + .halt_reg = 0x8d028, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(29), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_cfg_ahb_clk = { + .halt_reg = 0x8d024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_clkref_en = { + .halt_reg = 0x8c008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_pcie_1_mstr_axi_clk = { + .halt_reg = 0x8d01c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x8d01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_pcie_1_pipe_clk = { + .halt_reg = 0x8d030, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(30), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_axi_clk = { + .halt_reg = 0x8d014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_q2a_axi_clk = { + .halt_reg = 0x8d010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x3300c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3300c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pdm2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x33004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x33004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x33004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_xo4_clk = { + .halt_reg = 0x33008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_xo4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = { + .halt_reg = 0x26008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x26008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_nrt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_rt_ahb_clk = { + .halt_reg = 0x2600c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2600c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_rt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_disp_ahb_clk = { + .halt_reg = 0x27008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x27008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x27008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_disp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_cvp_ahb_clk = { + .halt_reg = 0x28008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x28008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x28008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_cvp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = { + .halt_reg = 0x2800c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2800c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2800c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_vcodec_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_2x_clk = { + .halt_reg = 0x23008, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_clk = { + .halt_reg = 0x23000, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s0_clk = { + .halt_reg = 0x1700c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s1_clk = { + .halt_reg = 0x1713c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s2_clk = { + .halt_reg = 0x1726c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s3_clk = { + .halt_reg = 0x1739c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s4_clk = { + .halt_reg = 0x174cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s5_clk = { + .halt_reg = 0x175fc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s6_clk = { + .halt_reg = 0x1772c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s6_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s7_clk = { + .halt_reg = 0x1785c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s7_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s7_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_2x_clk = { + .halt_reg = 0x23140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_clk = { + .halt_reg = 0x23138, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_m_ahb_clk = { + .halt_reg = 0x18004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x18004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = { + .halt_reg = 0x18008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x18008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s0_clk = { + .halt_reg = 0x1800c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s1_clk = { + .halt_reg = 0x1813c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s2_clk = { + .halt_reg = 0x1826c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(24), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s3_clk = { + .halt_reg = 0x1839c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s4_clk = { + .halt_reg = 0x184cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s5_clk = { + .halt_reg = 0x185fc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_core_2x_clk = { + .halt_reg = 0x23278, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_core_clk = { + .halt_reg = 0x23270, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s0_clk = { + .halt_reg = 0x1e00c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s1_clk = { + .halt_reg = 0x1e13c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s2_clk = { + .halt_reg = 0x1e26c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s3_clk = { + .halt_reg = 0x1e39c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s4_clk = { + .halt_reg = 0x1e4cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s5_clk = { + .halt_reg = 0x1e5fc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = { + .halt_reg = 0x17004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = { + .halt_reg = 0x17008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_m_ahb_clk = { + .halt_reg = 0x1e004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1e004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_2_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_s_ahb_clk = { + .halt_reg = 0x1e008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1e008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_2_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x14008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x14004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_ahb_clk = { + .halt_reg = 0x16008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_apps_clk = { + .halt_reg = 0x16004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc4_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_throttle_pcie_ahb_clk = { + .halt_reg = 0x9044, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_throttle_pcie_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_1_clkref_en = { + .halt_reg = 0x8c000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_1_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ahb_clk = { + .halt_reg = 0x75018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_axi_clk = { + .halt_reg = 0x75010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_axi_hw_ctl_clk = { + .halt_reg = 0x75010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ice_core_clk = { + .halt_reg = 0x75064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ice_core_hw_ctl_clk = { + .halt_reg = 0x75064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75064, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_phy_aux_clk = { + .halt_reg = 0x7509c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7509c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7509c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_phy_aux_hw_ctl_clk = { + .halt_reg = 0x7509c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7509c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7509c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_card_rx_symbol_0_clk = { + .halt_reg = 0x75020, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x75020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_rx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_card_rx_symbol_1_clk = { + .halt_reg = 0x750b8, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x750b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_rx_symbol_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_card_tx_symbol_0_clk = { + .halt_reg = 0x7501c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x7501c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_tx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_tx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_unipro_core_clk = { + .halt_reg = 0x7505c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7505c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7505c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_unipro_core_hw_ctl_clk = { + .halt_reg = 0x7505c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7505c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7505c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ahb_clk = { + .halt_reg = 0x77018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_clk = { + .halt_reg = 0x77010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_hw_ctl_clk = { + .halt_reg = 0x77010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_clk = { + .halt_reg = 0x77064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_hw_ctl_clk = { + .halt_reg = 0x77064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77064, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_clk = { + .halt_reg = 0x7709c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7709c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7709c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_hw_ctl_clk = { + .halt_reg = 0x7709c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7709c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7709c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { + .halt_reg = 0x77020, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x77020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = { + .halt_reg = 0x770b8, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x770b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = { + .halt_reg = 0x7701c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x7701c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_clk = { + .halt_reg = 0x7705c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7705c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7705c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_hw_ctl_clk = { + .halt_reg = 0x7705c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7705c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7705c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk = { + .halt_reg = 0xf010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk__force_mem_core_on = { + .halt_reg = 0xf010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf010, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk__force_mem_core_on", + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_mock_utmi_clk = { + .halt_reg = 0xf01c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = + &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_sleep_clk = { + .halt_reg = 0xf018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_master_clk = { + .halt_reg = 0x10010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_master_clk__force_mem_core_on = { + .halt_reg = 0x10010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10010, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk__force_mem_core_on", + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_mock_utmi_clk = { + .halt_reg = 0x1001c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1001c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = + &gcc_usb30_sec_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_sleep_clk = { + .halt_reg = 0x10018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_aux_clk = { + .halt_reg = 0xf054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = { + .halt_reg = 0xf058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_com_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { + .halt_reg = 0xf05c, + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0xf05c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf05c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_clkref_en = { + .halt_reg = 0x8c010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_aux_clk = { + .halt_reg = 0x10054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_com_aux_clk = { + .halt_reg = 0x10058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_com_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_usb3_sec_phy_pipe_clk = { + .halt_reg = 0x1005c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x1005c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_video_axi0_clk = { + .halt_reg = 0x28010, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x28010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x28010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_video_axi1_clk = { + .halt_reg = 0x28018, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x28018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x28018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc pcie_0_gdsc = { + .gdscr = 0x6b004, + .pd = { + .name = "pcie_0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc pcie_1_gdsc = { + .gdscr = 0x8d004, + .pd = { + .name = "pcie_1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc ufs_card_gdsc = { + .gdscr = 0x75004, + .pd = { + .name = "ufs_card_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc ufs_phy_gdsc = { + .gdscr = 0x77004, + .pd = { + .name = "ufs_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc usb30_prim_gdsc = { + .gdscr = 0xf004, + .pd = { + .name = "usb30_prim_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc usb30_sec_gdsc = { + .gdscr = 0x10004, + .pd = { + .name = "usb30_sec_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc = { + .gdscr = 0x7d050, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc = { + .gdscr = 0x7d058, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc = { + .gdscr = 0x7d054, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_sf1_gdsc = { + .gdscr = 0x7d06c, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_sf1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct clk_regmap *gcc_sm8350_clocks[] = { + [GCC_AGGRE_NOC_PCIE_0_AXI_CLK] = &gcc_aggre_noc_pcie_0_axi_clk.clkr, + [GCC_AGGRE_NOC_PCIE_1_AXI_CLK] = &gcc_aggre_noc_pcie_1_axi_clk.clkr, + [GCC_AGGRE_NOC_PCIE_TBU_CLK] = &gcc_aggre_noc_pcie_tbu_clk.clkr, + [GCC_AGGRE_UFS_CARD_AXI_CLK] = &gcc_aggre_ufs_card_axi_clk.clkr, + [GCC_AGGRE_UFS_CARD_AXI_HW_CTL_CLK] = &gcc_aggre_ufs_card_axi_hw_ctl_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK] = &gcc_aggre_ufs_phy_axi_hw_ctl_clk.clkr, + [GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr, + [GCC_AGGRE_USB3_SEC_AXI_CLK] = &gcc_aggre_usb3_sec_axi_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr, + [GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr, + [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, + [GCC_CFG_NOC_USB3_SEC_AXI_CLK] = &gcc_cfg_noc_usb3_sec_axi_clk.clkr, + [GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr, + [GCC_DDRSS_PCIE_SF_TBU_CLK] = &gcc_ddrss_pcie_sf_tbu_clk.clkr, + [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, + [GCC_DISP_SF_AXI_CLK] = &gcc_disp_sf_axi_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_GPLL0] = &gcc_gpll0.clkr, + [GCC_GPLL0_OUT_EVEN] = &gcc_gpll0_out_even.clkr, + [GCC_GPLL4] = &gcc_gpll4.clkr, + [GCC_GPLL9] = &gcc_gpll9.clkr, + [GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr, + [GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr, + [GCC_GPU_IREF_EN] = &gcc_gpu_iref_en.clkr, + [GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr, + [GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr, + [GCC_PCIE0_PHY_RCHNG_CLK] = &gcc_pcie0_phy_rchng_clk.clkr, + [GCC_PCIE1_PHY_RCHNG_CLK] = &gcc_pcie1_phy_rchng_clk.clkr, + [GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr, + [GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr, + [GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr, + [GCC_PCIE_0_CLKREF_EN] = &gcc_pcie_0_clkref_en.clkr, + [GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr, + [GCC_PCIE_0_PHY_RCHNG_CLK_SRC] = &gcc_pcie_0_phy_rchng_clk_src.clkr, + [GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr, + [GCC_PCIE_0_PIPE_CLK_SRC] = &gcc_pcie_0_pipe_clk_src.clkr, + [GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr, + [GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr, + [GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr, + [GCC_PCIE_1_AUX_CLK_SRC] = &gcc_pcie_1_aux_clk_src.clkr, + [GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr, + [GCC_PCIE_1_CLKREF_EN] = &gcc_pcie_1_clkref_en.clkr, + [GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr, + [GCC_PCIE_1_PHY_RCHNG_CLK_SRC] = &gcc_pcie_1_phy_rchng_clk_src.clkr, + [GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr, + [GCC_PCIE_1_PIPE_CLK_SRC] = &gcc_pcie_1_pipe_clk_src.clkr, + [GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr, + [GCC_PCIE_1_SLV_Q2A_AXI_CLK] = &gcc_pcie_1_slv_q2a_axi_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr, + [GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr, + [GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_CVP_AHB_CLK] = &gcc_qmip_video_cvp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr, + [GCC_QUPV3_WRAP0_CORE_2X_CLK] = &gcc_qupv3_wrap0_core_2x_clk.clkr, + [GCC_QUPV3_WRAP0_CORE_CLK] = &gcc_qupv3_wrap0_core_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr, + [GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr, + [GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr, + [GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr, + [GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr, + [GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr, + [GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr, + [GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr, + [GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr, + [GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr, + [GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr, + [GCC_QUPV3_WRAP0_S6_CLK] = &gcc_qupv3_wrap0_s6_clk.clkr, + [GCC_QUPV3_WRAP0_S6_CLK_SRC] = &gcc_qupv3_wrap0_s6_clk_src.clkr, + [GCC_QUPV3_WRAP0_S7_CLK] = &gcc_qupv3_wrap0_s7_clk.clkr, + [GCC_QUPV3_WRAP0_S7_CLK_SRC] = &gcc_qupv3_wrap0_s7_clk_src.clkr, + [GCC_QUPV3_WRAP1_CORE_2X_CLK] = &gcc_qupv3_wrap1_core_2x_clk.clkr, + [GCC_QUPV3_WRAP1_CORE_CLK] = &gcc_qupv3_wrap1_core_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr, + [GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr, + [GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr, + [GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr, + [GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr, + [GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr, + [GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr, + [GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr, + [GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr, + [GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr, + [GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr, + [GCC_QUPV3_WRAP2_CORE_2X_CLK] = &gcc_qupv3_wrap2_core_2x_clk.clkr, + [GCC_QUPV3_WRAP2_CORE_CLK] = &gcc_qupv3_wrap2_core_clk.clkr, + [GCC_QUPV3_WRAP2_S0_CLK] = &gcc_qupv3_wrap2_s0_clk.clkr, + [GCC_QUPV3_WRAP2_S0_CLK_SRC] = &gcc_qupv3_wrap2_s0_clk_src.clkr, + [GCC_QUPV3_WRAP2_S1_CLK] = &gcc_qupv3_wrap2_s1_clk.clkr, + [GCC_QUPV3_WRAP2_S1_CLK_SRC] = &gcc_qupv3_wrap2_s1_clk_src.clkr, + [GCC_QUPV3_WRAP2_S2_CLK] = &gcc_qupv3_wrap2_s2_clk.clkr, + [GCC_QUPV3_WRAP2_S2_CLK_SRC] = &gcc_qupv3_wrap2_s2_clk_src.clkr, + [GCC_QUPV3_WRAP2_S3_CLK] = &gcc_qupv3_wrap2_s3_clk.clkr, + [GCC_QUPV3_WRAP2_S3_CLK_SRC] = &gcc_qupv3_wrap2_s3_clk_src.clkr, + [GCC_QUPV3_WRAP2_S4_CLK] = &gcc_qupv3_wrap2_s4_clk.clkr, + [GCC_QUPV3_WRAP2_S4_CLK_SRC] = &gcc_qupv3_wrap2_s4_clk_src.clkr, + [GCC_QUPV3_WRAP2_S5_CLK] = &gcc_qupv3_wrap2_s5_clk.clkr, + [GCC_QUPV3_WRAP2_S5_CLK_SRC] = &gcc_qupv3_wrap2_s5_clk_src.clkr, + [GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_M_AHB_CLK] = &gcc_qupv3_wrap_2_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_S_AHB_CLK] = &gcc_qupv3_wrap_2_s_ahb_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, + [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, + [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr, + [GCC_THROTTLE_PCIE_AHB_CLK] = &gcc_throttle_pcie_ahb_clk.clkr, + [GCC_UFS_1_CLKREF_EN] = &gcc_ufs_1_clkref_en.clkr, + [GCC_UFS_CARD_AHB_CLK] = &gcc_ufs_card_ahb_clk.clkr, + [GCC_UFS_CARD_AXI_CLK] = &gcc_ufs_card_axi_clk.clkr, + [GCC_UFS_CARD_AXI_CLK_SRC] = &gcc_ufs_card_axi_clk_src.clkr, + [GCC_UFS_CARD_AXI_HW_CTL_CLK] = &gcc_ufs_card_axi_hw_ctl_clk.clkr, + [GCC_UFS_CARD_ICE_CORE_CLK] = &gcc_ufs_card_ice_core_clk.clkr, + [GCC_UFS_CARD_ICE_CORE_CLK_SRC] = &gcc_ufs_card_ice_core_clk_src.clkr, + [GCC_UFS_CARD_ICE_CORE_HW_CTL_CLK] = &gcc_ufs_card_ice_core_hw_ctl_clk.clkr, + [GCC_UFS_CARD_PHY_AUX_CLK] = &gcc_ufs_card_phy_aux_clk.clkr, + [GCC_UFS_CARD_PHY_AUX_CLK_SRC] = &gcc_ufs_card_phy_aux_clk_src.clkr, + [GCC_UFS_CARD_PHY_AUX_HW_CTL_CLK] = &gcc_ufs_card_phy_aux_hw_ctl_clk.clkr, + [GCC_UFS_CARD_RX_SYMBOL_0_CLK] = &gcc_ufs_card_rx_symbol_0_clk.clkr, + [GCC_UFS_CARD_RX_SYMBOL_0_CLK_SRC] = &gcc_ufs_card_rx_symbol_0_clk_src.clkr, + [GCC_UFS_CARD_RX_SYMBOL_1_CLK] = &gcc_ufs_card_rx_symbol_1_clk.clkr, + [GCC_UFS_CARD_RX_SYMBOL_1_CLK_SRC] = &gcc_ufs_card_rx_symbol_1_clk_src.clkr, + [GCC_UFS_CARD_TX_SYMBOL_0_CLK] = &gcc_ufs_card_tx_symbol_0_clk.clkr, + [GCC_UFS_CARD_TX_SYMBOL_0_CLK_SRC] = &gcc_ufs_card_tx_symbol_0_clk_src.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_CLK] = &gcc_ufs_card_unipro_core_clk.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_card_unipro_core_clk_src.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_HW_CTL_CLK] = &gcc_ufs_card_unipro_core_hw_ctl_clk.clkr, + [GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr, + [GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr, + [GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr, + [GCC_UFS_PHY_AXI_HW_CTL_CLK] = &gcc_ufs_phy_axi_hw_ctl_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr, + [GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK] = &gcc_ufs_phy_ice_core_hw_ctl_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr, + [GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK] = &gcc_ufs_phy_phy_aux_hw_ctl_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC] = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_phy_unipro_core_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK] = &gcc_ufs_phy_unipro_core_hw_ctl_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK__FORCE_MEM_CORE_ON] = + &gcc_usb30_prim_master_clk__force_mem_core_on.clkr, + [GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK] = &gcc_usb30_sec_master_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK__FORCE_MEM_CORE_ON] = + &gcc_usb30_sec_master_clk__force_mem_core_on.clkr, + [GCC_USB30_SEC_MASTER_CLK_SRC] = &gcc_usb30_sec_master_clk_src.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK] = &gcc_usb30_sec_mock_utmi_clk.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK_SRC] = &gcc_usb30_sec_mock_utmi_clk_src.clkr, + [GCC_USB30_SEC_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_sec_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_SEC_SLEEP_CLK] = &gcc_usb30_sec_sleep_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr, + [GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb3_prim_phy_pipe_clk_src.clkr, + [GCC_USB3_SEC_CLKREF_EN] = &gcc_usb3_sec_clkref_en.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK] = &gcc_usb3_sec_phy_aux_clk.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK_SRC] = &gcc_usb3_sec_phy_aux_clk_src.clkr, + [GCC_USB3_SEC_PHY_COM_AUX_CLK] = &gcc_usb3_sec_phy_com_aux_clk.clkr, + [GCC_USB3_SEC_PHY_PIPE_CLK] = &gcc_usb3_sec_phy_pipe_clk.clkr, + [GCC_USB3_SEC_PHY_PIPE_CLK_SRC] = &gcc_usb3_sec_phy_pipe_clk_src.clkr, + [GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr, + [GCC_VIDEO_AXI1_CLK] = &gcc_video_axi1_clk.clkr, +}; + +static struct gdsc *gcc_sm8350_gdscs[] = { + [PCIE_0_GDSC] = &pcie_0_gdsc, + [PCIE_1_GDSC] = &pcie_1_gdsc, + [UFS_CARD_GDSC] = &ufs_card_gdsc, + [UFS_PHY_GDSC] = &ufs_phy_gdsc, + [USB30_PRIM_GDSC] = &usb30_prim_gdsc, + [USB30_SEC_GDSC] = &usb30_sec_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_SF0_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_SF1_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_sf1_gdsc, +}; + +static const struct qcom_reset_map gcc_sm8350_resets[] = { + [GCC_CAMERA_BCR] = { 0x26000 }, + [GCC_DISPLAY_BCR] = { 0x27000 }, + [GCC_GPU_BCR] = { 0x71000 }, + [GCC_MMSS_BCR] = { 0xb000 }, + [GCC_PCIE_0_BCR] = { 0x6b000 }, + [GCC_PCIE_0_LINK_DOWN_BCR] = { 0x6c014 }, + [GCC_PCIE_0_NOCSR_COM_PHY_BCR] = { 0x6c020 }, + [GCC_PCIE_0_PHY_BCR] = { 0x6c01c }, + [GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR] = { 0x6c028 }, + [GCC_PCIE_1_BCR] = { 0x8d000 }, + [GCC_PCIE_1_LINK_DOWN_BCR] = { 0x8e014 }, + [GCC_PCIE_1_NOCSR_COM_PHY_BCR] = { 0x8e020 }, + [GCC_PCIE_1_PHY_BCR] = { 0x8e01c }, + [GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR] = { 0x8e000 }, + [GCC_PCIE_PHY_CFG_AHB_BCR] = { 0x6f00c }, + [GCC_PCIE_PHY_COM_BCR] = { 0x6f010 }, + [GCC_PDM_BCR] = { 0x33000 }, + [GCC_QUPV3_WRAPPER_0_BCR] = { 0x17000 }, + [GCC_QUPV3_WRAPPER_1_BCR] = { 0x18000 }, + [GCC_QUPV3_WRAPPER_2_BCR] = { 0x1e000 }, + [GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 }, + [GCC_QUSB2PHY_SEC_BCR] = { 0x12004 }, + [GCC_SDCC2_BCR] = { 0x14000 }, + [GCC_SDCC4_BCR] = { 0x16000 }, + [GCC_UFS_CARD_BCR] = { 0x75000 }, + [GCC_UFS_PHY_BCR] = { 0x77000 }, + [GCC_USB30_PRIM_BCR] = { 0xf000 }, + [GCC_USB30_SEC_BCR] = { 0x10000 }, + [GCC_USB3_DP_PHY_PRIM_BCR] = { 0x50008 }, + [GCC_USB3_DP_PHY_SEC_BCR] = { 0x50014 }, + [GCC_USB3_PHY_PRIM_BCR] = { 0x50000 }, + [GCC_USB3_PHY_SEC_BCR] = { 0x5000c }, + [GCC_USB3PHY_PHY_PRIM_BCR] = { 0x50004 }, + [GCC_USB3PHY_PHY_SEC_BCR] = { 0x50010 }, + [GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x6a000 }, + [GCC_VIDEO_AXI0_CLK_ARES] = { 0x28010, 2 }, + [GCC_VIDEO_AXI1_CLK_ARES] = { 0x28018, 2 }, + [GCC_VIDEO_BCR] = { 0x28000 }, +}; + +static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = { + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s6_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s7_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s5_clk_src), +}; + +static const struct regmap_config gcc_sm8350_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x9c100, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_sm8350_desc = { + .config = &gcc_sm8350_regmap_config, + .clks = gcc_sm8350_clocks, + .num_clks = ARRAY_SIZE(gcc_sm8350_clocks), + .resets = gcc_sm8350_resets, + .num_resets = ARRAY_SIZE(gcc_sm8350_resets), + .gdscs = gcc_sm8350_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_sm8350_gdscs), +}; + +static const struct of_device_id gcc_sm8350_match_table[] = { + { .compatible = "qcom,gcc-sm8350" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_sm8350_match_table); + +static int gcc_sm8350_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + int ret; + + regmap = qcom_cc_map(pdev, &gcc_sm8350_desc); + if (IS_ERR(regmap)) { + dev_err(&pdev->dev, "Failed to map gcc registers\n"); + return PTR_ERR(regmap); + } + + /* + * Keep the critical clock always-On + * GCC_CAMERA_AHB_CLK, GCC_CAMERA_XO_CLK, GCC_DISP_AHB_CLK, GCC_DISP_XO_CLK, + * GCC_GPU_CFG_AHB_CLK, GCC_VIDEO_AHB_CLK, GCC_VIDEO_XO_CLK + */ + regmap_update_bits(regmap, 0x26004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x26018, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x27004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x2701c, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x71004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x28004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x28020, BIT(0), BIT(0)); + + ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, ARRAY_SIZE(gcc_dfs_clocks)); + if (ret) + return ret; + + /* FORCE_MEM_CORE_ON for ufs phy ice core clocks */ + regmap_update_bits(regmap, gcc_ufs_phy_ice_core_clk.halt_reg, BIT(14), BIT(14)); + + return qcom_cc_really_probe(pdev, &gcc_sm8350_desc, regmap); +} + +static struct platform_driver gcc_sm8350_driver = { + .probe = gcc_sm8350_probe, + .driver = { + .name = "sm8350-gcc", + .of_match_table = gcc_sm8350_match_table, + }, +}; + +static int __init gcc_sm8350_init(void) +{ + return platform_driver_register(&gcc_sm8350_driver); +} +subsys_initcall(gcc_sm8350_init); + +static void __exit gcc_sm8350_exit(void) +{ + platform_driver_unregister(&gcc_sm8350_driver); +} +module_exit(gcc_sm8350_exit); + +MODULE_DESCRIPTION("QTI GCC SM8350 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c index af26e0695b86..51ed640e527b 100644 --- a/drivers/clk/qcom/gdsc.c +++ b/drivers/clk/qcom/gdsc.c @@ -183,7 +183,10 @@ static inline int gdsc_assert_reset(struct gdsc *sc) static inline void gdsc_force_mem_on(struct gdsc *sc) { int i; - u32 mask = RETAIN_MEM | RETAIN_PERIPH; + u32 mask = RETAIN_MEM; + + if (!(sc->flags & NO_RET_PERIPH)) + mask |= RETAIN_PERIPH; for (i = 0; i < sc->cxc_count; i++) regmap_update_bits(sc->regmap, sc->cxcs[i], mask, mask); @@ -192,7 +195,10 @@ static inline void gdsc_force_mem_on(struct gdsc *sc) static inline void gdsc_clear_mem_on(struct gdsc *sc) { int i; - u32 mask = RETAIN_MEM | RETAIN_PERIPH; + u32 mask = RETAIN_MEM; + + if (!(sc->flags & NO_RET_PERIPH)) + mask |= RETAIN_PERIPH; for (i = 0; i < sc->cxc_count; i++) regmap_update_bits(sc->regmap, sc->cxcs[i], mask, 0); diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h index bd537438c793..5bb396b344d1 100644 --- a/drivers/clk/qcom/gdsc.h +++ b/drivers/clk/qcom/gdsc.h @@ -42,7 +42,7 @@ struct gdsc { #define PWRSTS_ON BIT(2) #define PWRSTS_OFF_ON (PWRSTS_OFF | PWRSTS_ON) #define PWRSTS_RET_ON (PWRSTS_RET | PWRSTS_ON) - const u8 flags; + const u16 flags; #define VOTABLE BIT(0) #define CLAMP_IO BIT(1) #define HW_CTRL BIT(2) @@ -51,6 +51,7 @@ struct gdsc { #define POLL_CFG_GDSCR BIT(5) #define ALWAYS_ON BIT(6) #define RETAIN_FF_ENABLE BIT(7) +#define NO_RET_PERIPH BIT(8) struct reset_controller_dev *rcdev; unsigned int *resets; unsigned int reset_count; diff --git a/drivers/clk/qcom/gpucc-msm8998.c b/drivers/clk/qcom/gpucc-msm8998.c index 9b3923af02a1..fedfffaf0a8d 100644 --- a/drivers/clk/qcom/gpucc-msm8998.c +++ b/drivers/clk/qcom/gpucc-msm8998.c @@ -50,6 +50,11 @@ static struct clk_branch gpucc_cxo_clk = { }, }; +static struct pll_vco fabia_vco[] = { + { 249600000, 2000000000, 0 }, + { 125000000, 1000000000, 1 }, +}; + static const struct clk_div_table post_div_table_fabia_even[] = { { 0x0, 1 }, { 0x1, 2 }, @@ -61,11 +66,13 @@ static const struct clk_div_table post_div_table_fabia_even[] = { static struct clk_alpha_pll gpupll0 = { .offset = 0x0, .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .vco_table = fabia_vco, + .num_vco = ARRAY_SIZE(fabia_vco), .clkr.hw.init = &(struct clk_init_data){ .name = "gpupll0", .parent_hws = (const struct clk_hw *[]){ &gpucc_cxo_clk.clkr.hw }, .num_parents = 1, - .ops = &clk_alpha_pll_fixed_fabia_ops, + .ops = &clk_alpha_pll_fabia_ops, }, }; @@ -80,6 +87,7 @@ static struct clk_alpha_pll_postdiv gpupll0_out_even = { .name = "gpupll0_out_even", .parent_hws = (const struct clk_hw *[]){ &gpupll0.clkr.hw }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; @@ -253,12 +261,16 @@ static struct gdsc gpu_cx_gdsc = { static struct gdsc gpu_gx_gdsc = { .gdscr = 0x1094, .clamp_io_ctrl = 0x130, + .resets = (unsigned int []){ GPU_GX_BCR }, + .reset_count = 1, + .cxcs = (unsigned int []){ 0x1098 }, + .cxc_count = 1, .pd = { .name = "gpu_gx", }, .parent = &gpu_cx_gdsc.pd, - .pwrsts = PWRSTS_OFF_ON, - .flags = CLAMP_IO | AON_RESET, + .pwrsts = PWRSTS_OFF_ON | PWRSTS_RET, + .flags = CLAMP_IO | SW_RESET | AON_RESET | NO_RET_PERIPH, }; static struct clk_regmap *gpucc_msm8998_clocks[] = { diff --git a/drivers/clk/qcom/gpucc-sdm660.c b/drivers/clk/qcom/gpucc-sdm660.c new file mode 100644 index 000000000000..1ebcceb3a50d --- /dev/null +++ b/drivers/clk/qcom/gpucc-sdm660.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2020, AngeloGioacchino Del Regno + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clk-alpha-pll.h" +#include "common.h" +#include "clk-regmap.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-branch.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_GPU_XO, + P_CORE_BI_PLL_TEST_SE, + P_GPLL0_OUT_MAIN, + P_GPLL0_OUT_MAIN_DIV, + P_GPU_PLL0_PLL_OUT_MAIN, + P_GPU_PLL1_PLL_OUT_MAIN, +}; + +static struct clk_branch gpucc_cxo_clk = { + .halt_reg = 0x1020, + .clkr = { + .enable_reg = 0x1020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpucc_cxo_clk", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + .name = "xo" + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_IS_CRITICAL, + }, + }, +}; + +static struct pll_vco gpu_vco[] = { + { 1000000000, 2000000000, 0 }, + { 500000000, 1000000000, 2 }, + { 250000000, 500000000, 3 }, +}; + +static struct clk_alpha_pll gpu_pll0_pll_out_main = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = gpu_vco, + .num_vco = ARRAY_SIZE(gpu_vco), + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpu_pll0_pll_out_main", + .parent_data = &(const struct clk_parent_data){ + .hw = &gpucc_cxo_clk.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, +}; + +static struct clk_alpha_pll gpu_pll1_pll_out_main = { + .offset = 0x40, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = gpu_vco, + .num_vco = ARRAY_SIZE(gpu_vco), + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpu_pll1_pll_out_main", + .parent_data = &(const struct clk_parent_data){ + .hw = &gpucc_cxo_clk.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, +}; + +static const struct parent_map gpucc_parent_map_1[] = { + { P_GPU_XO, 0 }, + { P_GPU_PLL0_PLL_OUT_MAIN, 1 }, + { P_GPU_PLL1_PLL_OUT_MAIN, 3 }, + { P_GPLL0_OUT_MAIN, 5 }, +}; + +static const struct clk_parent_data gpucc_parent_data_1[] = { + { .hw = &gpucc_cxo_clk.clkr.hw }, + { .hw = &gpu_pll0_pll_out_main.clkr.hw }, + { .hw = &gpu_pll1_pll_out_main.clkr.hw }, + { .fw_name = "gcc_gpu_gpll0_clk", .name = "gcc_gpu_gpll0_clk" }, +}; + +static struct clk_rcg2_gfx3d gfx3d_clk_src = { + .div = 2, + .rcg = { + .cmd_rcgr = 0x1070, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gpucc_parent_map_1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gfx3d_clk_src", + .parent_data = gpucc_parent_data_1, + .num_parents = 4, + .ops = &clk_gfx3d_ops, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + }, + }, + .hws = (struct clk_hw*[]){ + &gpucc_cxo_clk.clkr.hw, + &gpu_pll0_pll_out_main.clkr.hw, + &gpu_pll1_pll_out_main.clkr.hw, + } +}; + +static struct clk_branch gpucc_gfx3d_clk = { + .halt_reg = 0x1098, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x1098, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1098, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpucc_gfx3d_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gfx3d_clk_src.rcg.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static const struct parent_map gpucc_parent_map_0[] = { + { P_GPU_XO, 0 }, + { P_GPLL0_OUT_MAIN, 5 }, + { P_GPLL0_OUT_MAIN_DIV, 6 }, +}; + +static const struct clk_parent_data gpucc_parent_data_0[] = { + { .hw = &gpucc_cxo_clk.clkr.hw }, + { .fw_name = "gcc_gpu_gpll0_clk", .name = "gcc_gpu_gpll0_clk" }, + { .fw_name = "gcc_gpu_gpll0_div_clk", .name = "gcc_gpu_gpll0_div_clk" }, +}; + +static const struct freq_tbl ftbl_rbbmtimer_clk_src[] = { + F(19200000, P_GPU_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 rbbmtimer_clk_src = { + .cmd_rcgr = 0x10b0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gpucc_parent_map_0, + .freq_tbl = ftbl_rbbmtimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rbbmtimer_clk_src", + .parent_data = gpucc_parent_data_0, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_rbcpr_clk_src[] = { + F(19200000, P_GPU_XO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN_DIV, 6, 0, 0), + { } +}; + +static struct clk_rcg2 rbcpr_clk_src = { + .cmd_rcgr = 0x1030, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gpucc_parent_map_0, + .freq_tbl = ftbl_rbcpr_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rbcpr_clk_src", + .parent_data = gpucc_parent_data_0, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gpucc_rbbmtimer_clk = { + .halt_reg = 0x10d0, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10d0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpucc_rbbmtimer_clk", + .parent_names = (const char *[]){ + "rbbmtimer_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpucc_rbcpr_clk = { + .halt_reg = 0x1054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpucc_rbcpr_clk", + .parent_names = (const char *[]){ + "rbcpr_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc gpu_cx_gdsc = { + .gdscr = 0x1004, + .gds_hw_ctrl = 0x1008, + .pd = { + .name = "gpu_cx", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc gpu_gx_gdsc = { + .gdscr = 0x1094, + .clamp_io_ctrl = 0x130, + .resets = (unsigned int []){ GPU_GX_BCR }, + .reset_count = 1, + .cxcs = (unsigned int []){ 0x1098 }, + .cxc_count = 1, + .pd = { + .name = "gpu_gx", + }, + .parent = &gpu_cx_gdsc.pd, + .pwrsts = PWRSTS_OFF | PWRSTS_ON | PWRSTS_RET, + .flags = CLAMP_IO | SW_RESET | AON_RESET | NO_RET_PERIPH, +}; + +static struct gdsc *gpucc_sdm660_gdscs[] = { + [GPU_CX_GDSC] = &gpu_cx_gdsc, + [GPU_GX_GDSC] = &gpu_gx_gdsc, +}; + +static const struct qcom_reset_map gpucc_sdm660_resets[] = { + [GPU_CX_BCR] = { 0x1000 }, + [RBCPR_BCR] = { 0x1050 }, + [GPU_GX_BCR] = { 0x1090 }, + [SPDM_BCR] = { 0x10E0 }, +}; + +static struct clk_regmap *gpucc_sdm660_clocks[] = { + [GPUCC_CXO_CLK] = &gpucc_cxo_clk.clkr, + [GPU_PLL0_PLL] = &gpu_pll0_pll_out_main.clkr, + [GPU_PLL1_PLL] = &gpu_pll1_pll_out_main.clkr, + [GFX3D_CLK_SRC] = &gfx3d_clk_src.rcg.clkr, + [RBCPR_CLK_SRC] = &rbcpr_clk_src.clkr, + [RBBMTIMER_CLK_SRC] = &rbbmtimer_clk_src.clkr, + [GPUCC_RBCPR_CLK] = &gpucc_rbcpr_clk.clkr, + [GPUCC_GFX3D_CLK] = &gpucc_gfx3d_clk.clkr, + [GPUCC_RBBMTIMER_CLK] = &gpucc_rbbmtimer_clk.clkr, +}; + +static const struct regmap_config gpucc_660_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x9034, + .fast_io = true, +}; + +static const struct qcom_cc_desc gpucc_sdm660_desc = { + .config = &gpucc_660_regmap_config, + .clks = gpucc_sdm660_clocks, + .num_clks = ARRAY_SIZE(gpucc_sdm660_clocks), + .resets = gpucc_sdm660_resets, + .num_resets = ARRAY_SIZE(gpucc_sdm660_resets), + .gdscs = gpucc_sdm660_gdscs, + .num_gdscs = ARRAY_SIZE(gpucc_sdm660_gdscs), +}; + +static const struct of_device_id gpucc_sdm660_match_table[] = { + { .compatible = "qcom,gpucc-sdm660" }, + { .compatible = "qcom,gpucc-sdm630" }, + { } +}; +MODULE_DEVICE_TABLE(of, gpucc_sdm660_match_table); + +static int gpucc_sdm660_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + struct alpha_pll_config gpu_pll_config = { + .config_ctl_val = 0x4001055b, + .alpha = 0xaaaaab00, + .alpha_en_mask = BIT(24), + .vco_val = 0x2 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, + }; + + regmap = qcom_cc_map(pdev, &gpucc_sdm660_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* 800MHz configuration for GPU PLL0 */ + gpu_pll_config.l = 0x29; + gpu_pll_config.alpha_hi = 0xaa; + clk_alpha_pll_configure(&gpu_pll0_pll_out_main, regmap, &gpu_pll_config); + + /* 740MHz configuration for GPU PLL1 */ + gpu_pll_config.l = 0x26; + gpu_pll_config.alpha_hi = 0x8a; + clk_alpha_pll_configure(&gpu_pll1_pll_out_main, regmap, &gpu_pll_config); + + return qcom_cc_really_probe(pdev, &gpucc_sdm660_desc, regmap); +} + +static struct platform_driver gpucc_sdm660_driver = { + .probe = gpucc_sdm660_probe, + .driver = { + .name = "gpucc-sdm660", + .of_match_table = gpucc_sdm660_match_table, + }, +}; +module_platform_driver(gpucc_sdm660_driver); + +MODULE_DESCRIPTION("Qualcomm SDM630/SDM660 GPUCC Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/lpass-gfm-sm8250.c b/drivers/clk/qcom/lpass-gfm-sm8250.c index d366c7c2abc7..f5e31e692b9b 100644 --- a/drivers/clk/qcom/lpass-gfm-sm8250.c +++ b/drivers/clk/qcom/lpass-gfm-sm8250.c @@ -33,14 +33,13 @@ struct clk_gfm { void __iomem *gfm_mux; }; -#define GFM_MASK BIT(1) #define to_clk_gfm(_hw) container_of(_hw, struct clk_gfm, hw) static u8 clk_gfm_get_parent(struct clk_hw *hw) { struct clk_gfm *clk = to_clk_gfm(hw); - return readl(clk->gfm_mux) & GFM_MASK; + return readl(clk->gfm_mux) & clk->mux_mask; } static int clk_gfm_set_parent(struct clk_hw *hw, u8 index) @@ -51,9 +50,10 @@ static int clk_gfm_set_parent(struct clk_hw *hw, u8 index) val = readl(clk->gfm_mux); if (index) - val |= GFM_MASK; + val |= clk->mux_mask; else - val &= ~GFM_MASK; + val &= ~clk->mux_mask; + writel(val, clk->gfm_mux); diff --git a/drivers/clk/qcom/mmcc-msm8974.c b/drivers/clk/qcom/mmcc-msm8974.c index 015426262d08..a1552b6771bc 100644 --- a/drivers/clk/qcom/mmcc-msm8974.c +++ b/drivers/clk/qcom/mmcc-msm8974.c @@ -74,22 +74,6 @@ static const char * const mmcc_xo_mmpll0_dsi_hdmi_gpll0[] = { "dsi1pll", }; -static const struct parent_map mmcc_xo_mmpll0_1_2_gpll0_map[] = { - { P_XO, 0 }, - { P_MMPLL0, 1 }, - { P_MMPLL1, 2 }, - { P_GPLL0, 5 }, - { P_MMPLL2, 3 } -}; - -static const char * const mmcc_xo_mmpll0_1_2_gpll0[] = { - "xo", - "mmpll0_vote", - "mmpll1_vote", - "mmss_gpll0_vote", - "mmpll2", -}; - static const struct parent_map mmcc_xo_mmpll0_1_3_gpll0_map[] = { { P_XO, 0 }, { P_MMPLL0, 1 }, diff --git a/drivers/clk/qcom/mmcc-msm8996.c b/drivers/clk/qcom/mmcc-msm8996.c index 3b3aac07fb2d..24843e4f2599 100644 --- a/drivers/clk/qcom/mmcc-msm8996.c +++ b/drivers/clk/qcom/mmcc-msm8996.c @@ -528,16 +528,23 @@ static struct clk_rcg2 maxi_clk_src = { }, }; -static struct clk_rcg2 gfx3d_clk_src = { - .cmd_rcgr = 0x4000, - .hid_width = 5, - .parent_map = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0_map, - .clkr.hw.init = &(struct clk_init_data){ - .name = "gfx3d_clk_src", - .parent_names = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0, - .num_parents = 6, - .ops = &clk_gfx3d_ops, - .flags = CLK_SET_RATE_PARENT, +static struct clk_rcg2_gfx3d gfx3d_clk_src = { + .rcg = { + .cmd_rcgr = 0x4000, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gfx3d_clk_src", + .parent_names = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0, + .num_parents = 6, + .ops = &clk_gfx3d_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, + .hws = (struct clk_hw*[]) { + &mmpll9.clkr.hw, + &mmpll2.clkr.hw, + &mmpll8.clkr.hw }, }; @@ -3089,7 +3096,7 @@ static struct clk_regmap *mmcc_msm8996_clocks[] = { [AHB_CLK_SRC] = &ahb_clk_src.clkr, [AXI_CLK_SRC] = &axi_clk_src.clkr, [MAXI_CLK_SRC] = &maxi_clk_src.clkr, - [GFX3D_CLK_SRC] = &gfx3d_clk_src.clkr, + [GFX3D_CLK_SRC] = &gfx3d_clk_src.rcg.clkr, [RBBMTIMER_CLK_SRC] = &rbbmtimer_clk_src.clkr, [ISENSE_CLK_SRC] = &isense_clk_src.clkr, [RBCPR_CLK_SRC] = &rbcpr_clk_src.clkr, diff --git a/drivers/clk/qcom/mmcc-msm8998.c b/drivers/clk/qcom/mmcc-msm8998.c index dd68983fe22e..467dadccde02 100644 --- a/drivers/clk/qcom/mmcc-msm8998.c +++ b/drivers/clk/qcom/mmcc-msm8998.c @@ -1211,6 +1211,8 @@ static struct clk_rcg2 vfe1_clk_src = { static struct clk_branch misc_ahb_clk = { .halt_reg = 0x328, + .hwcg_reg = 0x328, + .hwcg_bit = 1, .clkr = { .enable_reg = 0x328, .enable_mask = BIT(0), @@ -1241,6 +1243,8 @@ static struct clk_branch video_core_clk = { static struct clk_branch video_ahb_clk = { .halt_reg = 0x1030, + .hwcg_reg = 0x1030, + .hwcg_bit = 1, .clkr = { .enable_reg = 0x1030, .enable_mask = BIT(0), @@ -1315,6 +1319,8 @@ static struct clk_branch video_subcore1_clk = { static struct clk_branch mdss_ahb_clk = { .halt_reg = 0x2308, + .hwcg_reg = 0x2308, + .hwcg_bit = 1, .clkr = { .enable_reg = 0x2308, .enable_mask = BIT(0), @@ -2496,6 +2502,8 @@ static struct clk_branch mnoc_ahb_clk = { static struct clk_branch bimc_smmu_ahb_clk = { .halt_reg = 0xe004, + .hwcg_reg = 0xe004, + .hwcg_bit = 1, .clkr = { .enable_reg = 0xe004, .enable_mask = BIT(0), @@ -2511,6 +2519,8 @@ static struct clk_branch bimc_smmu_ahb_clk = { static struct clk_branch bimc_smmu_axi_clk = { .halt_reg = 0xe008, + .hwcg_reg = 0xe008, + .hwcg_bit = 1, .clkr = { .enable_reg = 0xe008, .enable_mask = BIT(0), @@ -2653,7 +2663,7 @@ static struct gdsc bimc_smmu_gdsc = { .name = "bimc_smmu", }, .pwrsts = PWRSTS_OFF_ON, - .flags = HW_CTRL, + .flags = HW_CTRL | ALWAYS_ON, }; static struct clk_regmap *mmcc_msm8998_clocks[] = { diff --git a/drivers/clk/qcom/mmcc-sdm660.c b/drivers/clk/qcom/mmcc-sdm660.c new file mode 100644 index 000000000000..941993bc610d --- /dev/null +++ b/drivers/clk/qcom/mmcc-sdm660.c @@ -0,0 +1,2864 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2020, Martin Botka + * Copyright (c) 2020, Konrad Dybcio + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include + +#include "common.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-alpha-pll.h" +#include "clk-rcg.h" +#include "clk-branch.h" +#include "reset.h" +#include "gdsc.h" + +enum { + P_XO, + P_DSI0PLL_BYTE, + P_DSI0PLL, + P_DSI1PLL_BYTE, + P_DSI1PLL, + P_GPLL0, + P_GPLL0_DIV, + P_MMPLL0, + P_MMPLL10, + P_MMPLL3, + P_MMPLL4, + P_MMPLL5, + P_MMPLL6, + P_MMPLL7, + P_MMPLL8, + P_SLEEP_CLK, + P_DP_PHY_PLL_LINK_CLK, + P_DP_PHY_PLL_VCO_DIV, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL4, 2 }, + { P_MMPLL7, 3 }, + { P_MMPLL8, 4 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +/* Voteable PLL */ +static struct clk_alpha_pll mmpll0 = { + .offset = 0xc000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x1f0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static struct clk_alpha_pll mmpll6 = { + .offset = 0xf0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x1f0, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "mmpll6", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +/* APSS controlled PLLs */ +static struct pll_vco vco[] = { + { 1000000000, 2000000000, 0 }, + { 750000000, 1500000000, 1 }, + { 500000000, 1000000000, 2 }, + { 250000000, 500000000, 3 }, +}; + +static struct pll_vco mmpll3_vco[] = { + { 750000000, 1500000000, 1 }, +}; + +static const struct alpha_pll_config mmpll10_config = { + .l = 0x1e, + .config_ctl_val = 0x00004289, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll10 = { + .offset = 0x190, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll10", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct alpha_pll_config mmpll3_config = { + .l = 0x2e, + .config_ctl_val = 0x4001055b, + .vco_val = 0x1 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll3 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = mmpll3_vco, + .num_vco = ARRAY_SIZE(mmpll3_vco), + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll3", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct alpha_pll_config mmpll4_config = { + .l = 0x28, + .config_ctl_val = 0x4001055b, + .vco_val = 0x2 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll4 = { + .offset = 0x50, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = vco, + .num_vco = ARRAY_SIZE(vco), + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct alpha_pll_config mmpll5_config = { + .l = 0x2a, + .config_ctl_val = 0x4001055b, + .alpha_hi = 0xf8, + .alpha_en_mask = BIT(24), + .vco_val = 0x2 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll5 = { + .offset = 0xa0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = vco, + .num_vco = ARRAY_SIZE(vco), + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll5", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct alpha_pll_config mmpll7_config = { + .l = 0x32, + .config_ctl_val = 0x4001055b, + .vco_val = 0x2 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll7 = { + .offset = 0x140, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = vco, + .num_vco = ARRAY_SIZE(vco), + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll7", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct alpha_pll_config mmpll8_config = { + .l = 0x30, + .alpha_hi = 0x70, + .alpha_en_mask = BIT(24), + .config_ctl_val = 0x4001055b, + .vco_val = 0x2 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll8 = { + .offset = 0x1c0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = vco, + .num_vco = ARRAY_SIZE(vco), + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll8", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll4.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll8.clkr.hw }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_dsibyte_map[] = { + { P_XO, 0 }, + { P_DSI0PLL_BYTE, 1 }, + { P_DSI1PLL_BYTE, 2 }, +}; + +static const struct clk_parent_data mmcc_xo_dsibyte[] = { + { .fw_name = "xo" }, + { .fw_name = "dsi0pllbyte" }, + { .fw_name = "dsi1pllbyte" }, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL4, 2 }, + { P_MMPLL7, 3 }, + { P_MMPLL10, 4 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll4.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll10.clkr.hw }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL4, 1 }, + { P_MMPLL7, 2 }, + { P_MMPLL10, 3 }, + { P_SLEEP_CLK, 4 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll4.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll10.clkr.hw }, + { .fw_name = "sleep_clk" }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL7, 2 }, + { P_MMPLL10, 3 }, + { P_SLEEP_CLK, 4 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll10.clkr.hw }, + { .fw_name = "sleep_clk" }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_dplink_dpvco_map[] = { + { P_XO, 0 }, + { P_DP_PHY_PLL_LINK_CLK, 1 }, + { P_DP_PHY_PLL_VCO_DIV, 2 }, +}; + +static const struct clk_parent_data mmcc_xo_dplink_dpvco[] = { + { .fw_name = "xo" }, + { .fw_name = "dp_link_2x_clk_divsel_five" }, + { .fw_name = "dp_vco_divided_clk_src_mux" }, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL5, 2 }, + { P_MMPLL7, 3 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll5.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_dsi0pll_dsi1pll_map[] = { + { P_XO, 0 }, + { P_DSI0PLL, 1 }, + { P_DSI1PLL, 2 }, +}; + +static const struct clk_parent_data mmcc_xo_dsi0pll_dsi1pll[] = { + { .fw_name = "xo" }, + { .fw_name = "dsi0pll" }, + { .fw_name = "dsi1pll" }, +}; + +static const struct parent_map mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL4, 2 }, + { P_MMPLL7, 3 }, + { P_MMPLL10, 4 }, + { P_MMPLL6, 5 }, + { P_GPLL0, 6 }, +}; + +static const struct clk_parent_data mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll4.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll10.clkr.hw }, + { .hw = &mmpll6.clkr.hw }, + { .fw_name = "gpll0" }, +}; + +static const struct parent_map mmcc_xo_mmpll0_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_mmpll6_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL4, 2 }, + { P_MMPLL7, 3 }, + { P_MMPLL10, 4 }, + { P_GPLL0, 5 }, + { P_MMPLL6, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_mmpll6[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll4.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll10.clkr.hw }, + { .fw_name = "gpll0" }, + { .hw = &mmpll6.clkr.hw }, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll8_mmpll3_mmpll6_gpll0_mmpll7_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL8, 2 }, + { P_MMPLL3, 3 }, + { P_MMPLL6, 4 }, + { P_GPLL0, 5 }, + { P_MMPLL7, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll8_mmpll3_mmpll6_gpll0_mmpll7[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll8.clkr.hw }, + { .hw = &mmpll3.clkr.hw }, + { .hw = &mmpll6.clkr.hw }, + { .fw_name = "gpll0" }, + { .hw = &mmpll7.clkr.hw }, +}; + +static const struct freq_tbl ftbl_ahb_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(40000000, P_GPLL0_DIV, 7.5, 0, 0), + F(80800000, P_MMPLL0, 10, 0, 0), + { } +}; + +static struct clk_rcg2 ahb_clk_src = { + .cmd_rcgr = 0x5000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_gpll0_gpll0_div_map, + .freq_tbl = ftbl_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "ahb_clk_src", + .parent_data = mmcc_xo_mmpll0_gpll0_gpll0_div, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 byte0_clk_src = { + .cmd_rcgr = 0x2120, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "byte0_clk_src", + .parent_data = mmcc_xo_dsibyte, + .num_parents = 3, + .ops = &clk_byte2_ops, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_rcg2 byte1_clk_src = { + .cmd_rcgr = 0x2140, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "byte1_clk_src", + .parent_data = mmcc_xo_dsibyte, + .num_parents = 3, + .ops = &clk_byte2_ops, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static const struct freq_tbl ftbl_camss_gp0_clk_src[] = { + F(10000, P_XO, 16, 1, 120), + F(24000, P_XO, 16, 1, 50), + F(6000000, P_GPLL0_DIV, 10, 1, 5), + F(12000000, P_GPLL0_DIV, 10, 2, 5), + F(13043478, P_GPLL0_DIV, 1, 1, 23), + F(24000000, P_GPLL0_DIV, 1, 2, 25), + F(50000000, P_GPLL0_DIV, 6, 0, 0), + F(100000000, P_GPLL0_DIV, 3, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + { } +}; + +static struct clk_rcg2 camss_gp0_clk_src = { + .cmd_rcgr = 0x3420, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_camss_gp0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "camss_gp0_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 camss_gp1_clk_src = { + .cmd_rcgr = 0x3450, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_camss_gp0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "camss_gp1_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_cci_clk_src[] = { + F(37500000, P_GPLL0_DIV, 8, 0, 0), + F(50000000, P_GPLL0_DIV, 6, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + { } +}; + +static struct clk_rcg2 cci_clk_src = { + .cmd_rcgr = 0x3300, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_cci_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "cci_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_cpp_clk_src[] = { + F(120000000, P_GPLL0, 5, 0, 0), + F(256000000, P_MMPLL4, 3, 0, 0), + F(384000000, P_MMPLL4, 2, 0, 0), + F(480000000, P_MMPLL7, 2, 0, 0), + F(540000000, P_MMPLL6, 2, 0, 0), + F(576000000, P_MMPLL10, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cpp_clk_src = { + .cmd_rcgr = 0x3640, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_mmpll6_map, + .freq_tbl = ftbl_cpp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "cpp_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_mmpll6, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csi0_clk_src[] = { + F(100000000, P_GPLL0_DIV, 3, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(310000000, P_MMPLL8, 3, 0, 0), + F(404000000, P_MMPLL0, 2, 0, 0), + F(465000000, P_MMPLL8, 2, 0, 0), + { } +}; + +static struct clk_rcg2 csi0_clk_src = { + .cmd_rcgr = 0x3090, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi0_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csi0phytimer_clk_src[] = { + F(100000000, P_GPLL0_DIV, 3, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(269333333, P_MMPLL0, 3, 0, 0), + { } +}; + +static struct clk_rcg2 csi0phytimer_clk_src = { + .cmd_rcgr = 0x3000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi0phytimer_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi1_clk_src = { + .cmd_rcgr = 0x3100, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi1_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi1phytimer_clk_src = { + .cmd_rcgr = 0x3030, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi1phytimer_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi2_clk_src = { + .cmd_rcgr = 0x3160, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi2_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi2phytimer_clk_src = { + .cmd_rcgr = 0x3060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi2phytimer_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi3_clk_src = { + .cmd_rcgr = 0x31c0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi3_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csiphy_clk_src[] = { + F(100000000, P_GPLL0_DIV, 3, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(269333333, P_MMPLL0, 3, 0, 0), + F(320000000, P_MMPLL7, 3, 0, 0), + { } +}; + +static struct clk_rcg2 csiphy_clk_src = { + .cmd_rcgr = 0x3800, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csiphy_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csiphy_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_aux_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 dp_aux_clk_src = { + .cmd_rcgr = 0x2260, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_gpll0_gpll0_div_map, + .freq_tbl = ftbl_dp_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_aux_clk_src", + .parent_data = mmcc_xo_gpll0_gpll0_div, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_crypto_clk_src[] = { + F(101250000, P_DP_PHY_PLL_VCO_DIV, 4, 0, 0), + F(168750000, P_DP_PHY_PLL_VCO_DIV, 4, 0, 0), + F(337500000, P_DP_PHY_PLL_VCO_DIV, 4, 0, 0), + { } +}; + +static struct clk_rcg2 dp_crypto_clk_src = { + .cmd_rcgr = 0x2220, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_dplink_dpvco_map, + .freq_tbl = ftbl_dp_crypto_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_crypto_clk_src", + .parent_data = mmcc_xo_dplink_dpvco, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_gtc_clk_src[] = { + F(40000000, P_GPLL0_DIV, 7.5, 0, 0), + F(60000000, P_GPLL0, 10, 0, 0), + { } +}; + +static struct clk_rcg2 dp_gtc_clk_src = { + .cmd_rcgr = 0x2280, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_gpll0_gpll0_div_map, + .freq_tbl = ftbl_dp_gtc_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_gtc_clk_src", + .parent_data = mmcc_xo_gpll0_gpll0_div, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_link_clk_src[] = { + F(162000000, P_DP_PHY_PLL_LINK_CLK, 2, 0, 0), + F(270000000, P_DP_PHY_PLL_LINK_CLK, 2, 0, 0), + F(540000000, P_DP_PHY_PLL_LINK_CLK, 2, 0, 0), + { } +}; + +static struct clk_rcg2 dp_link_clk_src = { + .cmd_rcgr = 0x2200, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_dplink_dpvco_map, + .freq_tbl = ftbl_dp_link_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_link_clk_src", + .parent_data = mmcc_xo_dplink_dpvco, + .num_parents = 3, + .ops = &clk_rcg2_ops, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_rcg2 dp_pixel_clk_src = { + .cmd_rcgr = 0x2240, + .mnd_width = 16, + .hid_width = 5, + .parent_map = mmcc_xo_dplink_dpvco_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_pixel_clk_src", + .parent_data = mmcc_xo_dplink_dpvco, + .num_parents = 3, + .ops = &clk_dp_ops, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_rcg2 esc0_clk_src = { + .cmd_rcgr = 0x2160, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "esc0_clk_src", + .parent_data = mmcc_xo_dsibyte, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 esc1_clk_src = { + .cmd_rcgr = 0x2180, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "esc1_clk_src", + .parent_data = mmcc_xo_dsibyte, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_jpeg0_clk_src[] = { + F(66666667, P_GPLL0_DIV, 4.5, 0, 0), + F(133333333, P_GPLL0, 4.5, 0, 0), + F(219428571, P_MMPLL4, 3.5, 0, 0), + F(320000000, P_MMPLL7, 3, 0, 0), + F(480000000, P_MMPLL7, 2, 0, 0), + { } +}; + +static struct clk_rcg2 jpeg0_clk_src = { + .cmd_rcgr = 0x3500, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_jpeg0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "jpeg0_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_mclk0_clk_src[] = { + F(4800000, P_XO, 4, 0, 0), + F(6000000, P_GPLL0_DIV, 10, 1, 5), + F(8000000, P_GPLL0_DIV, 1, 2, 75), + F(9600000, P_XO, 2, 0, 0), + F(16666667, P_GPLL0_DIV, 2, 1, 9), + F(19200000, P_XO, 1, 0, 0), + F(24000000, P_MMPLL10, 1, 1, 24), + F(33333333, P_GPLL0_DIV, 1, 1, 9), + F(48000000, P_GPLL0, 1, 2, 25), + F(66666667, P_GPLL0, 1, 1, 9), + { } +}; + +static struct clk_rcg2 mclk0_clk_src = { + .cmd_rcgr = 0x3360, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk0_clk_src", + .parent_data = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk1_clk_src = { + .cmd_rcgr = 0x3390, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk1_clk_src", + .parent_data = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk2_clk_src = { + .cmd_rcgr = 0x33c0, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk2_clk_src", + .parent_data = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk3_clk_src = { + .cmd_rcgr = 0x33f0, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk3_clk_src", + .parent_data = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_mdp_clk_src[] = { + F(100000000, P_GPLL0_DIV, 3, 0, 0), + F(150000000, P_GPLL0_DIV, 2, 0, 0), + F(171428571, P_GPLL0, 3.5, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(275000000, P_MMPLL5, 3, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(330000000, P_MMPLL5, 2.5, 0, 0), + F(412500000, P_MMPLL5, 2, 0, 0), + { } +}; + +static struct clk_rcg2 mdp_clk_src = { + .cmd_rcgr = 0x2040, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mdp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mdp_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 pclk0_clk_src = { + .cmd_rcgr = 0x2000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_dsi0pll_dsi1pll_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pclk0_clk_src", + .parent_data = mmcc_xo_dsi0pll_dsi1pll, + .num_parents = 3, + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_rcg2 pclk1_clk_src = { + .cmd_rcgr = 0x2020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_dsi0pll_dsi1pll_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pclk1_clk_src", + .parent_data = mmcc_xo_dsi0pll_dsi1pll, + .num_parents = 3, + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static const struct freq_tbl ftbl_rot_clk_src[] = { + F(171428571, P_GPLL0, 3.5, 0, 0), + F(275000000, P_MMPLL5, 3, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(330000000, P_MMPLL5, 2.5, 0, 0), + F(412500000, P_MMPLL5, 2, 0, 0), + { } +}; + +static struct clk_rcg2 rot_clk_src = { + .cmd_rcgr = 0x21a0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div_map, + .freq_tbl = ftbl_rot_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rot_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_vfe0_clk_src[] = { + F(120000000, P_GPLL0, 5, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(256000000, P_MMPLL4, 3, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(404000000, P_MMPLL0, 2, 0, 0), + F(480000000, P_MMPLL7, 2, 0, 0), + F(540000000, P_MMPLL6, 2, 0, 0), + F(576000000, P_MMPLL10, 1, 0, 0), + { } +}; + +static struct clk_rcg2 vfe0_clk_src = { + .cmd_rcgr = 0x3600, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0_map, + .freq_tbl = ftbl_vfe0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vfe0_clk_src", + .parent_data = mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 vfe1_clk_src = { + .cmd_rcgr = 0x3620, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0_map, + .freq_tbl = ftbl_vfe0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vfe1_clk_src", + .parent_data = mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_video_core_clk_src[] = { + F(133333333, P_GPLL0, 4.5, 0, 0), + F(269333333, P_MMPLL0, 3, 0, 0), + F(320000000, P_MMPLL7, 3, 0, 0), + F(404000000, P_MMPLL0, 2, 0, 0), + F(441600000, P_MMPLL3, 2, 0, 0), + F(518400000, P_MMPLL3, 2, 0, 0), + { } +}; + +static struct clk_rcg2 video_core_clk_src = { + .cmd_rcgr = 0x1000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll8_mmpll3_mmpll6_gpll0_mmpll7_map, + .freq_tbl = ftbl_video_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "video_core_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll8_mmpll3_mmpll6_gpll0_mmpll7, + .num_parents = 7, + .ops = &clk_rcg2_ops, + .flags = CLK_IS_CRITICAL, + }, +}; + +static struct clk_rcg2 vsync_clk_src = { + .cmd_rcgr = 0x2080, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_gpll0_gpll0_div_map, + .freq_tbl = ftbl_dp_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vsync_clk_src", + .parent_data = mmcc_xo_gpll0_gpll0_div, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch bimc_smmu_ahb_clk = { + .halt_reg = 0xe004, + .halt_check = BRANCH_VOTED, + .hwcg_reg = 0xe004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xe004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "bimc_smmu_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch bimc_smmu_axi_clk = { + .halt_reg = 0xe008, + .halt_check = BRANCH_VOTED, + .hwcg_reg = 0xe008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xe008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "bimc_smmu_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_ahb_clk = { + .halt_reg = 0x348c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x348c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x348c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cci_ahb_clk = { + .halt_reg = 0x3348, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3348, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cci_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cci_clk = { + .halt_reg = 0x3344, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3344, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cci_clk", + .parent_hws = (const struct clk_hw *[]){ &cci_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cpp_ahb_clk = { + .halt_reg = 0x36b4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cpp_axi_clk = { + .halt_reg = 0x36c4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36c4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cpp_clk = { + .halt_reg = 0x36b0, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36b0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_clk", + .parent_hws = (const struct clk_hw *[]){ &cpp_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cpp_vbif_ahb_clk = { + .halt_reg = 0x36c8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36c8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_vbif_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0_ahb_clk = { + .halt_reg = 0x30bc, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x30bc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0_clk = { + .halt_reg = 0x30b4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x30b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0phytimer_clk = { + .halt_reg = 0x3024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0phytimer_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0phytimer_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0pix_clk = { + .halt_reg = 0x30e4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x30e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0rdi_clk = { + .halt_reg = 0x30d4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x30d4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1_ahb_clk = { + .halt_reg = 0x3128, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3128, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1_clk = { + .halt_reg = 0x3124, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3124, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1phytimer_clk = { + .halt_reg = 0x3054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1phytimer_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1phytimer_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1pix_clk = { + .halt_reg = 0x3154, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1rdi_clk = { + .halt_reg = 0x3144, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3144, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2_ahb_clk = { + .halt_reg = 0x3188, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3188, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2_clk = { + .halt_reg = 0x3184, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3184, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2phytimer_clk = { + .halt_reg = 0x3084, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2phytimer_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2phytimer_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2pix_clk = { + .halt_reg = 0x31b4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x31b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2rdi_clk = { + .halt_reg = 0x31a4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x31a4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3_ahb_clk = { + .halt_reg = 0x31e8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x31e8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3_clk = { + .halt_reg = 0x31e4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x31e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3_clk", + .parent_hws = (const struct clk_hw *[]){ &csi3_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3pix_clk = { + .halt_reg = 0x3214, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3214, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi3_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3rdi_clk = { + .halt_reg = 0x3204, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3204, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi3_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi_vfe0_clk = { + .halt_reg = 0x3704, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3704, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi_vfe0_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi_vfe1_clk = { + .halt_reg = 0x3714, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3714, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi_vfe1_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csiphy0_clk = { + .halt_reg = 0x3740, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3740, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csiphy0_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csiphy1_clk = { + .halt_reg = 0x3744, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3744, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csiphy1_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csiphy2_clk = { + .halt_reg = 0x3748, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3748, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csiphy2_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + + +static struct clk_branch camss_cphy_csid0_clk = { + .halt_reg = 0x3730, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3730, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid0_clk", + .parent_hws = (const struct clk_hw *[]){ &camss_csiphy0_clk.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cphy_csid1_clk = { + .halt_reg = 0x3734, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3734, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid1_clk", + .parent_hws = (const struct clk_hw *[]){ &camss_csiphy1_clk.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cphy_csid2_clk = { + .halt_reg = 0x3738, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3738, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid2_clk", + .parent_hws = (const struct clk_hw *[]){ &camss_csiphy2_clk.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cphy_csid3_clk = { + .halt_reg = 0x373c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x373c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid3_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_gp0_clk = { + .halt_reg = 0x3444, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3444, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_gp0_clk", + .parent_hws = (const struct clk_hw *[]){ &camss_gp0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_gp1_clk = { + .halt_reg = 0x3474, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3474, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_gp1_clk", + .parent_hws = (const struct clk_hw *[]){ &camss_gp1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_ispif_ahb_clk = { + .halt_reg = 0x3224, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3224, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_ispif_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg0_clk = { + .halt_reg = 0x35a8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x35a8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg0_clk", + .parent_hws = (const struct clk_hw *[]){ &jpeg0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg_ahb_clk = { + .halt_reg = 0x35b4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x35b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg_axi_clk = { + .halt_reg = 0x35b8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x35b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch throttle_camss_axi_clk = { + .halt_reg = 0x3c3c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3c3c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "throttle_camss_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk0_clk = { + .halt_reg = 0x3384, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3384, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk0_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk1_clk = { + .halt_reg = 0x33b4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk1_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk2_clk = { + .halt_reg = 0x33e4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk2_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk2_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk3_clk = { + .halt_reg = 0x3414, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3414, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk3_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk3_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_micro_ahb_clk = { + .halt_reg = 0x3494, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3494, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_micro_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_top_ahb_clk = { + .halt_reg = 0x3484, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3484, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_top_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe0_ahb_clk = { + .halt_reg = 0x3668, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3668, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe0_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe0_clk = { + .halt_reg = 0x36a8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36a8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe0_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe0_stream_clk = { + .halt_reg = 0x3720, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3720, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe0_stream_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe1_ahb_clk = { + .halt_reg = 0x3678, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3678, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe1_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe1_clk = { + .halt_reg = 0x36ac, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe1_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe1_stream_clk = { + .halt_reg = 0x3724, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3724, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe1_stream_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe_vbif_ahb_clk = { + .halt_reg = 0x36b8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_vbif_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe_vbif_axi_clk = { + .halt_reg = 0x36bc, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36bc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_vbif_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch csiphy_ahb2crif_clk = { + .halt_reg = 0x374c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x374c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x374c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "csiphy_ahb2crif_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_ahb_clk = { + .halt_reg = 0x2308, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x8a004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2308, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .flags = CLK_SET_RATE_PARENT, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_axi_clk_src[] = { + F(75000000, P_GPLL0, 8, 0, 0), + F(171428571, P_GPLL0, 3.5, 0, 0), + F(240000000, P_GPLL0, 2.5, 0, 0), + F(323200000, P_MMPLL0, 2.5, 0, 0), + F(406000000, P_MMPLL0, 2, 0, 0), + { } +}; + +/* RO to linux */ +static struct clk_rcg2 axi_clk_src = { + .cmd_rcgr = 0xd000, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "axi_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch mdss_axi_clk = { + .halt_reg = 0x2310, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2310, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_axi_clk", + .parent_hws = (const struct clk_hw *[]){ &axi_clk_src.clkr.hw }, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch throttle_mdss_axi_clk = { + .halt_reg = 0x246c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x246c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x246c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "throttle_mdss_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_byte0_clk = { + .halt_reg = 0x233c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x233c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte0_clk", + .parent_hws = (const struct clk_hw *[]){ &byte0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap_div mdss_byte0_intf_div_clk = { + .reg = 0x237c, + .shift = 0, + .width = 2, + /* + * NOTE: Op does not work for div-3. Current assumption is that div-3 + * is not a recommended setting for this divider. + */ + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte0_intf_div_clk", + .parent_hws = (const struct clk_hw *[]){ &byte0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_GET_RATE_NOCACHE, + }, + }, +}; + +static struct clk_branch mdss_byte0_intf_clk = { + .halt_reg = 0x2374, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2374, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte0_intf_clk", + .parent_hws = (const struct clk_hw *[]){ &mdss_byte0_intf_div_clk.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_byte1_clk = { + .halt_reg = 0x2340, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2340, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte1_clk", + .parent_hws = (const struct clk_hw *[]){ &byte1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap_div mdss_byte1_intf_div_clk = { + .reg = 0x2380, + .shift = 0, + .width = 2, + /* + * NOTE: Op does not work for div-3. Current assumption is that div-3 + * is not a recommended setting for this divider. + */ + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte1_intf_div_clk", + .parent_hws = (const struct clk_hw *[]){ &byte1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_GET_RATE_NOCACHE, + }, + }, +}; + +static struct clk_branch mdss_byte1_intf_clk = { + .halt_reg = 0x2378, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2378, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte1_intf_clk", + .parent_hws = (const struct clk_hw *[]){ &mdss_byte1_intf_div_clk.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_dp_aux_clk = { + .halt_reg = 0x2364, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2364, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_aux_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_aux_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_dp_crypto_clk = { + .halt_reg = 0x235c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x235c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_crypto_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_crypto_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_dp_gtc_clk = { + .halt_reg = 0x2368, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2368, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_gtc_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_gtc_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_dp_link_clk = { + .halt_reg = 0x2354, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2354, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_link_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_link_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Reset state of MDSS_DP_LINK_INTF_DIV is 0x3 (div-4) */ +static struct clk_branch mdss_dp_link_intf_clk = { + .halt_reg = 0x2358, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2358, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_link_intf_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_link_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_dp_pixel_clk = { + .halt_reg = 0x2360, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2360, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_pixel_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_pixel_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_esc0_clk = { + .halt_reg = 0x2344, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2344, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_esc0_clk", + .parent_hws = (const struct clk_hw *[]){ &esc0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_esc1_clk = { + .halt_reg = 0x2348, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2348, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_esc1_clk", + .parent_hws = (const struct clk_hw *[]){ &esc1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_hdmi_dp_ahb_clk = { + .halt_reg = 0x230c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x230c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_hdmi_dp_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_mdp_clk = { + .halt_reg = 0x231c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x231c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_mdp_clk", + .parent_hws = (const struct clk_hw *[]){ &mdp_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_pclk0_clk = { + .halt_reg = 0x2314, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2314, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_pclk0_clk", + .parent_hws = (const struct clk_hw *[]){ &pclk0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_pclk1_clk = { + .halt_reg = 0x2318, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2318, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_pclk1_clk", + .parent_hws = (const struct clk_hw *[]){ &pclk1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_rot_clk = { + .halt_reg = 0x2350, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2350, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_rot_clk", + .parent_hws = (const struct clk_hw *[]){ &rot_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_vsync_clk = { + .halt_reg = 0x2328, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2328, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_vsync_clk", + .parent_hws = (const struct clk_hw *[]){ &vsync_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mnoc_ahb_clk = { + .halt_reg = 0x5024, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x5024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mnoc_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch misc_ahb_clk = { + .halt_reg = 0x328, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x328, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x328, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "misc_ahb_clk", + /* + * Dependency to be enabled before the branch is + * enabled. + */ + .parent_hws = (const struct clk_hw *[]){ &mnoc_ahb_clk.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch misc_cxo_clk = { + .halt_reg = 0x324, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x324, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "misc_cxo_clk", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch snoc_dvm_axi_clk = { + .halt_reg = 0xe040, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xe040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "snoc_dvm_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_ahb_clk = { + .halt_reg = 0x1030, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x1030, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_axi_clk = { + .halt_reg = 0x1034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch throttle_video_axi_clk = { + .halt_reg = 0x118c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x118c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x118c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "throttle_video_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_core_clk = { + .halt_reg = 0x1028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_core_clk", + .parent_hws = (const struct clk_hw *[]){ &video_core_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_subcore0_clk = { + .halt_reg = 0x1048, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_subcore0_clk", + .parent_hws = (const struct clk_hw *[]){ &video_core_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc venus_gdsc = { + .gdscr = 0x1024, + .pd = { + .name = "venus", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc venus_core0_gdsc = { + .gdscr = 0x1040, + .pd = { + .name = "venus_core0", + }, + .parent = &venus_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc mdss_gdsc = { + .gdscr = 0x2304, + .pd = { + .name = "mdss", + }, + .cxcs = (unsigned int []){ 0x2040 }, + .cxc_count = 1, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_top_gdsc = { + .gdscr = 0x34a0, + .pd = { + .name = "camss_top", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_vfe0_gdsc = { + .gdscr = 0x3664, + .pd = { + .name = "camss_vfe0", + }, + .parent = &camss_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_vfe1_gdsc = { + .gdscr = 0x3674, + .pd = { + .name = "camss_vfe1_gdsc", + }, + .parent = &camss_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_cpp_gdsc = { + .gdscr = 0x36d4, + .pd = { + .name = "camss_cpp", + }, + .parent = &camss_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +/* This GDSC seems to hang the whole multimedia subsystem. +static struct gdsc bimc_smmu_gdsc = { + .gdscr = 0xe020, + .gds_hw_ctrl = 0xe024, + .pd = { + .name = "bimc_smmu", + }, + .pwrsts = PWRSTS_OFF_ON, + .parent = &bimc_smmu_gdsc.pd, + .flags = HW_CTRL, +}; +*/ + +static struct clk_regmap *mmcc_660_clocks[] = { + [AHB_CLK_SRC] = &ahb_clk_src.clkr, + [BYTE0_CLK_SRC] = &byte0_clk_src.clkr, + [BYTE1_CLK_SRC] = &byte1_clk_src.clkr, + [CAMSS_GP0_CLK_SRC] = &camss_gp0_clk_src.clkr, + [CAMSS_GP1_CLK_SRC] = &camss_gp1_clk_src.clkr, + [CCI_CLK_SRC] = &cci_clk_src.clkr, + [CPP_CLK_SRC] = &cpp_clk_src.clkr, + [CSI0_CLK_SRC] = &csi0_clk_src.clkr, + [CSI0PHYTIMER_CLK_SRC] = &csi0phytimer_clk_src.clkr, + [CSI1_CLK_SRC] = &csi1_clk_src.clkr, + [CSI1PHYTIMER_CLK_SRC] = &csi1phytimer_clk_src.clkr, + [CSI2_CLK_SRC] = &csi2_clk_src.clkr, + [CSI2PHYTIMER_CLK_SRC] = &csi2phytimer_clk_src.clkr, + [CSI3_CLK_SRC] = &csi3_clk_src.clkr, + [CSIPHY_CLK_SRC] = &csiphy_clk_src.clkr, + [DP_AUX_CLK_SRC] = &dp_aux_clk_src.clkr, + [DP_CRYPTO_CLK_SRC] = &dp_crypto_clk_src.clkr, + [DP_GTC_CLK_SRC] = &dp_gtc_clk_src.clkr, + [DP_LINK_CLK_SRC] = &dp_link_clk_src.clkr, + [DP_PIXEL_CLK_SRC] = &dp_pixel_clk_src.clkr, + [ESC0_CLK_SRC] = &esc0_clk_src.clkr, + [ESC1_CLK_SRC] = &esc1_clk_src.clkr, + [JPEG0_CLK_SRC] = &jpeg0_clk_src.clkr, + [MCLK0_CLK_SRC] = &mclk0_clk_src.clkr, + [MCLK1_CLK_SRC] = &mclk1_clk_src.clkr, + [MCLK2_CLK_SRC] = &mclk2_clk_src.clkr, + [MCLK3_CLK_SRC] = &mclk3_clk_src.clkr, + [MDP_CLK_SRC] = &mdp_clk_src.clkr, + [MMPLL0_PLL] = &mmpll0.clkr, + [MMPLL10_PLL] = &mmpll10.clkr, + [MMPLL3_PLL] = &mmpll3.clkr, + [MMPLL4_PLL] = &mmpll4.clkr, + [MMPLL5_PLL] = &mmpll5.clkr, + [MMPLL6_PLL] = &mmpll6.clkr, + [MMPLL7_PLL] = &mmpll7.clkr, + [MMPLL8_PLL] = &mmpll8.clkr, + [BIMC_SMMU_AHB_CLK] = &bimc_smmu_ahb_clk.clkr, + [BIMC_SMMU_AXI_CLK] = &bimc_smmu_axi_clk.clkr, + [CAMSS_AHB_CLK] = &camss_ahb_clk.clkr, + [CAMSS_CCI_AHB_CLK] = &camss_cci_ahb_clk.clkr, + [CAMSS_CCI_CLK] = &camss_cci_clk.clkr, + [CAMSS_CPHY_CSID0_CLK] = &camss_cphy_csid0_clk.clkr, + [CAMSS_CPHY_CSID1_CLK] = &camss_cphy_csid1_clk.clkr, + [CAMSS_CPHY_CSID2_CLK] = &camss_cphy_csid2_clk.clkr, + [CAMSS_CPHY_CSID3_CLK] = &camss_cphy_csid3_clk.clkr, + [CAMSS_CPP_AHB_CLK] = &camss_cpp_ahb_clk.clkr, + [CAMSS_CPP_AXI_CLK] = &camss_cpp_axi_clk.clkr, + [CAMSS_CPP_CLK] = &camss_cpp_clk.clkr, + [CAMSS_CPP_VBIF_AHB_CLK] = &camss_cpp_vbif_ahb_clk.clkr, + [CAMSS_CSI0_AHB_CLK] = &camss_csi0_ahb_clk.clkr, + [CAMSS_CSI0_CLK] = &camss_csi0_clk.clkr, + [CAMSS_CSI0PHYTIMER_CLK] = &camss_csi0phytimer_clk.clkr, + [CAMSS_CSI0PIX_CLK] = &camss_csi0pix_clk.clkr, + [CAMSS_CSI0RDI_CLK] = &camss_csi0rdi_clk.clkr, + [CAMSS_CSI1_AHB_CLK] = &camss_csi1_ahb_clk.clkr, + [CAMSS_CSI1_CLK] = &camss_csi1_clk.clkr, + [CAMSS_CSI1PHYTIMER_CLK] = &camss_csi1phytimer_clk.clkr, + [CAMSS_CSI1PIX_CLK] = &camss_csi1pix_clk.clkr, + [CAMSS_CSI1RDI_CLK] = &camss_csi1rdi_clk.clkr, + [CAMSS_CSI2_AHB_CLK] = &camss_csi2_ahb_clk.clkr, + [CAMSS_CSI2_CLK] = &camss_csi2_clk.clkr, + [CAMSS_CSI2PHYTIMER_CLK] = &camss_csi2phytimer_clk.clkr, + [CAMSS_CSI2PIX_CLK] = &camss_csi2pix_clk.clkr, + [CAMSS_CSI2RDI_CLK] = &camss_csi2rdi_clk.clkr, + [CAMSS_CSI3_AHB_CLK] = &camss_csi3_ahb_clk.clkr, + [CAMSS_CSI3_CLK] = &camss_csi3_clk.clkr, + [CAMSS_CSI3PIX_CLK] = &camss_csi3pix_clk.clkr, + [CAMSS_CSI3RDI_CLK] = &camss_csi3rdi_clk.clkr, + [CAMSS_CSI_VFE0_CLK] = &camss_csi_vfe0_clk.clkr, + [CAMSS_CSI_VFE1_CLK] = &camss_csi_vfe1_clk.clkr, + [CAMSS_CSIPHY0_CLK] = &camss_csiphy0_clk.clkr, + [CAMSS_CSIPHY1_CLK] = &camss_csiphy1_clk.clkr, + [CAMSS_CSIPHY2_CLK] = &camss_csiphy2_clk.clkr, + [CAMSS_GP0_CLK] = &camss_gp0_clk.clkr, + [CAMSS_GP1_CLK] = &camss_gp1_clk.clkr, + [CAMSS_ISPIF_AHB_CLK] = &camss_ispif_ahb_clk.clkr, + [CAMSS_JPEG0_CLK] = &camss_jpeg0_clk.clkr, + [CAMSS_JPEG_AHB_CLK] = &camss_jpeg_ahb_clk.clkr, + [CAMSS_JPEG_AXI_CLK] = &camss_jpeg_axi_clk.clkr, + [CAMSS_MCLK0_CLK] = &camss_mclk0_clk.clkr, + [CAMSS_MCLK1_CLK] = &camss_mclk1_clk.clkr, + [CAMSS_MCLK2_CLK] = &camss_mclk2_clk.clkr, + [CAMSS_MCLK3_CLK] = &camss_mclk3_clk.clkr, + [CAMSS_MICRO_AHB_CLK] = &camss_micro_ahb_clk.clkr, + [CAMSS_TOP_AHB_CLK] = &camss_top_ahb_clk.clkr, + [CAMSS_VFE0_AHB_CLK] = &camss_vfe0_ahb_clk.clkr, + [CAMSS_VFE0_CLK] = &camss_vfe0_clk.clkr, + [CAMSS_VFE0_STREAM_CLK] = &camss_vfe0_stream_clk.clkr, + [CAMSS_VFE1_AHB_CLK] = &camss_vfe1_ahb_clk.clkr, + [CAMSS_VFE1_CLK] = &camss_vfe1_clk.clkr, + [CAMSS_VFE1_STREAM_CLK] = &camss_vfe1_stream_clk.clkr, + [CAMSS_VFE_VBIF_AHB_CLK] = &camss_vfe_vbif_ahb_clk.clkr, + [CAMSS_VFE_VBIF_AXI_CLK] = &camss_vfe_vbif_axi_clk.clkr, + [CSIPHY_AHB2CRIF_CLK] = &csiphy_ahb2crif_clk.clkr, + [MDSS_AHB_CLK] = &mdss_ahb_clk.clkr, + [MDSS_AXI_CLK] = &mdss_axi_clk.clkr, + [MDSS_BYTE0_CLK] = &mdss_byte0_clk.clkr, + [MDSS_BYTE0_INTF_CLK] = &mdss_byte0_intf_clk.clkr, + [MDSS_BYTE0_INTF_DIV_CLK] = &mdss_byte0_intf_div_clk.clkr, + [MDSS_BYTE1_CLK] = &mdss_byte1_clk.clkr, + [MDSS_BYTE1_INTF_CLK] = &mdss_byte1_intf_clk.clkr, + [MDSS_DP_AUX_CLK] = &mdss_dp_aux_clk.clkr, + [MDSS_DP_CRYPTO_CLK] = &mdss_dp_crypto_clk.clkr, + [MDSS_DP_GTC_CLK] = &mdss_dp_gtc_clk.clkr, + [MDSS_DP_LINK_CLK] = &mdss_dp_link_clk.clkr, + [MDSS_DP_LINK_INTF_CLK] = &mdss_dp_link_intf_clk.clkr, + [MDSS_DP_PIXEL_CLK] = &mdss_dp_pixel_clk.clkr, + [MDSS_ESC0_CLK] = &mdss_esc0_clk.clkr, + [MDSS_ESC1_CLK] = &mdss_esc1_clk.clkr, + [MDSS_HDMI_DP_AHB_CLK] = &mdss_hdmi_dp_ahb_clk.clkr, + [MDSS_MDP_CLK] = &mdss_mdp_clk.clkr, + [MDSS_PCLK0_CLK] = &mdss_pclk0_clk.clkr, + [MDSS_PCLK1_CLK] = &mdss_pclk1_clk.clkr, + [MDSS_ROT_CLK] = &mdss_rot_clk.clkr, + [MDSS_VSYNC_CLK] = &mdss_vsync_clk.clkr, + [MISC_AHB_CLK] = &misc_ahb_clk.clkr, + [MISC_CXO_CLK] = &misc_cxo_clk.clkr, + [MNOC_AHB_CLK] = &mnoc_ahb_clk.clkr, + [SNOC_DVM_AXI_CLK] = &snoc_dvm_axi_clk.clkr, + [THROTTLE_CAMSS_AXI_CLK] = &throttle_camss_axi_clk.clkr, + [THROTTLE_MDSS_AXI_CLK] = &throttle_mdss_axi_clk.clkr, + [THROTTLE_VIDEO_AXI_CLK] = &throttle_video_axi_clk.clkr, + [VIDEO_AHB_CLK] = &video_ahb_clk.clkr, + [VIDEO_AXI_CLK] = &video_axi_clk.clkr, + [VIDEO_CORE_CLK] = &video_core_clk.clkr, + [VIDEO_SUBCORE0_CLK] = &video_subcore0_clk.clkr, + [PCLK0_CLK_SRC] = &pclk0_clk_src.clkr, + [PCLK1_CLK_SRC] = &pclk1_clk_src.clkr, + [ROT_CLK_SRC] = &rot_clk_src.clkr, + [VFE0_CLK_SRC] = &vfe0_clk_src.clkr, + [VFE1_CLK_SRC] = &vfe1_clk_src.clkr, + [VIDEO_CORE_CLK_SRC] = &video_core_clk_src.clkr, + [VSYNC_CLK_SRC] = &vsync_clk_src.clkr, + [MDSS_BYTE1_INTF_DIV_CLK] = &mdss_byte1_intf_div_clk.clkr, + [AXI_CLK_SRC] = &axi_clk_src.clkr, +}; + +static struct gdsc *mmcc_sdm660_gdscs[] = { + [VENUS_GDSC] = &venus_gdsc, + [VENUS_CORE0_GDSC] = &venus_core0_gdsc, + [MDSS_GDSC] = &mdss_gdsc, + [CAMSS_TOP_GDSC] = &camss_top_gdsc, + [CAMSS_VFE0_GDSC] = &camss_vfe0_gdsc, + [CAMSS_VFE1_GDSC] = &camss_vfe1_gdsc, + [CAMSS_CPP_GDSC] = &camss_cpp_gdsc, +}; + +static const struct qcom_reset_map mmcc_660_resets[] = { + [CAMSS_MICRO_BCR] = { 0x3490 }, +}; + +static const struct regmap_config mmcc_660_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x40000, + .fast_io = true, +}; + +static const struct qcom_cc_desc mmcc_660_desc = { + .config = &mmcc_660_regmap_config, + .clks = mmcc_660_clocks, + .num_clks = ARRAY_SIZE(mmcc_660_clocks), + .resets = mmcc_660_resets, + .num_resets = ARRAY_SIZE(mmcc_660_resets), + .gdscs = mmcc_sdm660_gdscs, + .num_gdscs = ARRAY_SIZE(mmcc_sdm660_gdscs), +}; + +static const struct of_device_id mmcc_660_match_table[] = { + { .compatible = "qcom,mmcc-sdm660" }, + { .compatible = "qcom,mmcc-sdm630", .data = (void *)1UL }, + { } +}; +MODULE_DEVICE_TABLE(of, mmcc_660_match_table); + +static void sdm630_clock_override(void) +{ + /* SDM630 has only one DSI */ + mmcc_660_desc.clks[BYTE1_CLK_SRC] = NULL; + mmcc_660_desc.clks[MDSS_BYTE1_CLK] = NULL; + mmcc_660_desc.clks[MDSS_BYTE1_INTF_DIV_CLK] = NULL; + mmcc_660_desc.clks[MDSS_BYTE1_INTF_CLK] = NULL; + mmcc_660_desc.clks[ESC1_CLK_SRC] = NULL; + mmcc_660_desc.clks[MDSS_ESC1_CLK] = NULL; + mmcc_660_desc.clks[PCLK1_CLK_SRC] = NULL; + mmcc_660_desc.clks[MDSS_PCLK1_CLK] = NULL; +} + +static int mmcc_660_probe(struct platform_device *pdev) +{ + const struct of_device_id *id; + struct regmap *regmap; + bool is_sdm630; + + id = of_match_device(mmcc_660_match_table, &pdev->dev); + if (!id) + return -ENODEV; + is_sdm630 = !!(id->data); + + regmap = qcom_cc_map(pdev, &mmcc_660_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + if (is_sdm630) + sdm630_clock_override(); + + clk_alpha_pll_configure(&mmpll3, regmap, &mmpll3_config); + clk_alpha_pll_configure(&mmpll4, regmap, &mmpll4_config); + clk_alpha_pll_configure(&mmpll5, regmap, &mmpll5_config); + clk_alpha_pll_configure(&mmpll7, regmap, &mmpll7_config); + clk_alpha_pll_configure(&mmpll8, regmap, &mmpll8_config); + clk_alpha_pll_configure(&mmpll10, regmap, &mmpll10_config); + + return qcom_cc_really_probe(pdev, &mmcc_660_desc, regmap); +} + +static struct platform_driver mmcc_660_driver = { + .probe = mmcc_660_probe, + .driver = { + .name = "mmcc-sdm660", + .of_match_table = mmcc_660_match_table, + }, +}; +module_platform_driver(mmcc_660_driver); + +MODULE_DESCRIPTION("Qualcomm SDM630/SDM660 MMCC driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/videocc-sm8250.c b/drivers/clk/qcom/videocc-sm8250.c index 2797c61f5938..b0efadc19634 100644 --- a/drivers/clk/qcom/videocc-sm8250.c +++ b/drivers/clk/qcom/videocc-sm8250.c @@ -169,6 +169,21 @@ static struct clk_regmap_div video_cc_mvs0c_div2_div_clk_src = { }, }; +static struct clk_regmap_div video_cc_mvs0_div_clk_src = { + .reg = 0xd54, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "video_cc_mvs0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_mvs0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + static struct clk_regmap_div video_cc_mvs1c_div2_div_clk_src = { .reg = 0xcf4, .shift = 0, @@ -202,6 +217,24 @@ static struct clk_branch video_cc_mvs0c_clk = { }, }; +static struct clk_branch video_cc_mvs0_clk = { + .halt_reg = 0xd34, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0xd34, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_mvs0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_mvs0_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch video_cc_mvs1_div2_clk = { .halt_reg = 0xdf4, .halt_check = BRANCH_HALT_VOTED, @@ -245,6 +278,7 @@ static struct gdsc mvs0c_gdsc = { }, .flags = 0, .pwrsts = PWRSTS_OFF_ON, + .supply = "mmcx", }; static struct gdsc mvs1c_gdsc = { @@ -254,6 +288,7 @@ static struct gdsc mvs1c_gdsc = { }, .flags = 0, .pwrsts = PWRSTS_OFF_ON, + .supply = "mmcx", }; static struct gdsc mvs0_gdsc = { @@ -263,6 +298,7 @@ static struct gdsc mvs0_gdsc = { }, .flags = HW_CTRL, .pwrsts = PWRSTS_OFF_ON, + .supply = "mmcx", }; static struct gdsc mvs1_gdsc = { @@ -272,10 +308,13 @@ static struct gdsc mvs1_gdsc = { }, .flags = HW_CTRL, .pwrsts = PWRSTS_OFF_ON, + .supply = "mmcx", }; static struct clk_regmap *video_cc_sm8250_clocks[] = { + [VIDEO_CC_MVS0_CLK] = &video_cc_mvs0_clk.clkr, [VIDEO_CC_MVS0_CLK_SRC] = &video_cc_mvs0_clk_src.clkr, + [VIDEO_CC_MVS0_DIV_CLK_SRC] = &video_cc_mvs0_div_clk_src.clkr, [VIDEO_CC_MVS0C_CLK] = &video_cc_mvs0c_clk.clkr, [VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC] = &video_cc_mvs0c_div2_div_clk_src.clkr, [VIDEO_CC_MVS1_CLK_SRC] = &video_cc_mvs1_clk_src.clkr, diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig index 18915d668a30..607e64a17d72 100644 --- a/drivers/clk/renesas/Kconfig +++ b/drivers/clk/renesas/Kconfig @@ -148,6 +148,7 @@ config CLK_R8A77995 config CLK_R8A779A0 bool "R-Car V3U clock support" if COMPILE_TEST + select CLK_RCAR_CPG_LIB select CLK_RENESAS_CPG_MSSR config CLK_R9A06G032 @@ -162,12 +163,16 @@ config CLK_SH73A0 # Family +config CLK_RCAR_CPG_LIB + bool "CPG/MSSR library functions" if COMPILE_TEST + config CLK_RCAR_GEN2_CPG bool "R-Car Gen2 CPG clock support" if COMPILE_TEST select CLK_RENESAS_CPG_MSSR config CLK_RCAR_GEN3_CPG bool "R-Car Gen3 and RZ/G2 CPG clock support" if COMPILE_TEST + select CLK_RCAR_CPG_LIB select CLK_RENESAS_CPG_MSSR config CLK_RCAR_USB2_CLOCK_SEL diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile index c803912ef2ce..ef0d2bba92bf 100644 --- a/drivers/clk/renesas/Makefile +++ b/drivers/clk/renesas/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_CLK_R9A06G032) += r9a06g032-clocks.o obj-$(CONFIG_CLK_SH73A0) += clk-sh73a0.o # Family +obj-$(CONFIG_CLK_RCAR_CPG_LIB) += rcar-cpg-lib.o obj-$(CONFIG_CLK_RCAR_GEN2_CPG) += rcar-gen2-cpg.o obj-$(CONFIG_CLK_RCAR_GEN3_CPG) += rcar-gen3-cpg.o obj-$(CONFIG_CLK_RCAR_USB2_CLOCK_SEL) += rcar-usb2-clock-sel.o diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c index 2cd6e3876fbd..41593c126faf 100644 --- a/drivers/clk/renesas/r8a7796-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c @@ -128,6 +128,11 @@ static const struct cpg_core_clk r8a7796_core_clks[] __initconst = { static struct mssr_mod_clk r8a7796_mod_clks[] __initdata = { DEF_MOD("fdp1-0", 119, R8A7796_CLK_S0D1), + DEF_MOD("tmu4", 121, R8A7796_CLK_S0D6), + DEF_MOD("tmu3", 122, R8A7796_CLK_S3D2), + DEF_MOD("tmu2", 123, R8A7796_CLK_S3D2), + DEF_MOD("tmu1", 124, R8A7796_CLK_S3D2), + DEF_MOD("tmu0", 125, R8A7796_CLK_CP), DEF_MOD("scif5", 202, R8A7796_CLK_S3D4), DEF_MOD("scif4", 203, R8A7796_CLK_S3D4), DEF_MOD("scif3", 204, R8A7796_CLK_S3D4), diff --git a/drivers/clk/renesas/r8a77965-cpg-mssr.c b/drivers/clk/renesas/r8a77965-cpg-mssr.c index 2b55a06ac5cf..46a157732759 100644 --- a/drivers/clk/renesas/r8a77965-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77965-cpg-mssr.c @@ -123,6 +123,11 @@ static const struct cpg_core_clk r8a77965_core_clks[] __initconst = { static const struct mssr_mod_clk r8a77965_mod_clks[] __initconst = { DEF_MOD("fdp1-0", 119, R8A77965_CLK_S0D1), + DEF_MOD("tmu4", 121, R8A77965_CLK_S0D6), + DEF_MOD("tmu3", 122, R8A77965_CLK_S3D2), + DEF_MOD("tmu2", 123, R8A77965_CLK_S3D2), + DEF_MOD("tmu1", 124, R8A77965_CLK_S3D2), + DEF_MOD("tmu0", 125, R8A77965_CLK_CP), DEF_MOD("scif5", 202, R8A77965_CLK_S3D4), DEF_MOD("scif4", 203, R8A77965_CLK_S3D4), DEF_MOD("scif3", 204, R8A77965_CLK_S3D4), diff --git a/drivers/clk/renesas/r8a77990-cpg-mssr.c b/drivers/clk/renesas/r8a77990-cpg-mssr.c index 2b97ab61d044..2d172f80b34c 100644 --- a/drivers/clk/renesas/r8a77990-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77990-cpg-mssr.c @@ -124,6 +124,11 @@ static const struct cpg_core_clk r8a77990_core_clks[] __initconst = { }; static const struct mssr_mod_clk r8a77990_mod_clks[] __initconst = { + DEF_MOD("tmu4", 121, R8A77990_CLK_S0D6C), + DEF_MOD("tmu3", 122, R8A77990_CLK_S3D2C), + DEF_MOD("tmu2", 123, R8A77990_CLK_S3D2C), + DEF_MOD("tmu1", 124, R8A77990_CLK_S3D2C), + DEF_MOD("tmu0", 125, R8A77990_CLK_CP), DEF_MOD("scif5", 202, R8A77990_CLK_S3D4C), DEF_MOD("scif4", 203, R8A77990_CLK_S3D4C), DEF_MOD("scif3", 204, R8A77990_CLK_S3D4C), diff --git a/drivers/clk/renesas/r8a77995-cpg-mssr.c b/drivers/clk/renesas/r8a77995-cpg-mssr.c index 5b4691117b47..9cfd00cf4e69 100644 --- a/drivers/clk/renesas/r8a77995-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77995-cpg-mssr.c @@ -111,6 +111,11 @@ static const struct cpg_core_clk r8a77995_core_clks[] __initconst = { }; static const struct mssr_mod_clk r8a77995_mod_clks[] __initconst = { + DEF_MOD("tmu4", 121, R8A77995_CLK_S1D4C), + DEF_MOD("tmu3", 122, R8A77995_CLK_S3D2C), + DEF_MOD("tmu2", 123, R8A77995_CLK_S3D2C), + DEF_MOD("tmu1", 124, R8A77995_CLK_S3D2C), + DEF_MOD("tmu0", 125, R8A77995_CLK_CP), DEF_MOD("scif5", 202, R8A77995_CLK_S3D4C), DEF_MOD("scif4", 203, R8A77995_CLK_S3D4C), DEF_MOD("scif3", 204, R8A77995_CLK_S3D4C), diff --git a/drivers/clk/renesas/r8a779a0-cpg-mssr.c b/drivers/clk/renesas/r8a779a0-cpg-mssr.c index aa5389b04d74..f23fe9d5e5e1 100644 --- a/drivers/clk/renesas/r8a779a0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779a0-cpg-mssr.c @@ -25,6 +25,7 @@ #include +#include "rcar-cpg-lib.h" #include "renesas-cpg-mssr.h" enum rcar_r8a779a0_clk_types { @@ -32,6 +33,7 @@ enum rcar_r8a779a0_clk_types { CLK_TYPE_R8A779A0_PLL1, CLK_TYPE_R8A779A0_PLL2X_3X, /* PLL[23][01] */ CLK_TYPE_R8A779A0_PLL5, + CLK_TYPE_R8A779A0_SD, CLK_TYPE_R8A779A0_MDSEL, /* Select parent/divider using mode pin */ CLK_TYPE_R8A779A0_OSC, /* OSC EXTAL predivider and fixed divider */ }; @@ -69,7 +71,6 @@ enum clk_ids { CLK_PLL5_DIV2, CLK_PLL5_DIV4, CLK_S1, - CLK_S2, CLK_S3, CLK_SDSRC, CLK_RPCSRC, @@ -83,6 +84,9 @@ enum clk_ids { DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_PLL2X_3X, CLK_MAIN, \ .offset = _offset) +#define DEF_SD(_name, _id, _parent, _offset) \ + DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_SD, _parent, .offset = _offset) + #define DEF_MDSEL(_name, _id, _md, _parent0, _div0, _parent1, _div1) \ DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_MDSEL, \ (_parent0) << 16 | (_parent1), \ @@ -114,6 +118,7 @@ static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = { DEF_FIXED(".pll5_div4", CLK_PLL5_DIV4, CLK_PLL5_DIV2, 2, 1), DEF_FIXED(".s1", CLK_S1, CLK_PLL1_DIV2, 2, 1), DEF_FIXED(".s3", CLK_S3, CLK_PLL1_DIV2, 4, 1), + DEF_FIXED(".sdsrc", CLK_SDSRC, CLK_PLL5_DIV4, 1, 1), DEF_RATE(".oco", CLK_OCO, 32768), /* Core Clock Outputs */ @@ -137,7 +142,10 @@ static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = { DEF_FIXED("icu", R8A779A0_CLK_ICU, CLK_PLL5_DIV4, 2, 1), DEF_FIXED("icud2", R8A779A0_CLK_ICUD2, CLK_PLL5_DIV4, 4, 1), DEF_FIXED("vcbus", R8A779A0_CLK_VCBUS, CLK_PLL5_DIV4, 1, 1), - DEF_FIXED("cbfusa", R8A779A0_CLK_CBFUSA, CLK_MAIN, 2, 1), + DEF_FIXED("cbfusa", R8A779A0_CLK_CBFUSA, CLK_EXTAL, 2, 1), + DEF_FIXED("cp", R8A779A0_CLK_CP, CLK_EXTAL, 2, 1), + + DEF_SD("sd0", R8A779A0_CLK_SD0, CLK_SDSRC, 0x870), DEF_DIV6P1("mso", R8A779A0_CLK_MSO, CLK_PLL5_DIV4, 0x87c), DEF_DIV6P1("canfd", R8A779A0_CLK_CANFD, CLK_PLL5_DIV4, 0x878), @@ -148,14 +156,42 @@ static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = { }; static const struct mssr_mod_clk r8a779a0_mod_clks[] __initconst = { + DEF_MOD("avb0", 211, R8A779A0_CLK_S3D2), + DEF_MOD("avb1", 212, R8A779A0_CLK_S3D2), + DEF_MOD("avb2", 213, R8A779A0_CLK_S3D2), + DEF_MOD("avb3", 214, R8A779A0_CLK_S3D2), + DEF_MOD("avb4", 215, R8A779A0_CLK_S3D2), + DEF_MOD("avb5", 216, R8A779A0_CLK_S3D2), DEF_MOD("csi40", 331, R8A779A0_CLK_CSI0), DEF_MOD("csi41", 400, R8A779A0_CLK_CSI0), DEF_MOD("csi42", 401, R8A779A0_CLK_CSI0), DEF_MOD("csi43", 402, R8A779A0_CLK_CSI0), + DEF_MOD("fcpvd0", 508, R8A779A0_CLK_S3D1), + DEF_MOD("fcpvd1", 509, R8A779A0_CLK_S3D1), + DEF_MOD("hscif0", 514, R8A779A0_CLK_S1D2), + DEF_MOD("hscif1", 515, R8A779A0_CLK_S1D2), + DEF_MOD("hscif2", 516, R8A779A0_CLK_S1D2), + DEF_MOD("hscif3", 517, R8A779A0_CLK_S1D2), + DEF_MOD("i2c0", 518, R8A779A0_CLK_S1D4), + DEF_MOD("i2c1", 519, R8A779A0_CLK_S1D4), + DEF_MOD("i2c2", 520, R8A779A0_CLK_S1D4), + DEF_MOD("i2c3", 521, R8A779A0_CLK_S1D4), + DEF_MOD("i2c4", 522, R8A779A0_CLK_S1D4), + DEF_MOD("i2c5", 523, R8A779A0_CLK_S1D4), + DEF_MOD("i2c6", 524, R8A779A0_CLK_S1D4), + DEF_MOD("msi0", 618, R8A779A0_CLK_MSO), + DEF_MOD("msi1", 619, R8A779A0_CLK_MSO), + DEF_MOD("msi2", 620, R8A779A0_CLK_MSO), + DEF_MOD("msi3", 621, R8A779A0_CLK_MSO), + DEF_MOD("msi4", 622, R8A779A0_CLK_MSO), + DEF_MOD("msi5", 623, R8A779A0_CLK_MSO), DEF_MOD("scif0", 702, R8A779A0_CLK_S1D8), DEF_MOD("scif1", 703, R8A779A0_CLK_S1D8), DEF_MOD("scif3", 704, R8A779A0_CLK_S1D8), DEF_MOD("scif4", 705, R8A779A0_CLK_S1D8), + DEF_MOD("sdhi0", 706, R8A779A0_CLK_SD0), + DEF_MOD("sydm1", 709, R8A779A0_CLK_S1D2), + DEF_MOD("sydm2", 710, R8A779A0_CLK_S1D2), DEF_MOD("vin00", 730, R8A779A0_CLK_S1D1), DEF_MOD("vin01", 731, R8A779A0_CLK_S1D1), DEF_MOD("vin02", 800, R8A779A0_CLK_S1D1), @@ -188,10 +224,19 @@ static const struct mssr_mod_clk r8a779a0_mod_clks[] __initconst = { DEF_MOD("vin35", 827, R8A779A0_CLK_S1D1), DEF_MOD("vin36", 828, R8A779A0_CLK_S1D1), DEF_MOD("vin37", 829, R8A779A0_CLK_S1D1), + DEF_MOD("vspd0", 830, R8A779A0_CLK_S3D1), + DEF_MOD("vspd1", 831, R8A779A0_CLK_S3D1), + DEF_MOD("rwdt", 907, R8A779A0_CLK_R), + DEF_MOD("pfc0", 915, R8A779A0_CLK_CP), + DEF_MOD("pfc1", 916, R8A779A0_CLK_CP), + DEF_MOD("pfc2", 917, R8A779A0_CLK_CP), + DEF_MOD("pfc3", 918, R8A779A0_CLK_CP), + DEF_MOD("vspx0", 1028, R8A779A0_CLK_S1D1), + DEF_MOD("vspx1", 1029, R8A779A0_CLK_S1D1), + DEF_MOD("vspx2", 1030, R8A779A0_CLK_S1D1), + DEF_MOD("vspx3", 1031, R8A779A0_CLK_S1D1), }; -static spinlock_t cpg_lock; - static const struct rcar_r8a779a0_cpg_pll_config *cpg_pll_config __initdata; static unsigned int cpg_clk_extalr __initdata; static u32 cpg_mode __initdata; @@ -230,6 +275,12 @@ static struct clk * __init rcar_r8a779a0_cpg_clk_register(struct device *dev, div = cpg_pll_config->pll5_div; break; + case CLK_TYPE_R8A779A0_SD: + return cpg_sd_clk_register(core->name, base, core->offset, + __clk_get_name(parent), notifiers, + false); + break; + case CLK_TYPE_R8A779A0_MDSEL: /* * Clock selectable between two parents and two fixed dividers @@ -261,6 +312,10 @@ static struct clk * __init rcar_r8a779a0_cpg_clk_register(struct device *dev, __clk_get_name(parent), 0, mult, div); } +static const unsigned int r8a779a0_crit_mod_clks[] __initconst = { + MOD_CLK_ID(907), /* RWDT */ +}; + /* * CPG Clock Data */ @@ -311,6 +366,10 @@ const struct cpg_mssr_info r8a779a0_cpg_mssr_info __initconst = { .num_mod_clks = ARRAY_SIZE(r8a779a0_mod_clks), .num_hw_mod_clks = 15 * 32, + /* Critical Module Clocks */ + .crit_mod_clks = r8a779a0_crit_mod_clks, + .num_crit_mod_clks = ARRAY_SIZE(r8a779a0_crit_mod_clks), + /* Callbacks */ .init = r8a779a0_cpg_mssr_init, .cpg_clk_register = rcar_r8a779a0_cpg_clk_register, diff --git a/drivers/clk/renesas/rcar-cpg-lib.c b/drivers/clk/renesas/rcar-cpg-lib.c new file mode 100644 index 000000000000..7e7e5d1341d5 --- /dev/null +++ b/drivers/clk/renesas/rcar-cpg-lib.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * R-Car Gen3 Clock Pulse Generator Library + * + * Copyright (C) 2015-2018 Glider bvba + * Copyright (C) 2019 Renesas Electronics Corp. + * + * Based on clk-rcar-gen3.c + * + * Copyright (C) 2015 Renesas Electronics Corp. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rcar-cpg-lib.h" + +spinlock_t cpg_lock; + +void cpg_reg_modify(void __iomem *reg, u32 clear, u32 set) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&cpg_lock, flags); + val = readl(reg); + val &= ~clear; + val |= set; + writel(val, reg); + spin_unlock_irqrestore(&cpg_lock, flags); +}; + +static int cpg_simple_notifier_call(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct cpg_simple_notifier *csn = + container_of(nb, struct cpg_simple_notifier, nb); + + switch (action) { + case PM_EVENT_SUSPEND: + csn->saved = readl(csn->reg); + return NOTIFY_OK; + + case PM_EVENT_RESUME: + writel(csn->saved, csn->reg); + return NOTIFY_OK; + } + return NOTIFY_DONE; +} + +void cpg_simple_notifier_register(struct raw_notifier_head *notifiers, + struct cpg_simple_notifier *csn) +{ + csn->nb.notifier_call = cpg_simple_notifier_call; + raw_notifier_chain_register(notifiers, &csn->nb); +} + +/* + * SDn Clock + */ +#define CPG_SD_STP_HCK BIT(9) +#define CPG_SD_STP_CK BIT(8) + +#define CPG_SD_STP_MASK (CPG_SD_STP_HCK | CPG_SD_STP_CK) +#define CPG_SD_FC_MASK (0x7 << 2 | 0x3 << 0) + +#define CPG_SD_DIV_TABLE_DATA(stp_hck, sd_srcfc, sd_fc, sd_div) \ +{ \ + .val = ((stp_hck) ? CPG_SD_STP_HCK : 0) | \ + ((sd_srcfc) << 2) | \ + ((sd_fc) << 0), \ + .div = (sd_div), \ +} + +struct sd_div_table { + u32 val; + unsigned int div; +}; + +struct sd_clock { + struct clk_hw hw; + const struct sd_div_table *div_table; + struct cpg_simple_notifier csn; + unsigned int div_num; + unsigned int cur_div_idx; +}; + +/* SDn divider + * sd_srcfc sd_fc div + * stp_hck (div) (div) = sd_srcfc x sd_fc + *--------------------------------------------------------- + * 0 0 (1) 1 (4) 4 : SDR104 / HS200 / HS400 (8 TAP) + * 0 1 (2) 1 (4) 8 : SDR50 + * 1 2 (4) 1 (4) 16 : HS / SDR25 + * 1 3 (8) 1 (4) 32 : NS / SDR12 + * 1 4 (16) 1 (4) 64 + * 0 0 (1) 0 (2) 2 + * 0 1 (2) 0 (2) 4 : SDR104 / HS200 / HS400 (4 TAP) + * 1 2 (4) 0 (2) 8 + * 1 3 (8) 0 (2) 16 + * 1 4 (16) 0 (2) 32 + * + * NOTE: There is a quirk option to ignore the first row of the dividers + * table when searching for suitable settings. This is because HS400 on + * early ES versions of H3 and M3-W requires a specific setting to work. + */ +static const struct sd_div_table cpg_sd_div_table[] = { +/* CPG_SD_DIV_TABLE_DATA(stp_hck, sd_srcfc, sd_fc, sd_div) */ + CPG_SD_DIV_TABLE_DATA(0, 0, 1, 4), + CPG_SD_DIV_TABLE_DATA(0, 1, 1, 8), + CPG_SD_DIV_TABLE_DATA(1, 2, 1, 16), + CPG_SD_DIV_TABLE_DATA(1, 3, 1, 32), + CPG_SD_DIV_TABLE_DATA(1, 4, 1, 64), + CPG_SD_DIV_TABLE_DATA(0, 0, 0, 2), + CPG_SD_DIV_TABLE_DATA(0, 1, 0, 4), + CPG_SD_DIV_TABLE_DATA(1, 2, 0, 8), + CPG_SD_DIV_TABLE_DATA(1, 3, 0, 16), + CPG_SD_DIV_TABLE_DATA(1, 4, 0, 32), +}; + +#define to_sd_clock(_hw) container_of(_hw, struct sd_clock, hw) + +static int cpg_sd_clock_enable(struct clk_hw *hw) +{ + struct sd_clock *clock = to_sd_clock(hw); + + cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK, + clock->div_table[clock->cur_div_idx].val & + CPG_SD_STP_MASK); + + return 0; +} + +static void cpg_sd_clock_disable(struct clk_hw *hw) +{ + struct sd_clock *clock = to_sd_clock(hw); + + cpg_reg_modify(clock->csn.reg, 0, CPG_SD_STP_MASK); +} + +static int cpg_sd_clock_is_enabled(struct clk_hw *hw) +{ + struct sd_clock *clock = to_sd_clock(hw); + + return !(readl(clock->csn.reg) & CPG_SD_STP_MASK); +} + +static unsigned long cpg_sd_clock_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct sd_clock *clock = to_sd_clock(hw); + + return DIV_ROUND_CLOSEST(parent_rate, + clock->div_table[clock->cur_div_idx].div); +} + +static int cpg_sd_clock_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + unsigned long best_rate = ULONG_MAX, diff_min = ULONG_MAX; + struct sd_clock *clock = to_sd_clock(hw); + unsigned long calc_rate, diff; + unsigned int i; + + for (i = 0; i < clock->div_num; i++) { + calc_rate = DIV_ROUND_CLOSEST(req->best_parent_rate, + clock->div_table[i].div); + if (calc_rate < req->min_rate || calc_rate > req->max_rate) + continue; + + diff = calc_rate > req->rate ? calc_rate - req->rate + : req->rate - calc_rate; + if (diff < diff_min) { + best_rate = calc_rate; + diff_min = diff; + } + } + + if (best_rate == ULONG_MAX) + return -EINVAL; + + req->rate = best_rate; + return 0; +} + +static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct sd_clock *clock = to_sd_clock(hw); + unsigned int i; + + for (i = 0; i < clock->div_num; i++) + if (rate == DIV_ROUND_CLOSEST(parent_rate, + clock->div_table[i].div)) + break; + + if (i >= clock->div_num) + return -EINVAL; + + clock->cur_div_idx = i; + + cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK | CPG_SD_FC_MASK, + clock->div_table[i].val & + (CPG_SD_STP_MASK | CPG_SD_FC_MASK)); + + return 0; +} + +static const struct clk_ops cpg_sd_clock_ops = { + .enable = cpg_sd_clock_enable, + .disable = cpg_sd_clock_disable, + .is_enabled = cpg_sd_clock_is_enabled, + .recalc_rate = cpg_sd_clock_recalc_rate, + .determine_rate = cpg_sd_clock_determine_rate, + .set_rate = cpg_sd_clock_set_rate, +}; + +struct clk * __init cpg_sd_clk_register(const char *name, + void __iomem *base, unsigned int offset, const char *parent_name, + struct raw_notifier_head *notifiers, bool skip_first) +{ + struct clk_init_data init; + struct sd_clock *clock; + struct clk *clk; + u32 val; + + clock = kzalloc(sizeof(*clock), GFP_KERNEL); + if (!clock) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.ops = &cpg_sd_clock_ops; + init.flags = CLK_SET_RATE_PARENT; + init.parent_names = &parent_name; + init.num_parents = 1; + + clock->csn.reg = base + offset; + clock->hw.init = &init; + clock->div_table = cpg_sd_div_table; + clock->div_num = ARRAY_SIZE(cpg_sd_div_table); + + if (skip_first) { + clock->div_table++; + clock->div_num--; + } + + val = readl(clock->csn.reg) & ~CPG_SD_FC_MASK; + val |= CPG_SD_STP_MASK | (clock->div_table[0].val & CPG_SD_FC_MASK); + writel(val, clock->csn.reg); + + clk = clk_register(NULL, &clock->hw); + if (IS_ERR(clk)) + goto free_clock; + + cpg_simple_notifier_register(notifiers, &clock->csn); + return clk; + +free_clock: + kfree(clock); + return clk; +} + + diff --git a/drivers/clk/renesas/rcar-cpg-lib.h b/drivers/clk/renesas/rcar-cpg-lib.h new file mode 100644 index 000000000000..d00c91b116ca --- /dev/null +++ b/drivers/clk/renesas/rcar-cpg-lib.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * R-Car Gen3 Clock Pulse Generator Library + * + * Copyright (C) 2015-2018 Glider bvba + * Copyright (C) 2019 Renesas Electronics Corp. + * + * Based on clk-rcar-gen3.c + * + * Copyright (C) 2015 Renesas Electronics Corp. + */ + +#ifndef __CLK_RENESAS_RCAR_CPG_LIB_H__ +#define __CLK_RENESAS_RCAR_CPG_LIB_H__ + +extern spinlock_t cpg_lock; + +struct cpg_simple_notifier { + struct notifier_block nb; + void __iomem *reg; + u32 saved; +}; + +void cpg_simple_notifier_register(struct raw_notifier_head *notifiers, + struct cpg_simple_notifier *csn); + +void cpg_reg_modify(void __iomem *reg, u32 clear, u32 set); + +struct clk * __init cpg_sd_clk_register(const char *name, + void __iomem *base, unsigned int offset, const char *parent_name, + struct raw_notifier_head *notifiers, bool skip_first); + +#endif diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c index 063b61151488..17826599e9dd 100644 --- a/drivers/clk/renesas/rcar-gen3-cpg.c +++ b/drivers/clk/renesas/rcar-gen3-cpg.c @@ -23,6 +23,7 @@ #include #include "renesas-cpg-mssr.h" +#include "rcar-cpg-lib.h" #include "rcar-gen3-cpg.h" #define CPG_PLL0CR 0x00d8 @@ -31,52 +32,6 @@ #define CPG_RCKCR_CKSEL BIT(15) /* RCLK Clock Source Select */ -static spinlock_t cpg_lock; - -static void cpg_reg_modify(void __iomem *reg, u32 clear, u32 set) -{ - unsigned long flags; - u32 val; - - spin_lock_irqsave(&cpg_lock, flags); - val = readl(reg); - val &= ~clear; - val |= set; - writel(val, reg); - spin_unlock_irqrestore(&cpg_lock, flags); -}; - -struct cpg_simple_notifier { - struct notifier_block nb; - void __iomem *reg; - u32 saved; -}; - -static int cpg_simple_notifier_call(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct cpg_simple_notifier *csn = - container_of(nb, struct cpg_simple_notifier, nb); - - switch (action) { - case PM_EVENT_SUSPEND: - csn->saved = readl(csn->reg); - return NOTIFY_OK; - - case PM_EVENT_RESUME: - writel(csn->saved, csn->reg); - return NOTIFY_OK; - } - return NOTIFY_DONE; -} - -static void cpg_simple_notifier_register(struct raw_notifier_head *notifiers, - struct cpg_simple_notifier *csn) -{ - csn->nb.notifier_call = cpg_simple_notifier_call; - raw_notifier_chain_register(notifiers, &csn->nb); -} - /* * Z Clock & Z2 Clock * @@ -215,217 +170,6 @@ static struct clk * __init cpg_z_clk_register(const char *name, return clk; } -/* - * SDn Clock - */ -#define CPG_SD_STP_HCK BIT(9) -#define CPG_SD_STP_CK BIT(8) - -#define CPG_SD_STP_MASK (CPG_SD_STP_HCK | CPG_SD_STP_CK) -#define CPG_SD_FC_MASK (0x7 << 2 | 0x3 << 0) - -#define CPG_SD_DIV_TABLE_DATA(stp_hck, sd_srcfc, sd_fc, sd_div) \ -{ \ - .val = ((stp_hck) ? CPG_SD_STP_HCK : 0) | \ - ((sd_srcfc) << 2) | \ - ((sd_fc) << 0), \ - .div = (sd_div), \ -} - -struct sd_div_table { - u32 val; - unsigned int div; -}; - -struct sd_clock { - struct clk_hw hw; - const struct sd_div_table *div_table; - struct cpg_simple_notifier csn; - unsigned int div_num; - unsigned int cur_div_idx; -}; - -/* SDn divider - * sd_srcfc sd_fc div - * stp_hck (div) (div) = sd_srcfc x sd_fc - *--------------------------------------------------------- - * 0 0 (1) 1 (4) 4 : SDR104 / HS200 / HS400 (8 TAP) - * 0 1 (2) 1 (4) 8 : SDR50 - * 1 2 (4) 1 (4) 16 : HS / SDR25 - * 1 3 (8) 1 (4) 32 : NS / SDR12 - * 1 4 (16) 1 (4) 64 - * 0 0 (1) 0 (2) 2 - * 0 1 (2) 0 (2) 4 : SDR104 / HS200 / HS400 (4 TAP) - * 1 2 (4) 0 (2) 8 - * 1 3 (8) 0 (2) 16 - * 1 4 (16) 0 (2) 32 - * - * NOTE: There is a quirk option to ignore the first row of the dividers - * table when searching for suitable settings. This is because HS400 on - * early ES versions of H3 and M3-W requires a specific setting to work. - */ -static const struct sd_div_table cpg_sd_div_table[] = { -/* CPG_SD_DIV_TABLE_DATA(stp_hck, sd_srcfc, sd_fc, sd_div) */ - CPG_SD_DIV_TABLE_DATA(0, 0, 1, 4), - CPG_SD_DIV_TABLE_DATA(0, 1, 1, 8), - CPG_SD_DIV_TABLE_DATA(1, 2, 1, 16), - CPG_SD_DIV_TABLE_DATA(1, 3, 1, 32), - CPG_SD_DIV_TABLE_DATA(1, 4, 1, 64), - CPG_SD_DIV_TABLE_DATA(0, 0, 0, 2), - CPG_SD_DIV_TABLE_DATA(0, 1, 0, 4), - CPG_SD_DIV_TABLE_DATA(1, 2, 0, 8), - CPG_SD_DIV_TABLE_DATA(1, 3, 0, 16), - CPG_SD_DIV_TABLE_DATA(1, 4, 0, 32), -}; - -#define to_sd_clock(_hw) container_of(_hw, struct sd_clock, hw) - -static int cpg_sd_clock_enable(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - - cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK, - clock->div_table[clock->cur_div_idx].val & - CPG_SD_STP_MASK); - - return 0; -} - -static void cpg_sd_clock_disable(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - - cpg_reg_modify(clock->csn.reg, 0, CPG_SD_STP_MASK); -} - -static int cpg_sd_clock_is_enabled(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - - return !(readl(clock->csn.reg) & CPG_SD_STP_MASK); -} - -static unsigned long cpg_sd_clock_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct sd_clock *clock = to_sd_clock(hw); - - return DIV_ROUND_CLOSEST(parent_rate, - clock->div_table[clock->cur_div_idx].div); -} - -static int cpg_sd_clock_determine_rate(struct clk_hw *hw, - struct clk_rate_request *req) -{ - unsigned long best_rate = ULONG_MAX, diff_min = ULONG_MAX; - struct sd_clock *clock = to_sd_clock(hw); - unsigned long calc_rate, diff; - unsigned int i; - - for (i = 0; i < clock->div_num; i++) { - calc_rate = DIV_ROUND_CLOSEST(req->best_parent_rate, - clock->div_table[i].div); - if (calc_rate < req->min_rate || calc_rate > req->max_rate) - continue; - - diff = calc_rate > req->rate ? calc_rate - req->rate - : req->rate - calc_rate; - if (diff < diff_min) { - best_rate = calc_rate; - diff_min = diff; - } - } - - if (best_rate == ULONG_MAX) - return -EINVAL; - - req->rate = best_rate; - return 0; -} - -static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct sd_clock *clock = to_sd_clock(hw); - unsigned int i; - - for (i = 0; i < clock->div_num; i++) - if (rate == DIV_ROUND_CLOSEST(parent_rate, - clock->div_table[i].div)) - break; - - if (i >= clock->div_num) - return -EINVAL; - - clock->cur_div_idx = i; - - cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK | CPG_SD_FC_MASK, - clock->div_table[i].val & - (CPG_SD_STP_MASK | CPG_SD_FC_MASK)); - - return 0; -} - -static const struct clk_ops cpg_sd_clock_ops = { - .enable = cpg_sd_clock_enable, - .disable = cpg_sd_clock_disable, - .is_enabled = cpg_sd_clock_is_enabled, - .recalc_rate = cpg_sd_clock_recalc_rate, - .determine_rate = cpg_sd_clock_determine_rate, - .set_rate = cpg_sd_clock_set_rate, -}; - -static u32 cpg_quirks __initdata; - -#define PLL_ERRATA BIT(0) /* Missing PLL0/2/4 post-divider */ -#define RCKCR_CKSEL BIT(1) /* Manual RCLK parent selection */ -#define SD_SKIP_FIRST BIT(2) /* Skip first clock in SD table */ - -static struct clk * __init cpg_sd_clk_register(const char *name, - void __iomem *base, unsigned int offset, const char *parent_name, - struct raw_notifier_head *notifiers) -{ - struct clk_init_data init; - struct sd_clock *clock; - struct clk *clk; - u32 val; - - clock = kzalloc(sizeof(*clock), GFP_KERNEL); - if (!clock) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = &cpg_sd_clock_ops; - init.flags = CLK_SET_RATE_PARENT; - init.parent_names = &parent_name; - init.num_parents = 1; - - clock->csn.reg = base + offset; - clock->hw.init = &init; - clock->div_table = cpg_sd_div_table; - clock->div_num = ARRAY_SIZE(cpg_sd_div_table); - - if (cpg_quirks & SD_SKIP_FIRST) { - clock->div_table++; - clock->div_num--; - } - - val = readl(clock->csn.reg) & ~CPG_SD_FC_MASK; - val |= CPG_SD_STP_MASK | (clock->div_table[0].val & CPG_SD_FC_MASK); - writel(val, clock->csn.reg); - - clk = clk_register(NULL, &clock->hw); - if (IS_ERR(clk)) - goto free_clock; - - cpg_simple_notifier_register(notifiers, &clock->csn); - return clk; - -free_clock: - kfree(clock); - return clk; -} - struct rpc_clock { struct clk_divider div; struct clk_gate gate; @@ -518,6 +262,12 @@ static struct clk * __init cpg_rpcd2_clk_register(const char *name, static const struct rcar_gen3_cpg_pll_config *cpg_pll_config __initdata; static unsigned int cpg_clk_extalr __initdata; static u32 cpg_mode __initdata; +static u32 cpg_quirks __initdata; + +#define PLL_ERRATA BIT(0) /* Missing PLL0/2/4 post-divider */ +#define RCKCR_CKSEL BIT(1) /* Manual RCLK parent selection */ +#define SD_SKIP_FIRST BIT(2) /* Skip first clock in SD table */ + static const struct soc_device_attribute cpg_quirks_match[] __initconst = { { @@ -613,7 +363,8 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev, case CLK_TYPE_GEN3_SD: return cpg_sd_clk_register(core->name, base, core->offset, - __clk_get_name(parent), notifiers); + __clk_get_name(parent), notifiers, + cpg_quirks & SD_SKIP_FIRST); case CLK_TYPE_GEN3_R: if (cpg_quirks & RCKCR_CKSEL) { diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c index 1c3215dc4877..bffbc3d2faf5 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.c +++ b/drivers/clk/renesas/renesas-cpg-mssr.c @@ -136,8 +136,8 @@ static const u16 srstclr_for_v3u[] = { * @control_regs: Pointer to control registers array * @reset_regs: Pointer to reset registers array * @reset_clear_regs: Pointer to reset clearing registers array - * @smstpcr_saved[].mask: Mask of SMSTPCR[] bits under our control - * @smstpcr_saved[].val: Saved values of SMSTPCR[] + * @smstpcr_saved: [].mask: Mask of SMSTPCR[] bits under our control + * [].val: Saved values of SMSTPCR[] * @clks: Array containing all Core and Module Clocks */ struct cpg_mssr_priv { diff --git a/drivers/clk/rockchip/clk-cpu.c b/drivers/clk/rockchip/clk-cpu.c index 0dc478a19451..fa9027fb1920 100644 --- a/drivers/clk/rockchip/clk-cpu.c +++ b/drivers/clk/rockchip/clk-cpu.c @@ -51,10 +51,6 @@ */ struct rockchip_cpuclk { struct clk_hw hw; - - struct clk_mux cpu_mux; - const struct clk_ops *cpu_mux_ops; - struct clk *alt_parent; void __iomem *reg_base; struct notifier_block clk_nb; diff --git a/drivers/clk/rockchip/clk-half-divider.c b/drivers/clk/rockchip/clk-half-divider.c index ccd5c270c213..64f7faad2148 100644 --- a/drivers/clk/rockchip/clk-half-divider.c +++ b/drivers/clk/rockchip/clk-half-divider.c @@ -145,7 +145,7 @@ static const struct clk_ops clk_half_divider_ops = { .set_rate = clk_half_divider_set_rate, }; -/** +/* * Register a clock branch. * Most clock branches have a form like * diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c index 4c6c9167ef50..fe937bcdb487 100644 --- a/drivers/clk/rockchip/clk-pll.c +++ b/drivers/clk/rockchip/clk-pll.c @@ -97,7 +97,7 @@ static int rockchip_pll_wait_lock(struct rockchip_clk_pll *pll) return ret; } -/** +/* * PLL used in RK3036 */ @@ -358,7 +358,7 @@ static const struct clk_ops rockchip_rk3036_pll_clk_ops = { .init = rockchip_rk3036_pll_init, }; -/** +/* * PLL used in RK3066, RK3188 and RK3288 */ @@ -577,7 +577,7 @@ static const struct clk_ops rockchip_rk3066_pll_clk_ops = { .init = rockchip_rk3066_pll_init, }; -/** +/* * PLL used in RK3399 */ diff --git a/drivers/clk/rockchip/clk-rk3368.c b/drivers/clk/rockchip/clk-rk3368.c index 55443349439b..9a0dab9448db 100644 --- a/drivers/clk/rockchip/clk-rk3368.c +++ b/drivers/clk/rockchip/clk-rk3368.c @@ -474,7 +474,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { COMPOSITE_NODIV(0, "vip_src", mux_pll_src_cpll_gpll_p, 0, RK3368_CLKSEL_CON(21), 15, 1, MFLAGS, RK3368_CLKGATE_CON(4), 5, GFLAGS), - COMPOSITE_NOGATE(0, "sclk_vip_out", mux_vip_out_p, 0, + COMPOSITE_NOGATE(SCLK_VIP_OUT, "sclk_vip_out", mux_vip_out_p, 0, RK3368_CLKSEL_CON(21), 14, 1, MFLAGS, 8, 5, DFLAGS), COMPOSITE_NODIV(SCLK_EDP_24M, "sclk_edp_24m", mux_edp_24m_p, 0, @@ -818,8 +818,8 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { * pclk_vio gates * pclk_vio comes from the exactly same source as hclk_vio */ - GATE(0, "pclk_dphyrx", "hclk_vio", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(14), 8, GFLAGS), - GATE(0, "pclk_dphytx", "hclk_vio", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(14), 8, GFLAGS), + GATE(PCLK_DPHYRX, "pclk_dphyrx", "hclk_vio", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(22), 11, GFLAGS), + GATE(PCLK_DPHYTX0, "pclk_dphytx0", "hclk_vio", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(22), 10, GFLAGS), /* pclk_pd_pmu gates */ GATE(PCLK_PMUGRF, "pclk_pmugrf", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(23), 5, GFLAGS), diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c index 336481bc6cc7..049e5e0b64f6 100644 --- a/drivers/clk/rockchip/clk.c +++ b/drivers/clk/rockchip/clk.c @@ -24,7 +24,7 @@ #include #include "clk.h" -/** +/* * Register a clock branch. * Most clock branches have a form like * @@ -170,7 +170,7 @@ static int rockchip_clk_frac_notifier_cb(struct notifier_block *nb, return notifier_from_errno(ret); } -/** +/* * fractional divider must set that denominator is 20 times larger than * numerator to generate precise clock frequency. */ diff --git a/drivers/clk/sifive/fu540-prci.h b/drivers/clk/sifive/fu540-prci.h index c8271efa7bdc..c220677dc010 100644 --- a/drivers/clk/sifive/fu540-prci.h +++ b/drivers/clk/sifive/fu540-prci.h @@ -13,9 +13,4 @@ extern struct __prci_clock __prci_init_clocks_fu540[NUM_CLOCK_FU540]; -static const struct prci_clk_desc prci_clk_fu540 = { - .clks = __prci_init_clocks_fu540, - .num_clks = ARRAY_SIZE(__prci_init_clocks_fu540), -}; - #endif /* __SIFIVE_CLK_FU540_PRCI_H */ diff --git a/drivers/clk/sifive/sifive-prci.c b/drivers/clk/sifive/sifive-prci.c index c78b042750e2..1490b01ce629 100644 --- a/drivers/clk/sifive/sifive-prci.c +++ b/drivers/clk/sifive/sifive-prci.c @@ -12,6 +12,11 @@ #include "fu540-prci.h" #include "fu740-prci.h" +static const struct prci_clk_desc prci_clk_fu540 = { + .clks = __prci_init_clocks_fu540, + .num_clks = ARRAY_SIZE(__prci_init_clocks_fu540), +}; + /* * Private functions */ diff --git a/drivers/clk/sirf/Makefile b/drivers/clk/sirf/Makefile deleted file mode 100644 index 0ff61f87cddb..000000000000 --- a/drivers/clk/sirf/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for sirf specific clk -# - -obj-$(CONFIG_ARCH_SIRF) += clk-prima2.o clk-atlas6.o clk-atlas7.o diff --git a/drivers/clk/sirf/atlas6.h b/drivers/clk/sirf/atlas6.h deleted file mode 100644 index cb871e30a175..000000000000 --- a/drivers/clk/sirf/atlas6.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#define SIRFSOC_CLKC_CLK_EN0 0x0000 -#define SIRFSOC_CLKC_CLK_EN1 0x0004 -#define SIRFSOC_CLKC_REF_CFG 0x0020 -#define SIRFSOC_CLKC_CPU_CFG 0x0024 -#define SIRFSOC_CLKC_MEM_CFG 0x0028 -#define SIRFSOC_CLKC_MEMDIV_CFG 0x002C -#define SIRFSOC_CLKC_SYS_CFG 0x0030 -#define SIRFSOC_CLKC_IO_CFG 0x0034 -#define SIRFSOC_CLKC_DSP_CFG 0x0038 -#define SIRFSOC_CLKC_GFX_CFG 0x003c -#define SIRFSOC_CLKC_MM_CFG 0x0040 -#define SIRFSOC_CLKC_GFX2D_CFG 0x0040 -#define SIRFSOC_CLKC_LCD_CFG 0x0044 -#define SIRFSOC_CLKC_MMC01_CFG 0x0048 -#define SIRFSOC_CLKC_MMC23_CFG 0x004C -#define SIRFSOC_CLKC_MMC45_CFG 0x0050 -#define SIRFSOC_CLKC_NAND_CFG 0x0054 -#define SIRFSOC_CLKC_NANDDIV_CFG 0x0058 -#define SIRFSOC_CLKC_PLL1_CFG0 0x0080 -#define SIRFSOC_CLKC_PLL2_CFG0 0x0084 -#define SIRFSOC_CLKC_PLL3_CFG0 0x0088 -#define SIRFSOC_CLKC_PLL1_CFG1 0x008c -#define SIRFSOC_CLKC_PLL2_CFG1 0x0090 -#define SIRFSOC_CLKC_PLL3_CFG1 0x0094 -#define SIRFSOC_CLKC_PLL1_CFG2 0x0098 -#define SIRFSOC_CLKC_PLL2_CFG2 0x009c -#define SIRFSOC_CLKC_PLL3_CFG2 0x00A0 -#define SIRFSOC_USBPHY_PLL_CTRL 0x0008 -#define SIRFSOC_USBPHY_PLL_POWERDOWN BIT(1) -#define SIRFSOC_USBPHY_PLL_BYPASS BIT(2) -#define SIRFSOC_USBPHY_PLL_LOCK BIT(3) diff --git a/drivers/clk/sirf/clk-atlas6.c b/drivers/clk/sirf/clk-atlas6.c deleted file mode 100644 index b95483bb6a5e..000000000000 --- a/drivers/clk/sirf/clk-atlas6.c +++ /dev/null @@ -1,150 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Clock tree for CSR SiRFatlasVI - * - * Copyright (c) 2011 - 2014 Cambridge Silicon Radio Limited, a CSR plc group - * company. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "atlas6.h" -#include "clk-common.c" - -static struct clk_dmn clk_mmc01 = { - .regofs = SIRFSOC_CLKC_MMC01_CFG, - .enable_bit = 59, - .hw = { - .init = &clk_mmc01_init, - }, -}; - -static struct clk_dmn clk_mmc23 = { - .regofs = SIRFSOC_CLKC_MMC23_CFG, - .enable_bit = 60, - .hw = { - .init = &clk_mmc23_init, - }, -}; - -static struct clk_dmn clk_mmc45 = { - .regofs = SIRFSOC_CLKC_MMC45_CFG, - .enable_bit = 61, - .hw = { - .init = &clk_mmc45_init, - }, -}; - -static const struct clk_init_data clk_nand_init = { - .name = "nand", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_nand = { - .regofs = SIRFSOC_CLKC_NAND_CFG, - .enable_bit = 34, - .hw = { - .init = &clk_nand_init, - }, -}; - -enum atlas6_clk_index { - /* 0 1 2 3 4 5 6 7 8 9 */ - rtc, osc, pll1, pll2, pll3, mem, sys, security, dsp, gps, - mf, io, cpu, uart0, uart1, uart2, tsc, i2c0, i2c1, spi0, - spi1, pwmc, efuse, pulse, dmac0, dmac1, nand, audio, usp0, usp1, - usp2, vip, gfx, gfx2d, lcd, vpp, mmc01, mmc23, mmc45, usbpll, - usb0, usb1, cphif, maxclk, -}; - -static __initdata struct clk_hw *atlas6_clk_hw_array[maxclk] = { - NULL, /* dummy */ - NULL, - &clk_pll1.hw, - &clk_pll2.hw, - &clk_pll3.hw, - &clk_mem.hw, - &clk_sys.hw, - &clk_security.hw, - &clk_dsp.hw, - &clk_gps.hw, - &clk_mf.hw, - &clk_io.hw, - &clk_cpu.hw, - &clk_uart0.hw, - &clk_uart1.hw, - &clk_uart2.hw, - &clk_tsc.hw, - &clk_i2c0.hw, - &clk_i2c1.hw, - &clk_spi0.hw, - &clk_spi1.hw, - &clk_pwmc.hw, - &clk_efuse.hw, - &clk_pulse.hw, - &clk_dmac0.hw, - &clk_dmac1.hw, - &clk_nand.hw, - &clk_audio.hw, - &clk_usp0.hw, - &clk_usp1.hw, - &clk_usp2.hw, - &clk_vip.hw, - &clk_gfx.hw, - &clk_gfx2d.hw, - &clk_lcd.hw, - &clk_vpp.hw, - &clk_mmc01.hw, - &clk_mmc23.hw, - &clk_mmc45.hw, - &usb_pll_clk_hw, - &clk_usb0.hw, - &clk_usb1.hw, - &clk_cphif.hw, -}; - -static struct clk *atlas6_clks[maxclk]; - -static void __init atlas6_clk_init(struct device_node *np) -{ - struct device_node *rscnp; - int i; - - rscnp = of_find_compatible_node(NULL, NULL, "sirf,prima2-rsc"); - sirfsoc_rsc_vbase = of_iomap(rscnp, 0); - if (!sirfsoc_rsc_vbase) - panic("unable to map rsc registers\n"); - of_node_put(rscnp); - - sirfsoc_clk_vbase = of_iomap(np, 0); - if (!sirfsoc_clk_vbase) - panic("unable to map clkc registers\n"); - - /* These are always available (RTC and 26MHz OSC)*/ - atlas6_clks[rtc] = clk_register_fixed_rate(NULL, "rtc", NULL, 0, 32768); - atlas6_clks[osc] = clk_register_fixed_rate(NULL, "osc", NULL, 0, - 26000000); - - for (i = pll1; i < maxclk; i++) { - atlas6_clks[i] = clk_register(NULL, atlas6_clk_hw_array[i]); - BUG_ON(IS_ERR(atlas6_clks[i])); - } - clk_register_clkdev(atlas6_clks[cpu], NULL, "cpu"); - clk_register_clkdev(atlas6_clks[io], NULL, "io"); - clk_register_clkdev(atlas6_clks[mem], NULL, "mem"); - clk_register_clkdev(atlas6_clks[mem], NULL, "osc"); - - clk_data.clks = atlas6_clks; - clk_data.clk_num = maxclk; - - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); -} -CLK_OF_DECLARE(atlas6_clk, "sirf,atlas6-clkc", atlas6_clk_init); diff --git a/drivers/clk/sirf/clk-atlas7.c b/drivers/clk/sirf/clk-atlas7.c deleted file mode 100644 index 3f57fefd13bb..000000000000 --- a/drivers/clk/sirf/clk-atlas7.c +++ /dev/null @@ -1,1682 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Clock tree for CSR SiRFAtlas7 - * - * Copyright (c) 2014 Cambridge Silicon Radio Limited, a CSR plc group company. - */ - -#include -#include -#include -#include -#include -#include -#include - -#define SIRFSOC_CLKC_MEMPLL_AB_FREQ 0x0000 -#define SIRFSOC_CLKC_MEMPLL_AB_SSC 0x0004 -#define SIRFSOC_CLKC_MEMPLL_AB_CTRL0 0x0008 -#define SIRFSOC_CLKC_MEMPLL_AB_CTRL1 0x000c -#define SIRFSOC_CLKC_MEMPLL_AB_STATUS 0x0010 -#define SIRFSOC_CLKC_MEMPLL_AB_SSRAM_ADDR 0x0014 -#define SIRFSOC_CLKC_MEMPLL_AB_SSRAM_DATA 0x0018 - -#define SIRFSOC_CLKC_CPUPLL_AB_FREQ 0x001c -#define SIRFSOC_CLKC_CPUPLL_AB_SSC 0x0020 -#define SIRFSOC_CLKC_CPUPLL_AB_CTRL0 0x0024 -#define SIRFSOC_CLKC_CPUPLL_AB_CTRL1 0x0028 -#define SIRFSOC_CLKC_CPUPLL_AB_STATUS 0x002c - -#define SIRFSOC_CLKC_SYS0PLL_AB_FREQ 0x0030 -#define SIRFSOC_CLKC_SYS0PLL_AB_SSC 0x0034 -#define SIRFSOC_CLKC_SYS0PLL_AB_CTRL0 0x0038 -#define SIRFSOC_CLKC_SYS0PLL_AB_CTRL1 0x003c -#define SIRFSOC_CLKC_SYS0PLL_AB_STATUS 0x0040 - -#define SIRFSOC_CLKC_SYS1PLL_AB_FREQ 0x0044 -#define SIRFSOC_CLKC_SYS1PLL_AB_SSC 0x0048 -#define SIRFSOC_CLKC_SYS1PLL_AB_CTRL0 0x004c -#define SIRFSOC_CLKC_SYS1PLL_AB_CTRL1 0x0050 -#define SIRFSOC_CLKC_SYS1PLL_AB_STATUS 0x0054 - -#define SIRFSOC_CLKC_SYS2PLL_AB_FREQ 0x0058 -#define SIRFSOC_CLKC_SYS2PLL_AB_SSC 0x005c -#define SIRFSOC_CLKC_SYS2PLL_AB_CTRL0 0x0060 -#define SIRFSOC_CLKC_SYS2PLL_AB_CTRL1 0x0064 -#define SIRFSOC_CLKC_SYS2PLL_AB_STATUS 0x0068 - -#define SIRFSOC_CLKC_SYS3PLL_AB_FREQ 0x006c -#define SIRFSOC_CLKC_SYS3PLL_AB_SSC 0x0070 -#define SIRFSOC_CLKC_SYS3PLL_AB_CTRL0 0x0074 -#define SIRFSOC_CLKC_SYS3PLL_AB_CTRL1 0x0078 -#define SIRFSOC_CLKC_SYS3PLL_AB_STATUS 0x007c - -#define SIRFSOC_ABPLL_CTRL0_SSEN 0x00001000 -#define SIRFSOC_ABPLL_CTRL0_BYPASS 0x00000010 -#define SIRFSOC_ABPLL_CTRL0_RESET 0x00000001 - -#define SIRFSOC_CLKC_AUDIO_DTO_INC 0x0088 -#define SIRFSOC_CLKC_DISP0_DTO_INC 0x008c -#define SIRFSOC_CLKC_DISP1_DTO_INC 0x0090 - -#define SIRFSOC_CLKC_AUDIO_DTO_SRC 0x0094 -#define SIRFSOC_CLKC_AUDIO_DTO_ENA 0x0098 -#define SIRFSOC_CLKC_AUDIO_DTO_DROFF 0x009c - -#define SIRFSOC_CLKC_DISP0_DTO_SRC 0x00a0 -#define SIRFSOC_CLKC_DISP0_DTO_ENA 0x00a4 -#define SIRFSOC_CLKC_DISP0_DTO_DROFF 0x00a8 - -#define SIRFSOC_CLKC_DISP1_DTO_SRC 0x00ac -#define SIRFSOC_CLKC_DISP1_DTO_ENA 0x00b0 -#define SIRFSOC_CLKC_DISP1_DTO_DROFF 0x00b4 - -#define SIRFSOC_CLKC_I2S_CLK_SEL 0x00b8 -#define SIRFSOC_CLKC_I2S_SEL_STAT 0x00bc - -#define SIRFSOC_CLKC_USBPHY_CLKDIV_CFG 0x00c0 -#define SIRFSOC_CLKC_USBPHY_CLKDIV_ENA 0x00c4 -#define SIRFSOC_CLKC_USBPHY_CLK_SEL 0x00c8 -#define SIRFSOC_CLKC_USBPHY_CLK_SEL_STAT 0x00cc - -#define SIRFSOC_CLKC_BTSS_CLKDIV_CFG 0x00d0 -#define SIRFSOC_CLKC_BTSS_CLKDIV_ENA 0x00d4 -#define SIRFSOC_CLKC_BTSS_CLK_SEL 0x00d8 -#define SIRFSOC_CLKC_BTSS_CLK_SEL_STAT 0x00dc - -#define SIRFSOC_CLKC_RGMII_CLKDIV_CFG 0x00e0 -#define SIRFSOC_CLKC_RGMII_CLKDIV_ENA 0x00e4 -#define SIRFSOC_CLKC_RGMII_CLK_SEL 0x00e8 -#define SIRFSOC_CLKC_RGMII_CLK_SEL_STAT 0x00ec - -#define SIRFSOC_CLKC_CPU_CLKDIV_CFG 0x00f0 -#define SIRFSOC_CLKC_CPU_CLKDIV_ENA 0x00f4 -#define SIRFSOC_CLKC_CPU_CLK_SEL 0x00f8 -#define SIRFSOC_CLKC_CPU_CLK_SEL_STAT 0x00fc - -#define SIRFSOC_CLKC_SDPHY01_CLKDIV_CFG 0x0100 -#define SIRFSOC_CLKC_SDPHY01_CLKDIV_ENA 0x0104 -#define SIRFSOC_CLKC_SDPHY01_CLK_SEL 0x0108 -#define SIRFSOC_CLKC_SDPHY01_CLK_SEL_STAT 0x010c - -#define SIRFSOC_CLKC_SDPHY23_CLKDIV_CFG 0x0110 -#define SIRFSOC_CLKC_SDPHY23_CLKDIV_ENA 0x0114 -#define SIRFSOC_CLKC_SDPHY23_CLK_SEL 0x0118 -#define SIRFSOC_CLKC_SDPHY23_CLK_SEL_STAT 0x011c - -#define SIRFSOC_CLKC_SDPHY45_CLKDIV_CFG 0x0120 -#define SIRFSOC_CLKC_SDPHY45_CLKDIV_ENA 0x0124 -#define SIRFSOC_CLKC_SDPHY45_CLK_SEL 0x0128 -#define SIRFSOC_CLKC_SDPHY45_CLK_SEL_STAT 0x012c - -#define SIRFSOC_CLKC_SDPHY67_CLKDIV_CFG 0x0130 -#define SIRFSOC_CLKC_SDPHY67_CLKDIV_ENA 0x0134 -#define SIRFSOC_CLKC_SDPHY67_CLK_SEL 0x0138 -#define SIRFSOC_CLKC_SDPHY67_CLK_SEL_STAT 0x013c - -#define SIRFSOC_CLKC_CAN_CLKDIV_CFG 0x0140 -#define SIRFSOC_CLKC_CAN_CLKDIV_ENA 0x0144 -#define SIRFSOC_CLKC_CAN_CLK_SEL 0x0148 -#define SIRFSOC_CLKC_CAN_CLK_SEL_STAT 0x014c - -#define SIRFSOC_CLKC_DEINT_CLKDIV_CFG 0x0150 -#define SIRFSOC_CLKC_DEINT_CLKDIV_ENA 0x0154 -#define SIRFSOC_CLKC_DEINT_CLK_SEL 0x0158 -#define SIRFSOC_CLKC_DEINT_CLK_SEL_STAT 0x015c - -#define SIRFSOC_CLKC_NAND_CLKDIV_CFG 0x0160 -#define SIRFSOC_CLKC_NAND_CLKDIV_ENA 0x0164 -#define SIRFSOC_CLKC_NAND_CLK_SEL 0x0168 -#define SIRFSOC_CLKC_NAND_CLK_SEL_STAT 0x016c - -#define SIRFSOC_CLKC_DISP0_CLKDIV_CFG 0x0170 -#define SIRFSOC_CLKC_DISP0_CLKDIV_ENA 0x0174 -#define SIRFSOC_CLKC_DISP0_CLK_SEL 0x0178 -#define SIRFSOC_CLKC_DISP0_CLK_SEL_STAT 0x017c - -#define SIRFSOC_CLKC_DISP1_CLKDIV_CFG 0x0180 -#define SIRFSOC_CLKC_DISP1_CLKDIV_ENA 0x0184 -#define SIRFSOC_CLKC_DISP1_CLK_SEL 0x0188 -#define SIRFSOC_CLKC_DISP1_CLK_SEL_STAT 0x018c - -#define SIRFSOC_CLKC_GPU_CLKDIV_CFG 0x0190 -#define SIRFSOC_CLKC_GPU_CLKDIV_ENA 0x0194 -#define SIRFSOC_CLKC_GPU_CLK_SEL 0x0198 -#define SIRFSOC_CLKC_GPU_CLK_SEL_STAT 0x019c - -#define SIRFSOC_CLKC_GNSS_CLKDIV_CFG 0x01a0 -#define SIRFSOC_CLKC_GNSS_CLKDIV_ENA 0x01a4 -#define SIRFSOC_CLKC_GNSS_CLK_SEL 0x01a8 -#define SIRFSOC_CLKC_GNSS_CLK_SEL_STAT 0x01ac - -#define SIRFSOC_CLKC_SHARED_DIVIDER_CFG0 0x01b0 -#define SIRFSOC_CLKC_SHARED_DIVIDER_CFG1 0x01b4 -#define SIRFSOC_CLKC_SHARED_DIVIDER_ENA 0x01b8 - -#define SIRFSOC_CLKC_SYS_CLK_SEL 0x01bc -#define SIRFSOC_CLKC_SYS_CLK_SEL_STAT 0x01c0 -#define SIRFSOC_CLKC_IO_CLK_SEL 0x01c4 -#define SIRFSOC_CLKC_IO_CLK_SEL_STAT 0x01c8 -#define SIRFSOC_CLKC_G2D_CLK_SEL 0x01cc -#define SIRFSOC_CLKC_G2D_CLK_SEL_STAT 0x01d0 -#define SIRFSOC_CLKC_JPENC_CLK_SEL 0x01d4 -#define SIRFSOC_CLKC_JPENC_CLK_SEL_STAT 0x01d8 -#define SIRFSOC_CLKC_VDEC_CLK_SEL 0x01dc -#define SIRFSOC_CLKC_VDEC_CLK_SEL_STAT 0x01e0 -#define SIRFSOC_CLKC_GMAC_CLK_SEL 0x01e4 -#define SIRFSOC_CLKC_GMAC_CLK_SEL_STAT 0x01e8 -#define SIRFSOC_CLKC_USB_CLK_SEL 0x01ec -#define SIRFSOC_CLKC_USB_CLK_SEL_STAT 0x01f0 -#define SIRFSOC_CLKC_KAS_CLK_SEL 0x01f4 -#define SIRFSOC_CLKC_KAS_CLK_SEL_STAT 0x01f8 -#define SIRFSOC_CLKC_SEC_CLK_SEL 0x01fc -#define SIRFSOC_CLKC_SEC_CLK_SEL_STAT 0x0200 -#define SIRFSOC_CLKC_SDR_CLK_SEL 0x0204 -#define SIRFSOC_CLKC_SDR_CLK_SEL_STAT 0x0208 -#define SIRFSOC_CLKC_VIP_CLK_SEL 0x020c -#define SIRFSOC_CLKC_VIP_CLK_SEL_STAT 0x0210 -#define SIRFSOC_CLKC_NOCD_CLK_SEL 0x0214 -#define SIRFSOC_CLKC_NOCD_CLK_SEL_STAT 0x0218 -#define SIRFSOC_CLKC_NOCR_CLK_SEL 0x021c -#define SIRFSOC_CLKC_NOCR_CLK_SEL_STAT 0x0220 -#define SIRFSOC_CLKC_TPIU_CLK_SEL 0x0224 -#define SIRFSOC_CLKC_TPIU_CLK_SEL_STAT 0x0228 - -#define SIRFSOC_CLKC_ROOT_CLK_EN0_SET 0x022c -#define SIRFSOC_CLKC_ROOT_CLK_EN0_CLR 0x0230 -#define SIRFSOC_CLKC_ROOT_CLK_EN0_STAT 0x0234 -#define SIRFSOC_CLKC_ROOT_CLK_EN1_SET 0x0238 -#define SIRFSOC_CLKC_ROOT_CLK_EN1_CLR 0x023c -#define SIRFSOC_CLKC_ROOT_CLK_EN1_STAT 0x0240 - -#define SIRFSOC_CLKC_LEAF_CLK_EN0_SET 0x0244 -#define SIRFSOC_CLKC_LEAF_CLK_EN0_CLR 0x0248 -#define SIRFSOC_CLKC_LEAF_CLK_EN0_STAT 0x024c - -#define SIRFSOC_CLKC_RSTC_A7_SW_RST 0x0308 - -#define SIRFSOC_CLKC_LEAF_CLK_EN1_SET 0x04a0 -#define SIRFSOC_CLKC_LEAF_CLK_EN2_SET 0x04b8 -#define SIRFSOC_CLKC_LEAF_CLK_EN3_SET 0x04d0 -#define SIRFSOC_CLKC_LEAF_CLK_EN4_SET 0x04e8 -#define SIRFSOC_CLKC_LEAF_CLK_EN5_SET 0x0500 -#define SIRFSOC_CLKC_LEAF_CLK_EN6_SET 0x0518 -#define SIRFSOC_CLKC_LEAF_CLK_EN7_SET 0x0530 -#define SIRFSOC_CLKC_LEAF_CLK_EN8_SET 0x0548 - -#define SIRFSOC_NOC_CLK_IDLEREQ_SET 0x02D0 -#define SIRFSOC_NOC_CLK_IDLEREQ_CLR 0x02D4 -#define SIRFSOC_NOC_CLK_SLVRDY_SET 0x02E8 -#define SIRFSOC_NOC_CLK_SLVRDY_CLR 0x02EC -#define SIRFSOC_NOC_CLK_IDLE_STATUS 0x02F4 - -struct clk_pll { - struct clk_hw hw; - u16 regofs; /* register offset */ -}; -#define to_pllclk(_hw) container_of(_hw, struct clk_pll, hw) - -struct clk_dto { - struct clk_hw hw; - u16 inc_offset; /* dto increment offset */ - u16 src_offset; /* dto src offset */ -}; -#define to_dtoclk(_hw) container_of(_hw, struct clk_dto, hw) - -enum clk_unit_type { - CLK_UNIT_NOC_OTHER, - CLK_UNIT_NOC_CLOCK, - CLK_UNIT_NOC_SOCKET, -}; - -struct clk_unit { - struct clk_hw hw; - u16 regofs; - u16 bit; - u32 type; - u8 idle_bit; - spinlock_t *lock; -}; -#define to_unitclk(_hw) container_of(_hw, struct clk_unit, hw) - -struct atlas7_div_init_data { - const char *div_name; - const char *parent_name; - const char *gate_name; - unsigned long flags; - u8 divider_flags; - u8 gate_flags; - u32 div_offset; - u8 shift; - u8 width; - u32 gate_offset; - u8 gate_bit; - spinlock_t *lock; -}; - -struct atlas7_mux_init_data { - const char *mux_name; - const char * const *parent_names; - u8 parent_num; - unsigned long flags; - u8 mux_flags; - u32 mux_offset; - u8 shift; - u8 width; -}; - -struct atlas7_unit_init_data { - u32 index; - const char *unit_name; - const char *parent_name; - unsigned long flags; - u32 regofs; - u8 bit; - u32 type; - u8 idle_bit; - spinlock_t *lock; -}; - -struct atlas7_reset_desc { - const char *name; - u32 clk_ofs; - u8 clk_bit; - u32 rst_ofs; - u8 rst_bit; - spinlock_t *lock; -}; - -static void __iomem *sirfsoc_clk_vbase; -static struct clk_onecell_data clk_data; - -static const struct clk_div_table pll_div_table[] = { - { .val = 0, .div = 1 }, - { .val = 1, .div = 2 }, - { .val = 2, .div = 4 }, - { .val = 3, .div = 8 }, - { .val = 4, .div = 16 }, - { .val = 5, .div = 32 }, -}; - -static DEFINE_SPINLOCK(cpupll_ctrl1_lock); -static DEFINE_SPINLOCK(mempll_ctrl1_lock); -static DEFINE_SPINLOCK(sys0pll_ctrl1_lock); -static DEFINE_SPINLOCK(sys1pll_ctrl1_lock); -static DEFINE_SPINLOCK(sys2pll_ctrl1_lock); -static DEFINE_SPINLOCK(sys3pll_ctrl1_lock); -static DEFINE_SPINLOCK(usbphy_div_lock); -static DEFINE_SPINLOCK(btss_div_lock); -static DEFINE_SPINLOCK(rgmii_div_lock); -static DEFINE_SPINLOCK(cpu_div_lock); -static DEFINE_SPINLOCK(sdphy01_div_lock); -static DEFINE_SPINLOCK(sdphy23_div_lock); -static DEFINE_SPINLOCK(sdphy45_div_lock); -static DEFINE_SPINLOCK(sdphy67_div_lock); -static DEFINE_SPINLOCK(can_div_lock); -static DEFINE_SPINLOCK(deint_div_lock); -static DEFINE_SPINLOCK(nand_div_lock); -static DEFINE_SPINLOCK(disp0_div_lock); -static DEFINE_SPINLOCK(disp1_div_lock); -static DEFINE_SPINLOCK(gpu_div_lock); -static DEFINE_SPINLOCK(gnss_div_lock); -/* gate register shared */ -static DEFINE_SPINLOCK(share_div_lock); -static DEFINE_SPINLOCK(root0_gate_lock); -static DEFINE_SPINLOCK(root1_gate_lock); -static DEFINE_SPINLOCK(leaf0_gate_lock); -static DEFINE_SPINLOCK(leaf1_gate_lock); -static DEFINE_SPINLOCK(leaf2_gate_lock); -static DEFINE_SPINLOCK(leaf3_gate_lock); -static DEFINE_SPINLOCK(leaf4_gate_lock); -static DEFINE_SPINLOCK(leaf5_gate_lock); -static DEFINE_SPINLOCK(leaf6_gate_lock); -static DEFINE_SPINLOCK(leaf7_gate_lock); -static DEFINE_SPINLOCK(leaf8_gate_lock); - -static inline unsigned long clkc_readl(unsigned reg) -{ - return readl(sirfsoc_clk_vbase + reg); -} - -static inline void clkc_writel(u32 val, unsigned reg) -{ - writel(val, sirfsoc_clk_vbase + reg); -} - -/* -* ABPLL -* integer mode: Fvco = Fin * 2 * NF / NR -* Spread Spectrum mode: Fvco = Fin * SSN / NR -* SSN = 2^24 / (256 * ((ssdiv >> ssdepth) << ssdepth) + (ssmod << ssdepth)) -*/ -static unsigned long pll_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - unsigned long fin = parent_rate; - struct clk_pll *clk = to_pllclk(hw); - u64 rate; - u32 regctrl0 = clkc_readl(clk->regofs + SIRFSOC_CLKC_MEMPLL_AB_CTRL0 - - SIRFSOC_CLKC_MEMPLL_AB_FREQ); - u32 regfreq = clkc_readl(clk->regofs); - u32 regssc = clkc_readl(clk->regofs + SIRFSOC_CLKC_MEMPLL_AB_SSC - - SIRFSOC_CLKC_MEMPLL_AB_FREQ); - u32 nr = (regfreq >> 16 & (BIT(3) - 1)) + 1; - u32 nf = (regfreq & (BIT(9) - 1)) + 1; - u32 ssdiv = regssc >> 8 & (BIT(12) - 1); - u32 ssdepth = regssc >> 20 & (BIT(2) - 1); - u32 ssmod = regssc & (BIT(8) - 1); - - if (regctrl0 & SIRFSOC_ABPLL_CTRL0_BYPASS) - return fin; - - if (regctrl0 & SIRFSOC_ABPLL_CTRL0_SSEN) { - rate = fin; - rate *= 1 << 24; - do_div(rate, nr); - do_div(rate, (256 * ((ssdiv >> ssdepth) << ssdepth) - + (ssmod << ssdepth))); - } else { - rate = 2 * fin; - rate *= nf; - do_div(rate, nr); - } - return rate; -} - -static const struct clk_ops ab_pll_ops = { - .recalc_rate = pll_clk_recalc_rate, -}; - -static const char * const pll_clk_parents[] = { - "xin", -}; - -static const struct clk_init_data clk_cpupll_init = { - .name = "cpupll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_cpupll = { - .regofs = SIRFSOC_CLKC_CPUPLL_AB_FREQ, - .hw = { - .init = &clk_cpupll_init, - }, -}; - -static const struct clk_init_data clk_mempll_init = { - .name = "mempll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_mempll = { - .regofs = SIRFSOC_CLKC_MEMPLL_AB_FREQ, - .hw = { - .init = &clk_mempll_init, - }, -}; - -static const struct clk_init_data clk_sys0pll_init = { - .name = "sys0pll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_sys0pll = { - .regofs = SIRFSOC_CLKC_SYS0PLL_AB_FREQ, - .hw = { - .init = &clk_sys0pll_init, - }, -}; - -static const struct clk_init_data clk_sys1pll_init = { - .name = "sys1pll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_sys1pll = { - .regofs = SIRFSOC_CLKC_SYS1PLL_AB_FREQ, - .hw = { - .init = &clk_sys1pll_init, - }, -}; - -static const struct clk_init_data clk_sys2pll_init = { - .name = "sys2pll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_sys2pll = { - .regofs = SIRFSOC_CLKC_SYS2PLL_AB_FREQ, - .hw = { - .init = &clk_sys2pll_init, - }, -}; - -static const struct clk_init_data clk_sys3pll_init = { - .name = "sys3pll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_sys3pll = { - .regofs = SIRFSOC_CLKC_SYS3PLL_AB_FREQ, - .hw = { - .init = &clk_sys3pll_init, - }, -}; - -/* - * DTO in clkc, default enable double resolution mode - * double resolution mode:fout = fin * finc / 2^29 - * normal mode:fout = fin * finc / 2^28 - */ -#define DTO_RESL_DOUBLE (1ULL << 29) -#define DTO_RESL_NORMAL (1ULL << 28) - -static int dto_clk_is_enabled(struct clk_hw *hw) -{ - struct clk_dto *clk = to_dtoclk(hw); - int reg; - - reg = clk->src_offset + SIRFSOC_CLKC_AUDIO_DTO_ENA - SIRFSOC_CLKC_AUDIO_DTO_SRC; - - return !!(clkc_readl(reg) & BIT(0)); -} - -static int dto_clk_enable(struct clk_hw *hw) -{ - u32 val, reg; - struct clk_dto *clk = to_dtoclk(hw); - - reg = clk->src_offset + SIRFSOC_CLKC_AUDIO_DTO_ENA - SIRFSOC_CLKC_AUDIO_DTO_SRC; - - val = clkc_readl(reg) | BIT(0); - clkc_writel(val, reg); - return 0; -} - -static void dto_clk_disable(struct clk_hw *hw) -{ - u32 val, reg; - struct clk_dto *clk = to_dtoclk(hw); - - reg = clk->src_offset + SIRFSOC_CLKC_AUDIO_DTO_ENA - SIRFSOC_CLKC_AUDIO_DTO_SRC; - - val = clkc_readl(reg) & ~BIT(0); - clkc_writel(val, reg); -} - -static unsigned long dto_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - u64 rate = parent_rate; - struct clk_dto *clk = to_dtoclk(hw); - u32 finc = clkc_readl(clk->inc_offset); - u32 droff = clkc_readl(clk->src_offset + SIRFSOC_CLKC_AUDIO_DTO_DROFF - SIRFSOC_CLKC_AUDIO_DTO_SRC); - - rate *= finc; - if (droff & BIT(0)) - /* Double resolution off */ - do_div(rate, DTO_RESL_NORMAL); - else - do_div(rate, DTO_RESL_DOUBLE); - - return rate; -} - -static long dto_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate) -{ - u64 dividend = rate * DTO_RESL_DOUBLE; - - do_div(dividend, *parent_rate); - dividend *= *parent_rate; - do_div(dividend, DTO_RESL_DOUBLE); - - return dividend; -} - -static int dto_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - u64 dividend = rate * DTO_RESL_DOUBLE; - struct clk_dto *clk = to_dtoclk(hw); - - do_div(dividend, parent_rate); - clkc_writel(0, clk->src_offset + SIRFSOC_CLKC_AUDIO_DTO_DROFF - SIRFSOC_CLKC_AUDIO_DTO_SRC); - clkc_writel(dividend, clk->inc_offset); - - return 0; -} - -static u8 dto_clk_get_parent(struct clk_hw *hw) -{ - struct clk_dto *clk = to_dtoclk(hw); - - return clkc_readl(clk->src_offset); -} - -/* - * dto need CLK_SET_PARENT_GATE - */ -static int dto_clk_set_parent(struct clk_hw *hw, u8 index) -{ - struct clk_dto *clk = to_dtoclk(hw); - - clkc_writel(index, clk->src_offset); - return 0; -} - -static const struct clk_ops dto_ops = { - .is_enabled = dto_clk_is_enabled, - .enable = dto_clk_enable, - .disable = dto_clk_disable, - .recalc_rate = dto_clk_recalc_rate, - .round_rate = dto_clk_round_rate, - .set_rate = dto_clk_set_rate, - .get_parent = dto_clk_get_parent, - .set_parent = dto_clk_set_parent, -}; - -/* dto parent clock as syspllvco/clk1 */ -static const char * const audiodto_clk_parents[] = { - "sys0pll_clk1", - "sys1pll_clk1", - "sys3pll_clk1", -}; - -static const struct clk_init_data clk_audiodto_init = { - .name = "audio_dto", - .ops = &dto_ops, - .parent_names = audiodto_clk_parents, - .num_parents = ARRAY_SIZE(audiodto_clk_parents), -}; - -static struct clk_dto clk_audio_dto = { - .inc_offset = SIRFSOC_CLKC_AUDIO_DTO_INC, - .src_offset = SIRFSOC_CLKC_AUDIO_DTO_SRC, - .hw = { - .init = &clk_audiodto_init, - }, -}; - -static const char * const disp0dto_clk_parents[] = { - "sys0pll_clk1", - "sys1pll_clk1", - "sys3pll_clk1", -}; - -static const struct clk_init_data clk_disp0dto_init = { - .name = "disp0_dto", - .ops = &dto_ops, - .parent_names = disp0dto_clk_parents, - .num_parents = ARRAY_SIZE(disp0dto_clk_parents), -}; - -static struct clk_dto clk_disp0_dto = { - .inc_offset = SIRFSOC_CLKC_DISP0_DTO_INC, - .src_offset = SIRFSOC_CLKC_DISP0_DTO_SRC, - .hw = { - .init = &clk_disp0dto_init, - }, -}; - -static const char * const disp1dto_clk_parents[] = { - "sys0pll_clk1", - "sys1pll_clk1", - "sys3pll_clk1", -}; - -static const struct clk_init_data clk_disp1dto_init = { - .name = "disp1_dto", - .ops = &dto_ops, - .parent_names = disp1dto_clk_parents, - .num_parents = ARRAY_SIZE(disp1dto_clk_parents), -}; - -static struct clk_dto clk_disp1_dto = { - .inc_offset = SIRFSOC_CLKC_DISP1_DTO_INC, - .src_offset = SIRFSOC_CLKC_DISP1_DTO_SRC, - .hw = { - .init = &clk_disp1dto_init, - }, -}; - -static struct atlas7_div_init_data divider_list[] __initdata = { - /* div_name, parent_name, gate_name, clk_flag, divider_flag, gate_flag, div_offset, shift, wdith, gate_offset, bit_enable, lock */ - { "sys0pll_qa1", "sys0pll_fixdiv", "sys0pll_a1", 0, 0, 0, SIRFSOC_CLKC_USBPHY_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_USBPHY_CLKDIV_ENA, 0, &usbphy_div_lock }, - { "sys1pll_qa1", "sys1pll_fixdiv", "sys1pll_a1", 0, 0, 0, SIRFSOC_CLKC_USBPHY_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_USBPHY_CLKDIV_ENA, 4, &usbphy_div_lock }, - { "sys2pll_qa1", "sys2pll_fixdiv", "sys2pll_a1", 0, 0, 0, SIRFSOC_CLKC_USBPHY_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_USBPHY_CLKDIV_ENA, 8, &usbphy_div_lock }, - { "sys3pll_qa1", "sys3pll_fixdiv", "sys3pll_a1", 0, 0, 0, SIRFSOC_CLKC_USBPHY_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_USBPHY_CLKDIV_ENA, 12, &usbphy_div_lock }, - { "sys0pll_qa2", "sys0pll_fixdiv", "sys0pll_a2", 0, 0, 0, SIRFSOC_CLKC_BTSS_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_BTSS_CLKDIV_ENA, 0, &btss_div_lock }, - { "sys1pll_qa2", "sys1pll_fixdiv", "sys1pll_a2", 0, 0, 0, SIRFSOC_CLKC_BTSS_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_BTSS_CLKDIV_ENA, 4, &btss_div_lock }, - { "sys2pll_qa2", "sys2pll_fixdiv", "sys2pll_a2", 0, 0, 0, SIRFSOC_CLKC_BTSS_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_BTSS_CLKDIV_ENA, 8, &btss_div_lock }, - { "sys3pll_qa2", "sys3pll_fixdiv", "sys3pll_a2", 0, 0, 0, SIRFSOC_CLKC_BTSS_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_BTSS_CLKDIV_ENA, 12, &btss_div_lock }, - { "sys0pll_qa3", "sys0pll_fixdiv", "sys0pll_a3", 0, 0, 0, SIRFSOC_CLKC_RGMII_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_RGMII_CLKDIV_ENA, 0, &rgmii_div_lock }, - { "sys1pll_qa3", "sys1pll_fixdiv", "sys1pll_a3", 0, 0, 0, SIRFSOC_CLKC_RGMII_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_RGMII_CLKDIV_ENA, 4, &rgmii_div_lock }, - { "sys2pll_qa3", "sys2pll_fixdiv", "sys2pll_a3", 0, 0, 0, SIRFSOC_CLKC_RGMII_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_RGMII_CLKDIV_ENA, 8, &rgmii_div_lock }, - { "sys3pll_qa3", "sys3pll_fixdiv", "sys3pll_a3", 0, 0, 0, SIRFSOC_CLKC_RGMII_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_RGMII_CLKDIV_ENA, 12, &rgmii_div_lock }, - { "sys0pll_qa4", "sys0pll_fixdiv", "sys0pll_a4", 0, 0, 0, SIRFSOC_CLKC_CPU_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_CPU_CLKDIV_ENA, 0, &cpu_div_lock }, - { "sys1pll_qa4", "sys1pll_fixdiv", "sys1pll_a4", 0, 0, CLK_IGNORE_UNUSED, SIRFSOC_CLKC_CPU_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_CPU_CLKDIV_ENA, 4, &cpu_div_lock }, - { "sys0pll_qa5", "sys0pll_fixdiv", "sys0pll_a5", 0, 0, 0, SIRFSOC_CLKC_SDPHY01_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_SDPHY01_CLKDIV_ENA, 0, &sdphy01_div_lock }, - { "sys1pll_qa5", "sys1pll_fixdiv", "sys1pll_a5", 0, 0, 0, SIRFSOC_CLKC_SDPHY01_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_SDPHY01_CLKDIV_ENA, 4, &sdphy01_div_lock }, - { "sys2pll_qa5", "sys2pll_fixdiv", "sys2pll_a5", 0, 0, 0, SIRFSOC_CLKC_SDPHY01_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_SDPHY01_CLKDIV_ENA, 8, &sdphy01_div_lock }, - { "sys3pll_qa5", "sys3pll_fixdiv", "sys3pll_a5", 0, 0, 0, SIRFSOC_CLKC_SDPHY01_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_SDPHY01_CLKDIV_ENA, 12, &sdphy01_div_lock }, - { "sys0pll_qa6", "sys0pll_fixdiv", "sys0pll_a6", 0, 0, 0, SIRFSOC_CLKC_SDPHY23_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_SDPHY23_CLKDIV_ENA, 0, &sdphy23_div_lock }, - { "sys1pll_qa6", "sys1pll_fixdiv", "sys1pll_a6", 0, 0, 0, SIRFSOC_CLKC_SDPHY23_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_SDPHY23_CLKDIV_ENA, 4, &sdphy23_div_lock }, - { "sys2pll_qa6", "sys2pll_fixdiv", "sys2pll_a6", 0, 0, 0, SIRFSOC_CLKC_SDPHY23_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_SDPHY23_CLKDIV_ENA, 8, &sdphy23_div_lock }, - { "sys3pll_qa6", "sys3pll_fixdiv", "sys3pll_a6", 0, 0, 0, SIRFSOC_CLKC_SDPHY23_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_SDPHY23_CLKDIV_ENA, 12, &sdphy23_div_lock }, - { "sys0pll_qa7", "sys0pll_fixdiv", "sys0pll_a7", 0, 0, 0, SIRFSOC_CLKC_SDPHY45_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_SDPHY45_CLKDIV_ENA, 0, &sdphy45_div_lock }, - { "sys1pll_qa7", "sys1pll_fixdiv", "sys1pll_a7", 0, 0, 0, SIRFSOC_CLKC_SDPHY45_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_SDPHY45_CLKDIV_ENA, 4, &sdphy45_div_lock }, - { "sys2pll_qa7", "sys2pll_fixdiv", "sys2pll_a7", 0, 0, 0, SIRFSOC_CLKC_SDPHY45_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_SDPHY45_CLKDIV_ENA, 8, &sdphy45_div_lock }, - { "sys3pll_qa7", "sys3pll_fixdiv", "sys3pll_a7", 0, 0, 0, SIRFSOC_CLKC_SDPHY45_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_SDPHY45_CLKDIV_ENA, 12, &sdphy45_div_lock }, - { "sys0pll_qa8", "sys0pll_fixdiv", "sys0pll_a8", 0, 0, 0, SIRFSOC_CLKC_SDPHY67_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_SDPHY67_CLKDIV_ENA, 0, &sdphy67_div_lock }, - { "sys1pll_qa8", "sys1pll_fixdiv", "sys1pll_a8", 0, 0, 0, SIRFSOC_CLKC_SDPHY67_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_SDPHY67_CLKDIV_ENA, 4, &sdphy67_div_lock }, - { "sys2pll_qa8", "sys2pll_fixdiv", "sys2pll_a8", 0, 0, 0, SIRFSOC_CLKC_SDPHY67_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_SDPHY67_CLKDIV_ENA, 8, &sdphy67_div_lock }, - { "sys3pll_qa8", "sys3pll_fixdiv", "sys3pll_a8", 0, 0, 0, SIRFSOC_CLKC_SDPHY67_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_SDPHY67_CLKDIV_ENA, 12, &sdphy67_div_lock }, - { "sys0pll_qa9", "sys0pll_fixdiv", "sys0pll_a9", 0, 0, 0, SIRFSOC_CLKC_CAN_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_CAN_CLKDIV_ENA, 0, &can_div_lock }, - { "sys1pll_qa9", "sys1pll_fixdiv", "sys1pll_a9", 0, 0, 0, SIRFSOC_CLKC_CAN_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_CAN_CLKDIV_ENA, 4, &can_div_lock }, - { "sys2pll_qa9", "sys2pll_fixdiv", "sys2pll_a9", 0, 0, 0, SIRFSOC_CLKC_CAN_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_CAN_CLKDIV_ENA, 8, &can_div_lock }, - { "sys3pll_qa9", "sys3pll_fixdiv", "sys3pll_a9", 0, 0, 0, SIRFSOC_CLKC_CAN_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_CAN_CLKDIV_ENA, 12, &can_div_lock }, - { "sys0pll_qa10", "sys0pll_fixdiv", "sys0pll_a10", 0, 0, 0, SIRFSOC_CLKC_DEINT_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_DEINT_CLKDIV_ENA, 0, &deint_div_lock }, - { "sys1pll_qa10", "sys1pll_fixdiv", "sys1pll_a10", 0, 0, 0, SIRFSOC_CLKC_DEINT_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_DEINT_CLKDIV_ENA, 4, &deint_div_lock }, - { "sys2pll_qa10", "sys2pll_fixdiv", "sys2pll_a10", 0, 0, 0, SIRFSOC_CLKC_DEINT_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_DEINT_CLKDIV_ENA, 8, &deint_div_lock }, - { "sys3pll_qa10", "sys3pll_fixdiv", "sys3pll_a10", 0, 0, 0, SIRFSOC_CLKC_DEINT_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_DEINT_CLKDIV_ENA, 12, &deint_div_lock }, - { "sys0pll_qa11", "sys0pll_fixdiv", "sys0pll_a11", 0, 0, 0, SIRFSOC_CLKC_NAND_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_NAND_CLKDIV_ENA, 0, &nand_div_lock }, - { "sys1pll_qa11", "sys1pll_fixdiv", "sys1pll_a11", 0, 0, 0, SIRFSOC_CLKC_NAND_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_NAND_CLKDIV_ENA, 4, &nand_div_lock }, - { "sys2pll_qa11", "sys2pll_fixdiv", "sys2pll_a11", 0, 0, 0, SIRFSOC_CLKC_NAND_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_NAND_CLKDIV_ENA, 8, &nand_div_lock }, - { "sys3pll_qa11", "sys3pll_fixdiv", "sys3pll_a11", 0, 0, 0, SIRFSOC_CLKC_NAND_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_NAND_CLKDIV_ENA, 12, &nand_div_lock }, - { "sys0pll_qa12", "sys0pll_fixdiv", "sys0pll_a12", 0, 0, 0, SIRFSOC_CLKC_DISP0_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_DISP0_CLKDIV_ENA, 0, &disp0_div_lock }, - { "sys1pll_qa12", "sys1pll_fixdiv", "sys1pll_a12", 0, 0, 0, SIRFSOC_CLKC_DISP0_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_DISP0_CLKDIV_ENA, 4, &disp0_div_lock }, - { "sys2pll_qa12", "sys2pll_fixdiv", "sys2pll_a12", 0, 0, 0, SIRFSOC_CLKC_DISP0_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_DISP0_CLKDIV_ENA, 8, &disp0_div_lock }, - { "sys3pll_qa12", "sys3pll_fixdiv", "sys3pll_a12", 0, 0, 0, SIRFSOC_CLKC_DISP0_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_DISP0_CLKDIV_ENA, 12, &disp0_div_lock }, - { "sys0pll_qa13", "sys0pll_fixdiv", "sys0pll_a13", 0, 0, 0, SIRFSOC_CLKC_DISP1_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_DISP1_CLKDIV_ENA, 0, &disp1_div_lock }, - { "sys1pll_qa13", "sys1pll_fixdiv", "sys1pll_a13", 0, 0, 0, SIRFSOC_CLKC_DISP1_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_DISP1_CLKDIV_ENA, 4, &disp1_div_lock }, - { "sys2pll_qa13", "sys2pll_fixdiv", "sys2pll_a13", 0, 0, 0, SIRFSOC_CLKC_DISP1_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_DISP1_CLKDIV_ENA, 8, &disp1_div_lock }, - { "sys3pll_qa13", "sys3pll_fixdiv", "sys3pll_a13", 0, 0, 0, SIRFSOC_CLKC_DISP1_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_DISP1_CLKDIV_ENA, 12, &disp1_div_lock }, - { "sys0pll_qa14", "sys0pll_fixdiv", "sys0pll_a14", 0, 0, 0, SIRFSOC_CLKC_GPU_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_GPU_CLKDIV_ENA, 0, &gpu_div_lock }, - { "sys1pll_qa14", "sys1pll_fixdiv", "sys1pll_a14", 0, 0, 0, SIRFSOC_CLKC_GPU_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_GPU_CLKDIV_ENA, 4, &gpu_div_lock }, - { "sys2pll_qa14", "sys2pll_fixdiv", "sys2pll_a14", 0, 0, 0, SIRFSOC_CLKC_GPU_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_GPU_CLKDIV_ENA, 8, &gpu_div_lock }, - { "sys3pll_qa14", "sys3pll_fixdiv", "sys3pll_a14", 0, 0, 0, SIRFSOC_CLKC_GPU_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_GPU_CLKDIV_ENA, 12, &gpu_div_lock }, - { "sys0pll_qa15", "sys0pll_fixdiv", "sys0pll_a15", 0, 0, 0, SIRFSOC_CLKC_GNSS_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_GNSS_CLKDIV_ENA, 0, &gnss_div_lock }, - { "sys1pll_qa15", "sys1pll_fixdiv", "sys1pll_a15", 0, 0, 0, SIRFSOC_CLKC_GNSS_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_GNSS_CLKDIV_ENA, 4, &gnss_div_lock }, - { "sys2pll_qa15", "sys2pll_fixdiv", "sys2pll_a15", 0, 0, 0, SIRFSOC_CLKC_GNSS_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_GNSS_CLKDIV_ENA, 8, &gnss_div_lock }, - { "sys3pll_qa15", "sys3pll_fixdiv", "sys3pll_a15", 0, 0, 0, SIRFSOC_CLKC_GNSS_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_GNSS_CLKDIV_ENA, 12, &gnss_div_lock }, - { "sys1pll_qa18", "sys1pll_fixdiv", "sys1pll_a18", 0, 0, 0, SIRFSOC_CLKC_SHARED_DIVIDER_CFG0, 24, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 12, &share_div_lock }, - { "sys1pll_qa19", "sys1pll_fixdiv", "sys1pll_a19", 0, 0, CLK_IGNORE_UNUSED, SIRFSOC_CLKC_SHARED_DIVIDER_CFG0, 16, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 8, &share_div_lock }, - { "sys1pll_qa20", "sys1pll_fixdiv", "sys1pll_a20", 0, 0, 0, SIRFSOC_CLKC_SHARED_DIVIDER_CFG0, 8, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 4, &share_div_lock }, - { "sys2pll_qa20", "sys2pll_fixdiv", "sys2pll_a20", 0, 0, 0, SIRFSOC_CLKC_SHARED_DIVIDER_CFG0, 0, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 0, &share_div_lock }, - { "sys1pll_qa17", "sys1pll_fixdiv", "sys1pll_a17", 0, 0, CLK_IGNORE_UNUSED, SIRFSOC_CLKC_SHARED_DIVIDER_CFG1, 8, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 20, &share_div_lock }, - { "sys0pll_qa20", "sys0pll_fixdiv", "sys0pll_a20", 0, 0, 0, SIRFSOC_CLKC_SHARED_DIVIDER_CFG1, 0, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 16, &share_div_lock }, -}; - -static const char * const i2s_clk_parents[] = { - "xin", - "xinw", - "audio_dto", - /* "pwm_i2s01" */ -}; - -static const char * const usbphy_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a1", - "sys1pll_a1", - "sys2pll_a1", - "sys3pll_a1", -}; - -static const char * const btss_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a2", - "sys1pll_a2", - "sys2pll_a2", - "sys3pll_a2", -}; - -static const char * const rgmii_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a3", - "sys1pll_a3", - "sys2pll_a3", - "sys3pll_a3", -}; - -static const char * const cpu_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a4", - "sys1pll_a4", - "cpupll_clk1", -}; - -static const char * const sdphy01_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a5", - "sys1pll_a5", - "sys2pll_a5", - "sys3pll_a5", -}; - -static const char * const sdphy23_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a6", - "sys1pll_a6", - "sys2pll_a6", - "sys3pll_a6", -}; - -static const char * const sdphy45_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a7", - "sys1pll_a7", - "sys2pll_a7", - "sys3pll_a7", -}; - -static const char * const sdphy67_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a8", - "sys1pll_a8", - "sys2pll_a8", - "sys3pll_a8", -}; - -static const char * const can_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a9", - "sys1pll_a9", - "sys2pll_a9", - "sys3pll_a9", -}; - -static const char * const deint_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a10", - "sys1pll_a10", - "sys2pll_a10", - "sys3pll_a10", -}; - -static const char * const nand_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a11", - "sys1pll_a11", - "sys2pll_a11", - "sys3pll_a11", -}; - -static const char * const disp0_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a12", - "sys1pll_a12", - "sys2pll_a12", - "sys3pll_a12", - "disp0_dto", -}; - -static const char * const disp1_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a13", - "sys1pll_a13", - "sys2pll_a13", - "sys3pll_a13", - "disp1_dto", -}; - -static const char * const gpu_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a14", - "sys1pll_a14", - "sys2pll_a14", - "sys3pll_a14", -}; - -static const char * const gnss_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a15", - "sys1pll_a15", - "sys2pll_a15", - "sys3pll_a15", -}; - -static const char * const sys_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const io_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const g2d_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const jpenc_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const vdec_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const gmac_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const usb_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const kas_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const sec_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const sdr_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const vip_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const nocd_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const nocr_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const tpiu_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static struct atlas7_mux_init_data mux_list[] __initdata = { - /* mux_name, parent_names, parent_num, flags, mux_flags, mux_offset, shift, width */ - { "i2s_mux", i2s_clk_parents, ARRAY_SIZE(i2s_clk_parents), 0, 0, SIRFSOC_CLKC_I2S_CLK_SEL, 0, 2 }, - { "usbphy_mux", usbphy_clk_parents, ARRAY_SIZE(usbphy_clk_parents), 0, 0, SIRFSOC_CLKC_I2S_CLK_SEL, 0, 3 }, - { "btss_mux", btss_clk_parents, ARRAY_SIZE(btss_clk_parents), 0, 0, SIRFSOC_CLKC_BTSS_CLK_SEL, 0, 3 }, - { "rgmii_mux", rgmii_clk_parents, ARRAY_SIZE(rgmii_clk_parents), 0, 0, SIRFSOC_CLKC_RGMII_CLK_SEL, 0, 3 }, - { "cpu_mux", cpu_clk_parents, ARRAY_SIZE(cpu_clk_parents), 0, 0, SIRFSOC_CLKC_CPU_CLK_SEL, 0, 3 }, - { "sdphy01_mux", sdphy01_clk_parents, ARRAY_SIZE(sdphy01_clk_parents), 0, 0, SIRFSOC_CLKC_SDPHY01_CLK_SEL, 0, 3 }, - { "sdphy23_mux", sdphy23_clk_parents, ARRAY_SIZE(sdphy23_clk_parents), 0, 0, SIRFSOC_CLKC_SDPHY23_CLK_SEL, 0, 3 }, - { "sdphy45_mux", sdphy45_clk_parents, ARRAY_SIZE(sdphy45_clk_parents), 0, 0, SIRFSOC_CLKC_SDPHY45_CLK_SEL, 0, 3 }, - { "sdphy67_mux", sdphy67_clk_parents, ARRAY_SIZE(sdphy67_clk_parents), 0, 0, SIRFSOC_CLKC_SDPHY67_CLK_SEL, 0, 3 }, - { "can_mux", can_clk_parents, ARRAY_SIZE(can_clk_parents), 0, 0, SIRFSOC_CLKC_CAN_CLK_SEL, 0, 3 }, - { "deint_mux", deint_clk_parents, ARRAY_SIZE(deint_clk_parents), 0, 0, SIRFSOC_CLKC_DEINT_CLK_SEL, 0, 3 }, - { "nand_mux", nand_clk_parents, ARRAY_SIZE(nand_clk_parents), 0, 0, SIRFSOC_CLKC_NAND_CLK_SEL, 0, 3 }, - { "disp0_mux", disp0_clk_parents, ARRAY_SIZE(disp0_clk_parents), 0, 0, SIRFSOC_CLKC_DISP0_CLK_SEL, 0, 3 }, - { "disp1_mux", disp1_clk_parents, ARRAY_SIZE(disp1_clk_parents), 0, 0, SIRFSOC_CLKC_DISP1_CLK_SEL, 0, 3 }, - { "gpu_mux", gpu_clk_parents, ARRAY_SIZE(gpu_clk_parents), 0, 0, SIRFSOC_CLKC_GPU_CLK_SEL, 0, 3 }, - { "gnss_mux", gnss_clk_parents, ARRAY_SIZE(gnss_clk_parents), 0, 0, SIRFSOC_CLKC_GNSS_CLK_SEL, 0, 3 }, - { "sys_mux", sys_clk_parents, ARRAY_SIZE(sys_clk_parents), 0, 0, SIRFSOC_CLKC_SYS_CLK_SEL, 0, 3 }, - { "io_mux", io_clk_parents, ARRAY_SIZE(io_clk_parents), 0, 0, SIRFSOC_CLKC_IO_CLK_SEL, 0, 3 }, - { "g2d_mux", g2d_clk_parents, ARRAY_SIZE(g2d_clk_parents), 0, 0, SIRFSOC_CLKC_G2D_CLK_SEL, 0, 3 }, - { "jpenc_mux", jpenc_clk_parents, ARRAY_SIZE(jpenc_clk_parents), 0, 0, SIRFSOC_CLKC_JPENC_CLK_SEL, 0, 3 }, - { "vdec_mux", vdec_clk_parents, ARRAY_SIZE(vdec_clk_parents), 0, 0, SIRFSOC_CLKC_VDEC_CLK_SEL, 0, 3 }, - { "gmac_mux", gmac_clk_parents, ARRAY_SIZE(gmac_clk_parents), 0, 0, SIRFSOC_CLKC_GMAC_CLK_SEL, 0, 3 }, - { "usb_mux", usb_clk_parents, ARRAY_SIZE(usb_clk_parents), 0, 0, SIRFSOC_CLKC_USB_CLK_SEL, 0, 3 }, - { "kas_mux", kas_clk_parents, ARRAY_SIZE(kas_clk_parents), 0, 0, SIRFSOC_CLKC_KAS_CLK_SEL, 0, 3 }, - { "sec_mux", sec_clk_parents, ARRAY_SIZE(sec_clk_parents), 0, 0, SIRFSOC_CLKC_SEC_CLK_SEL, 0, 3 }, - { "sdr_mux", sdr_clk_parents, ARRAY_SIZE(sdr_clk_parents), 0, 0, SIRFSOC_CLKC_SDR_CLK_SEL, 0, 3 }, - { "vip_mux", vip_clk_parents, ARRAY_SIZE(vip_clk_parents), 0, 0, SIRFSOC_CLKC_VIP_CLK_SEL, 0, 3 }, - { "nocd_mux", nocd_clk_parents, ARRAY_SIZE(nocd_clk_parents), 0, 0, SIRFSOC_CLKC_NOCD_CLK_SEL, 0, 3 }, - { "nocr_mux", nocr_clk_parents, ARRAY_SIZE(nocr_clk_parents), 0, 0, SIRFSOC_CLKC_NOCR_CLK_SEL, 0, 3 }, - { "tpiu_mux", tpiu_clk_parents, ARRAY_SIZE(tpiu_clk_parents), 0, 0, SIRFSOC_CLKC_TPIU_CLK_SEL, 0, 3 }, -}; - - /* new unit should add start from the tail of list */ -static struct atlas7_unit_init_data unit_list[] __initdata = { - /* unit_name, parent_name, flags, regofs, bit, lock */ - { 0, "audmscm_kas", "kas_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 0, 0, 0, &root0_gate_lock }, - { 1, "gnssm_gnss", "gnss_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 1, 0, 0, &root0_gate_lock }, - { 2, "gpum_gpu", "gpu_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 2, 0, 0, &root0_gate_lock }, - { 3, "mediam_g2d", "g2d_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 3, 0, 0, &root0_gate_lock }, - { 4, "mediam_jpenc", "jpenc_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 4, 0, 0, &root0_gate_lock }, - { 5, "vdifm_disp0", "disp0_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 5, 0, 0, &root0_gate_lock }, - { 6, "vdifm_disp1", "disp1_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 6, 0, 0, &root0_gate_lock }, - { 7, "audmscm_i2s", "i2s_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 8, 0, 0, &root0_gate_lock }, - { 8, "audmscm_io", "io_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 11, 0, 0, &root0_gate_lock }, - { 9, "vdifm_io", "io_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 12, 0, 0, &root0_gate_lock }, - { 10, "gnssm_io", "io_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 13, 0, 0, &root0_gate_lock }, - { 11, "mediam_io", "io_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 14, 0, 0, &root0_gate_lock }, - { 12, "btm_io", "io_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 17, 0, 0, &root0_gate_lock }, - { 13, "mediam_sdphy01", "sdphy01_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 18, 0, 0, &root0_gate_lock }, - { 14, "vdifm_sdphy23", "sdphy23_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 19, 0, 0, &root0_gate_lock }, - { 15, "vdifm_sdphy45", "sdphy45_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 20, 0, 0, &root0_gate_lock }, - { 16, "vdifm_sdphy67", "sdphy67_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 21, 0, 0, &root0_gate_lock }, - { 17, "audmscm_xin", "xin", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 22, 0, 0, &root0_gate_lock }, - { 18, "mediam_nand", "nand_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 27, 0, 0, &root0_gate_lock }, - { 19, "gnssm_sec", "sec_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 28, 0, 0, &root0_gate_lock }, - { 20, "cpum_cpu", "cpu_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 29, 0, 0, &root0_gate_lock }, - { 21, "gnssm_xin", "xin", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 30, 0, 0, &root0_gate_lock }, - { 22, "vdifm_vip", "vip_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 31, 0, 0, &root0_gate_lock }, - { 23, "btm_btss", "btss_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 0, 0, 0, &root1_gate_lock }, - { 24, "mediam_usbphy", "usbphy_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 1, 0, 0, &root1_gate_lock }, - { 25, "rtcm_kas", "kas_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 2, 0, 0, &root1_gate_lock }, - { 26, "audmscm_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 3, 0, 0, &root1_gate_lock }, - { 27, "vdifm_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 4, 0, 0, &root1_gate_lock }, - { 28, "gnssm_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 5, 0, 0, &root1_gate_lock }, - { 29, "mediam_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 6, 0, 0, &root1_gate_lock }, - { 30, "cpum_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 8, 0, 0, &root1_gate_lock }, - { 31, "gpum_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 9, 0, 0, &root1_gate_lock }, - { 32, "audmscm_nocr", "nocr_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 11, 0, 0, &root1_gate_lock }, - { 33, "vdifm_nocr", "nocr_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 12, 0, 0, &root1_gate_lock }, - { 34, "gnssm_nocr", "nocr_mux", CLK_IGNORE_UNUSED, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 13, 0, 0, &root1_gate_lock }, - { 35, "mediam_nocr", "nocr_mux", CLK_IGNORE_UNUSED, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 14, 0, 0, &root1_gate_lock }, - { 36, "ddrm_nocr", "nocr_mux", CLK_IGNORE_UNUSED, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 15, 0, 0, &root1_gate_lock }, - { 37, "cpum_tpiu", "tpiu_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 16, 0, 0, &root1_gate_lock }, - { 38, "gpum_nocr", "nocr_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 17, 0, 0, &root1_gate_lock }, - { 39, "gnssm_rgmii", "rgmii_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 20, 0, 0, &root1_gate_lock }, - { 40, "mediam_vdec", "vdec_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 21, 0, 0, &root1_gate_lock }, - { 41, "gpum_sdr", "sdr_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 22, 0, 0, &root1_gate_lock }, - { 42, "vdifm_deint", "deint_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 23, 0, 0, &root1_gate_lock }, - { 43, "gnssm_can", "can_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 26, 0, 0, &root1_gate_lock }, - { 44, "mediam_usb", "usb_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 28, 0, 0, &root1_gate_lock }, - { 45, "gnssm_gmac", "gmac_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 29, 0, 0, &root1_gate_lock }, - { 46, "cvd_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 0, CLK_UNIT_NOC_CLOCK, 4, &leaf1_gate_lock }, - { 47, "timer_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 1, 0, 0, &leaf1_gate_lock }, - { 48, "pulse_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 2, 0, 0, &leaf1_gate_lock }, - { 49, "tsc_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 3, 0, 0, &leaf1_gate_lock }, - { 50, "tsc_xin", "audmscm_xin", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 21, 0, 0, &leaf1_gate_lock }, - { 51, "ioctop_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 4, 0, 0, &leaf1_gate_lock }, - { 52, "rsc_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 5, 0, 0, &leaf1_gate_lock }, - { 53, "dvm_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 6, CLK_UNIT_NOC_SOCKET, 7, &leaf1_gate_lock }, - { 54, "lvds_xin", "audmscm_xin", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 7, CLK_UNIT_NOC_SOCKET, 8, &leaf1_gate_lock }, - { 55, "kas_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 8, CLK_UNIT_NOC_CLOCK, 2, &leaf1_gate_lock }, - { 56, "ac97_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 9, 0, 0, &leaf1_gate_lock }, - { 57, "usp0_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 10, CLK_UNIT_NOC_SOCKET, 4, &leaf1_gate_lock }, - { 58, "usp1_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 11, CLK_UNIT_NOC_SOCKET, 5, &leaf1_gate_lock }, - { 59, "usp2_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 12, CLK_UNIT_NOC_SOCKET, 6, &leaf1_gate_lock }, - { 60, "dmac2_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 13, CLK_UNIT_NOC_SOCKET, 1, &leaf1_gate_lock }, - { 61, "dmac3_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 14, CLK_UNIT_NOC_SOCKET, 2, &leaf1_gate_lock }, - { 62, "audioif_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 15, CLK_UNIT_NOC_SOCKET, 0, &leaf1_gate_lock }, - { 63, "i2s1_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 17, CLK_UNIT_NOC_CLOCK, 2, &leaf1_gate_lock }, - { 64, "thaudmscm_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 22, 0, 0, &leaf1_gate_lock }, - { 65, "analogtest_xin", "audmscm_xin", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 23, 0, 0, &leaf1_gate_lock }, - { 66, "sys2pci_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 0, CLK_UNIT_NOC_CLOCK, 20, &leaf2_gate_lock }, - { 67, "pciarb_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 1, 0, 0, &leaf2_gate_lock }, - { 68, "pcicopy_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 2, 0, 0, &leaf2_gate_lock }, - { 69, "rom_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 3, 0, 0, &leaf2_gate_lock }, - { 70, "sdio23_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 4, 0, 0, &leaf2_gate_lock }, - { 71, "sdio45_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 5, 0, 0, &leaf2_gate_lock }, - { 72, "sdio67_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 6, 0, 0, &leaf2_gate_lock }, - { 73, "vip1_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 7, 0, 0, &leaf2_gate_lock }, - { 74, "vip1_vip", "vdifm_vip", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 16, CLK_UNIT_NOC_CLOCK, 21, &leaf2_gate_lock }, - { 75, "sdio23_sdphy23", "vdifm_sdphy23", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 8, 0, 0, &leaf2_gate_lock }, - { 76, "sdio45_sdphy45", "vdifm_sdphy45", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 9, 0, 0, &leaf2_gate_lock }, - { 77, "sdio67_sdphy67", "vdifm_sdphy67", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 10, 0, 0, &leaf2_gate_lock }, - { 78, "vpp0_disp0", "vdifm_disp0", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 11, CLK_UNIT_NOC_CLOCK, 22, &leaf2_gate_lock }, - { 79, "lcd0_disp0", "vdifm_disp0", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 12, CLK_UNIT_NOC_CLOCK, 18, &leaf2_gate_lock }, - { 80, "vpp1_disp1", "vdifm_disp1", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 13, CLK_UNIT_NOC_CLOCK, 23, &leaf2_gate_lock }, - { 81, "lcd1_disp1", "vdifm_disp1", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 14, CLK_UNIT_NOC_CLOCK, 19, &leaf2_gate_lock }, - { 82, "dcu_deint", "vdifm_deint", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 15, CLK_UNIT_NOC_CLOCK, 17, &leaf2_gate_lock }, - { 83, "vdifm_dapa_r_nocr", "vdifm_nocr", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 17, 0, 0, &leaf2_gate_lock }, - { 84, "gpio1_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 18, 0, 0, &leaf2_gate_lock }, - { 85, "thvdifm_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 19, 0, 0, &leaf2_gate_lock }, - { 86, "gmac_rgmii", "gnssm_rgmii", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 0, 0, 0, &leaf3_gate_lock }, - { 87, "gmac_gmac", "gnssm_gmac", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 1, CLK_UNIT_NOC_CLOCK, 10, &leaf3_gate_lock }, - { 88, "uart1_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 2, CLK_UNIT_NOC_SOCKET, 14, &leaf3_gate_lock }, - { 89, "dmac0_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 3, CLK_UNIT_NOC_SOCKET, 11, &leaf3_gate_lock }, - { 90, "uart0_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 4, CLK_UNIT_NOC_SOCKET, 13, &leaf3_gate_lock }, - { 91, "uart2_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 5, CLK_UNIT_NOC_SOCKET, 15, &leaf3_gate_lock }, - { 92, "uart3_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 6, CLK_UNIT_NOC_SOCKET, 16, &leaf3_gate_lock }, - { 93, "uart4_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 7, CLK_UNIT_NOC_SOCKET, 17, &leaf3_gate_lock }, - { 94, "uart5_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 8, CLK_UNIT_NOC_SOCKET, 18, &leaf3_gate_lock }, - { 95, "spi1_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 9, CLK_UNIT_NOC_SOCKET, 12, &leaf3_gate_lock }, - { 96, "gnss_gnss", "gnssm_gnss", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 10, 0, 0, &leaf3_gate_lock }, - { 97, "canbus1_can", "gnssm_can", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 12, CLK_UNIT_NOC_CLOCK, 7, &leaf3_gate_lock }, - { 98, "ccsec_sec", "gnssm_sec", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 15, CLK_UNIT_NOC_CLOCK, 9, &leaf3_gate_lock }, - { 99, "ccpub_sec", "gnssm_sec", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 16, CLK_UNIT_NOC_CLOCK, 8, &leaf3_gate_lock }, - { 100, "gnssm_dapa_r_nocr", "gnssm_nocr", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 13, 0, 0, &leaf3_gate_lock }, - { 101, "thgnssm_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 14, 0, 0, &leaf3_gate_lock }, - { 102, "media_vdec", "mediam_vdec", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 0, CLK_UNIT_NOC_CLOCK, 3, &leaf4_gate_lock }, - { 103, "media_jpenc", "mediam_jpenc", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 1, CLK_UNIT_NOC_CLOCK, 1, &leaf4_gate_lock }, - { 104, "g2d_g2d", "mediam_g2d", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 2, CLK_UNIT_NOC_CLOCK, 12, &leaf4_gate_lock }, - { 105, "i2c0_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 3, CLK_UNIT_NOC_SOCKET, 21, &leaf4_gate_lock }, - { 106, "i2c1_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 4, CLK_UNIT_NOC_SOCKET, 20, &leaf4_gate_lock }, - { 107, "gpio0_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 5, CLK_UNIT_NOC_SOCKET, 19, &leaf4_gate_lock }, - { 108, "nand_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 6, 0, 0, &leaf4_gate_lock }, - { 109, "sdio01_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 7, 0, 0, &leaf4_gate_lock }, - { 110, "sys2pci2_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 8, CLK_UNIT_NOC_CLOCK, 13, &leaf4_gate_lock }, - { 111, "sdio01_sdphy01", "mediam_sdphy01", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 9, 0, 0, &leaf4_gate_lock }, - { 112, "nand_nand", "mediam_nand", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 10, CLK_UNIT_NOC_CLOCK, 14, &leaf4_gate_lock }, - { 113, "usb0_usb", "mediam_usb", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 11, CLK_UNIT_NOC_CLOCK, 15, &leaf4_gate_lock }, - { 114, "usb1_usb", "mediam_usb", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 12, CLK_UNIT_NOC_CLOCK, 16, &leaf4_gate_lock }, - { 115, "usbphy0_usbphy", "mediam_usbphy", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 13, 0, 0, &leaf4_gate_lock }, - { 116, "usbphy1_usbphy", "mediam_usbphy", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 14, 0, 0, &leaf4_gate_lock }, - { 117, "thmediam_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 15, 0, 0, &leaf4_gate_lock }, - { 118, "memc_mem", "mempll_clk1", CLK_IGNORE_UNUSED, SIRFSOC_CLKC_LEAF_CLK_EN5_SET, 0, 0, 0, &leaf5_gate_lock }, - { 119, "dapa_mem", "mempll_clk1", 0, SIRFSOC_CLKC_LEAF_CLK_EN5_SET, 1, 0, 0, &leaf5_gate_lock }, - { 120, "nocddrm_nocr", "ddrm_nocr", 0, SIRFSOC_CLKC_LEAF_CLK_EN5_SET, 2, 0, 0, &leaf5_gate_lock }, - { 121, "thddrm_nocr", "ddrm_nocr", 0, SIRFSOC_CLKC_LEAF_CLK_EN5_SET, 3, 0, 0, &leaf5_gate_lock }, - { 122, "spram1_cpudiv2", "cpum_cpu", 0, SIRFSOC_CLKC_LEAF_CLK_EN6_SET, 0, CLK_UNIT_NOC_SOCKET, 9, &leaf6_gate_lock }, - { 123, "spram2_cpudiv2", "cpum_cpu", 0, SIRFSOC_CLKC_LEAF_CLK_EN6_SET, 1, CLK_UNIT_NOC_SOCKET, 10, &leaf6_gate_lock }, - { 124, "coresight_cpudiv2", "cpum_cpu", 0, SIRFSOC_CLKC_LEAF_CLK_EN6_SET, 2, 0, 0, &leaf6_gate_lock }, - { 125, "coresight_tpiu", "cpum_tpiu", 0, SIRFSOC_CLKC_LEAF_CLK_EN6_SET, 3, 0, 0, &leaf6_gate_lock }, - { 126, "graphic_gpu", "gpum_gpu", 0, SIRFSOC_CLKC_LEAF_CLK_EN7_SET, 0, CLK_UNIT_NOC_CLOCK, 0, &leaf7_gate_lock }, - { 127, "vss_sdr", "gpum_sdr", 0, SIRFSOC_CLKC_LEAF_CLK_EN7_SET, 1, CLK_UNIT_NOC_CLOCK, 11, &leaf7_gate_lock }, - { 128, "thgpum_nocr", "gpum_nocr", 0, SIRFSOC_CLKC_LEAF_CLK_EN7_SET, 2, 0, 0, &leaf7_gate_lock }, - { 129, "a7ca_btss", "btm_btss", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 1, 0, 0, &leaf8_gate_lock }, - { 130, "dmac4_io", "a7ca_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 2, 0, 0, &leaf8_gate_lock }, - { 131, "uart6_io", "dmac4_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 3, 0, 0, &leaf8_gate_lock }, - { 132, "usp3_io", "dmac4_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 4, 0, 0, &leaf8_gate_lock }, - { 133, "a7ca_io", "noc_btm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 5, 0, 0, &leaf8_gate_lock }, - { 134, "noc_btm_io", "btm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 6, 0, 0, &leaf8_gate_lock }, - { 135, "thbtm_io", "btm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 7, 0, 0, &leaf8_gate_lock }, - { 136, "btslow", "xinw_fixdiv_btslow", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 25, 0, 0, &root1_gate_lock }, - { 137, "a7ca_btslow", "btslow", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 0, 0, 0, &leaf8_gate_lock }, - { 138, "pwm_io", "io_mux", 0, SIRFSOC_CLKC_LEAF_CLK_EN0_SET, 0, 0, 0, &leaf0_gate_lock }, - { 139, "pwm_xin", "xin", 0, SIRFSOC_CLKC_LEAF_CLK_EN0_SET, 1, 0, 0, &leaf0_gate_lock }, - { 140, "pwm_xinw", "xinw", 0, SIRFSOC_CLKC_LEAF_CLK_EN0_SET, 2, 0, 0, &leaf0_gate_lock }, - { 141, "thcgum_sys", "sys_mux", 0, SIRFSOC_CLKC_LEAF_CLK_EN0_SET, 3, 0, 0, &leaf0_gate_lock }, -}; - -static struct clk *atlas7_clks[ARRAY_SIZE(unit_list) + ARRAY_SIZE(mux_list)]; - -static int unit_clk_is_enabled(struct clk_hw *hw) -{ - struct clk_unit *clk = to_unitclk(hw); - u32 reg; - - reg = clk->regofs + SIRFSOC_CLKC_ROOT_CLK_EN0_STAT - SIRFSOC_CLKC_ROOT_CLK_EN0_SET; - - return !!(clkc_readl(reg) & BIT(clk->bit)); -} - -static int unit_clk_enable(struct clk_hw *hw) -{ - u32 reg; - struct clk_unit *clk = to_unitclk(hw); - unsigned long flags; - - reg = clk->regofs; - - spin_lock_irqsave(clk->lock, flags); - clkc_writel(BIT(clk->bit), reg); - if (clk->type == CLK_UNIT_NOC_CLOCK) - clkc_writel(BIT(clk->idle_bit), SIRFSOC_NOC_CLK_IDLEREQ_CLR); - else if (clk->type == CLK_UNIT_NOC_SOCKET) - clkc_writel(BIT(clk->idle_bit), SIRFSOC_NOC_CLK_SLVRDY_SET); - - spin_unlock_irqrestore(clk->lock, flags); - return 0; -} - -static void unit_clk_disable(struct clk_hw *hw) -{ - u32 reg; - u32 i = 0; - struct clk_unit *clk = to_unitclk(hw); - unsigned long flags; - - reg = clk->regofs + SIRFSOC_CLKC_ROOT_CLK_EN0_CLR - SIRFSOC_CLKC_ROOT_CLK_EN0_SET; - spin_lock_irqsave(clk->lock, flags); - if (clk->type == CLK_UNIT_NOC_CLOCK) { - clkc_writel(BIT(clk->idle_bit), SIRFSOC_NOC_CLK_IDLEREQ_SET); - while (!(clkc_readl(SIRFSOC_NOC_CLK_IDLE_STATUS) & - BIT(clk->idle_bit)) && (i++ < 100)) { - cpu_relax(); - udelay(10); - } - - if (i == 100) { - pr_err("unit NoC Clock disconnect Error:timeout\n"); - /*once timeout, undo idlereq by CLR*/ - clkc_writel(BIT(clk->idle_bit), SIRFSOC_NOC_CLK_IDLEREQ_CLR); - goto err; - } - - } else if (clk->type == CLK_UNIT_NOC_SOCKET) - clkc_writel(BIT(clk->idle_bit), SIRFSOC_NOC_CLK_SLVRDY_CLR); - - clkc_writel(BIT(clk->bit), reg); -err: - spin_unlock_irqrestore(clk->lock, flags); -} - -static const struct clk_ops unit_clk_ops = { - .is_enabled = unit_clk_is_enabled, - .enable = unit_clk_enable, - .disable = unit_clk_disable, -}; - -static struct clk * __init -atlas7_unit_clk_register(struct device *dev, const char *name, - const char * const parent_name, unsigned long flags, - u32 regofs, u8 bit, u32 type, u8 idle_bit, spinlock_t *lock) -{ - struct clk *clk; - struct clk_unit *unit; - struct clk_init_data init; - - unit = kzalloc(sizeof(*unit), GFP_KERNEL); - if (!unit) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.parent_names = &parent_name; - init.num_parents = 1; - init.ops = &unit_clk_ops; - init.flags = flags; - - unit->hw.init = &init; - unit->regofs = regofs; - unit->bit = bit; - - unit->type = type; - unit->idle_bit = idle_bit; - unit->lock = lock; - - clk = clk_register(dev, &unit->hw); - if (IS_ERR(clk)) - kfree(unit); - - return clk; -} - -static struct atlas7_reset_desc atlas7_reset_unit[] = { - { "PWM", 0x0244, 0, 0x0320, 0, &leaf0_gate_lock }, /* 0-5 */ - { "THCGUM", 0x0244, 3, 0x0320, 1, &leaf0_gate_lock }, - { "CVD", 0x04A0, 0, 0x032C, 0, &leaf1_gate_lock }, - { "TIMER", 0x04A0, 1, 0x032C, 1, &leaf1_gate_lock }, - { "PULSEC", 0x04A0, 2, 0x032C, 2, &leaf1_gate_lock }, - { "TSC", 0x04A0, 3, 0x032C, 3, &leaf1_gate_lock }, - { "IOCTOP", 0x04A0, 4, 0x032C, 4, &leaf1_gate_lock }, /* 6-10 */ - { "RSC", 0x04A0, 5, 0x032C, 5, &leaf1_gate_lock }, - { "DVM", 0x04A0, 6, 0x032C, 6, &leaf1_gate_lock }, - { "LVDS", 0x04A0, 7, 0x032C, 7, &leaf1_gate_lock }, - { "KAS", 0x04A0, 8, 0x032C, 8, &leaf1_gate_lock }, - { "AC97", 0x04A0, 9, 0x032C, 9, &leaf1_gate_lock }, /* 11-15 */ - { "USP0", 0x04A0, 10, 0x032C, 10, &leaf1_gate_lock }, - { "USP1", 0x04A0, 11, 0x032C, 11, &leaf1_gate_lock }, - { "USP2", 0x04A0, 12, 0x032C, 12, &leaf1_gate_lock }, - { "DMAC2", 0x04A0, 13, 0x032C, 13, &leaf1_gate_lock }, - { "DMAC3", 0x04A0, 14, 0x032C, 14, &leaf1_gate_lock }, /* 16-20 */ - { "AUDIO", 0x04A0, 15, 0x032C, 15, &leaf1_gate_lock }, - { "I2S1", 0x04A0, 17, 0x032C, 16, &leaf1_gate_lock }, - { "PMU_AUDIO", 0x04A0, 22, 0x032C, 17, &leaf1_gate_lock }, - { "THAUDMSCM", 0x04A0, 23, 0x032C, 18, &leaf1_gate_lock }, - { "SYS2PCI", 0x04B8, 0, 0x0338, 0, &leaf2_gate_lock }, /* 21-25 */ - { "PCIARB", 0x04B8, 1, 0x0338, 1, &leaf2_gate_lock }, - { "PCICOPY", 0x04B8, 2, 0x0338, 2, &leaf2_gate_lock }, - { "ROM", 0x04B8, 3, 0x0338, 3, &leaf2_gate_lock }, - { "SDIO23", 0x04B8, 4, 0x0338, 4, &leaf2_gate_lock }, - { "SDIO45", 0x04B8, 5, 0x0338, 5, &leaf2_gate_lock }, /* 26-30 */ - { "SDIO67", 0x04B8, 6, 0x0338, 6, &leaf2_gate_lock }, - { "VIP1", 0x04B8, 7, 0x0338, 7, &leaf2_gate_lock }, - { "VPP0", 0x04B8, 11, 0x0338, 8, &leaf2_gate_lock }, - { "LCD0", 0x04B8, 12, 0x0338, 9, &leaf2_gate_lock }, - { "VPP1", 0x04B8, 13, 0x0338, 10, &leaf2_gate_lock }, /* 31-35 */ - { "LCD1", 0x04B8, 14, 0x0338, 11, &leaf2_gate_lock }, - { "DCU", 0x04B8, 15, 0x0338, 12, &leaf2_gate_lock }, - { "GPIO", 0x04B8, 18, 0x0338, 13, &leaf2_gate_lock }, - { "DAPA_VDIFM", 0x04B8, 17, 0x0338, 15, &leaf2_gate_lock }, - { "THVDIFM", 0x04B8, 19, 0x0338, 16, &leaf2_gate_lock }, /* 36-40 */ - { "RGMII", 0x04D0, 0, 0x0344, 0, &leaf3_gate_lock }, - { "GMAC", 0x04D0, 1, 0x0344, 1, &leaf3_gate_lock }, - { "UART1", 0x04D0, 2, 0x0344, 2, &leaf3_gate_lock }, - { "DMAC0", 0x04D0, 3, 0x0344, 3, &leaf3_gate_lock }, - { "UART0", 0x04D0, 4, 0x0344, 4, &leaf3_gate_lock }, /* 41-45 */ - { "UART2", 0x04D0, 5, 0x0344, 5, &leaf3_gate_lock }, - { "UART3", 0x04D0, 6, 0x0344, 6, &leaf3_gate_lock }, - { "UART4", 0x04D0, 7, 0x0344, 7, &leaf3_gate_lock }, - { "UART5", 0x04D0, 8, 0x0344, 8, &leaf3_gate_lock }, - { "SPI1", 0x04D0, 9, 0x0344, 9, &leaf3_gate_lock }, /* 46-50 */ - { "GNSS_SYS_M0", 0x04D0, 10, 0x0344, 10, &leaf3_gate_lock }, - { "CANBUS1", 0x04D0, 12, 0x0344, 11, &leaf3_gate_lock }, - { "CCSEC", 0x04D0, 15, 0x0344, 12, &leaf3_gate_lock }, - { "CCPUB", 0x04D0, 16, 0x0344, 13, &leaf3_gate_lock }, - { "DAPA_GNSSM", 0x04D0, 13, 0x0344, 14, &leaf3_gate_lock }, /* 51-55 */ - { "THGNSSM", 0x04D0, 14, 0x0344, 15, &leaf3_gate_lock }, - { "VDEC", 0x04E8, 0, 0x0350, 0, &leaf4_gate_lock }, - { "JPENC", 0x04E8, 1, 0x0350, 1, &leaf4_gate_lock }, - { "G2D", 0x04E8, 2, 0x0350, 2, &leaf4_gate_lock }, - { "I2C0", 0x04E8, 3, 0x0350, 3, &leaf4_gate_lock }, /* 56-60 */ - { "I2C1", 0x04E8, 4, 0x0350, 4, &leaf4_gate_lock }, - { "GPIO0", 0x04E8, 5, 0x0350, 5, &leaf4_gate_lock }, - { "NAND", 0x04E8, 6, 0x0350, 6, &leaf4_gate_lock }, - { "SDIO01", 0x04E8, 7, 0x0350, 7, &leaf4_gate_lock }, - { "SYS2PCI2", 0x04E8, 8, 0x0350, 8, &leaf4_gate_lock }, /* 61-65 */ - { "USB0", 0x04E8, 11, 0x0350, 9, &leaf4_gate_lock }, - { "USB1", 0x04E8, 12, 0x0350, 10, &leaf4_gate_lock }, - { "THMEDIAM", 0x04E8, 15, 0x0350, 11, &leaf4_gate_lock }, - { "MEMC_DDRPHY", 0x0500, 0, 0x035C, 0, &leaf5_gate_lock }, - { "MEMC_UPCTL", 0x0500, 0, 0x035C, 1, &leaf5_gate_lock }, /* 66-70 */ - { "DAPA_MEM", 0x0500, 1, 0x035C, 2, &leaf5_gate_lock }, - { "MEMC_MEMDIV", 0x0500, 0, 0x035C, 3, &leaf5_gate_lock }, - { "THDDRM", 0x0500, 3, 0x035C, 4, &leaf5_gate_lock }, - { "CORESIGHT", 0x0518, 3, 0x0368, 13, &leaf6_gate_lock }, - { "THCPUM", 0x0518, 4, 0x0368, 17, &leaf6_gate_lock }, /* 71-75 */ - { "GRAPHIC", 0x0530, 0, 0x0374, 0, &leaf7_gate_lock }, - { "VSS_SDR", 0x0530, 1, 0x0374, 1, &leaf7_gate_lock }, - { "THGPUM", 0x0530, 2, 0x0374, 2, &leaf7_gate_lock }, - { "DMAC4", 0x0548, 2, 0x0380, 1, &leaf8_gate_lock }, - { "UART6", 0x0548, 3, 0x0380, 2, &leaf8_gate_lock }, /* 76- */ - { "USP3", 0x0548, 4, 0x0380, 3, &leaf8_gate_lock }, - { "THBTM", 0x0548, 5, 0x0380, 5, &leaf8_gate_lock }, - { "A7CA", 0x0548, 1, 0x0380, 0, &leaf8_gate_lock }, - { "A7CA_APB", 0x0548, 5, 0x0380, 4, &leaf8_gate_lock }, -}; - -static int atlas7_reset_module(struct reset_controller_dev *rcdev, - unsigned long reset_idx) -{ - struct atlas7_reset_desc *reset = &atlas7_reset_unit[reset_idx]; - unsigned long flags; - - /* - * HW suggest unit reset sequence: - * assert sw reset (0) - * setting sw clk_en to if the clock was disabled before reset - * delay 16 clocks - * disable clock (sw clk_en = 0) - * de-assert reset (1) - * after this sequence, restore clock or not is decided by SW - */ - - spin_lock_irqsave(reset->lock, flags); - /* clock enable or not */ - if (clkc_readl(reset->clk_ofs + 8) & (1 << reset->clk_bit)) { - clkc_writel(1 << reset->rst_bit, reset->rst_ofs + 4); - udelay(2); - clkc_writel(1 << reset->clk_bit, reset->clk_ofs + 4); - clkc_writel(1 << reset->rst_bit, reset->rst_ofs); - /* restore clock enable */ - clkc_writel(1 << reset->clk_bit, reset->clk_ofs); - } else { - clkc_writel(1 << reset->rst_bit, reset->rst_ofs + 4); - clkc_writel(1 << reset->clk_bit, reset->clk_ofs); - udelay(2); - clkc_writel(1 << reset->clk_bit, reset->clk_ofs + 4); - clkc_writel(1 << reset->rst_bit, reset->rst_ofs); - } - spin_unlock_irqrestore(reset->lock, flags); - - return 0; -} - -static const struct reset_control_ops atlas7_rst_ops = { - .reset = atlas7_reset_module, -}; - -static struct reset_controller_dev atlas7_rst_ctlr = { - .ops = &atlas7_rst_ops, - .owner = THIS_MODULE, - .of_reset_n_cells = 1, -}; - -static void __init atlas7_clk_init(struct device_node *np) -{ - struct clk *clk; - struct atlas7_div_init_data *div; - struct atlas7_mux_init_data *mux; - struct atlas7_unit_init_data *unit; - int i; - int ret; - - sirfsoc_clk_vbase = of_iomap(np, 0); - if (!sirfsoc_clk_vbase) - panic("unable to map clkc registers\n"); - - of_node_put(np); - - clk = clk_register(NULL, &clk_cpupll.hw); - BUG_ON(!clk); - clk = clk_register(NULL, &clk_mempll.hw); - BUG_ON(!clk); - clk = clk_register(NULL, &clk_sys0pll.hw); - BUG_ON(!clk); - clk = clk_register(NULL, &clk_sys1pll.hw); - BUG_ON(!clk); - clk = clk_register(NULL, &clk_sys2pll.hw); - BUG_ON(!clk); - clk = clk_register(NULL, &clk_sys3pll.hw); - BUG_ON(!clk); - - clk = clk_register_divider_table(NULL, "cpupll_div1", "cpupll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &cpupll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "cpupll_div2", "cpupll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &cpupll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "cpupll_div3", "cpupll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &cpupll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_divider_table(NULL, "mempll_div1", "mempll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &mempll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "mempll_div2", "mempll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &mempll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "mempll_div3", "mempll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &mempll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_divider_table(NULL, "sys0pll_div1", "sys0pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys0pll_div2", "sys0pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys0pll_div3", "sys0pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_fixed_factor(NULL, "sys0pll_fixdiv", "sys0pll_vco", - CLK_SET_RATE_PARENT, 1, 2); - - clk = clk_register_divider_table(NULL, "sys1pll_div1", "sys1pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys1pll_div2", "sys1pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys1pll_div3", "sys1pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_fixed_factor(NULL, "sys1pll_fixdiv", "sys1pll_vco", - CLK_SET_RATE_PARENT, 1, 2); - - clk = clk_register_divider_table(NULL, "sys2pll_div1", "sys2pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys2pll_div2", "sys2pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys2pll_div3", "sys2pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_fixed_factor(NULL, "sys2pll_fixdiv", "sys2pll_vco", - CLK_SET_RATE_PARENT, 1, 2); - - clk = clk_register_divider_table(NULL, "sys3pll_div1", "sys3pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys3pll_div2", "sys3pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys3pll_div3", "sys3pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_fixed_factor(NULL, "sys3pll_fixdiv", "sys3pll_vco", - CLK_SET_RATE_PARENT, 1, 2); - - BUG_ON(!clk); - clk = clk_register_fixed_factor(NULL, "xinw_fixdiv_btslow", "xinw", - CLK_SET_RATE_PARENT, 1, 4); - - BUG_ON(!clk); - clk = clk_register_gate(NULL, "cpupll_clk1", "cpupll_div1", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, - 12, 0, &cpupll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "cpupll_clk2", "cpupll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, - 13, 0, &cpupll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "cpupll_clk3", "cpupll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, - 14, 0, &cpupll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_gate(NULL, "mempll_clk1", "mempll_div1", - CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, - sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, - 12, 0, &mempll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "mempll_clk2", "mempll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, - 13, 0, &mempll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "mempll_clk3", "mempll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, - 14, 0, &mempll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_gate(NULL, "sys0pll_clk1", "sys0pll_div1", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, - 12, 0, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys0pll_clk2", "sys0pll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, - 13, 0, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys0pll_clk3", "sys0pll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, - 14, 0, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_gate(NULL, "sys1pll_clk1", "sys1pll_div1", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, - 12, 0, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys1pll_clk2", "sys1pll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, - 13, 0, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys1pll_clk3", "sys1pll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, - 14, 0, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_gate(NULL, "sys2pll_clk1", "sys2pll_div1", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, - 12, 0, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys2pll_clk2", "sys2pll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, - 13, 0, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys2pll_clk3", "sys2pll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, - 14, 0, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_gate(NULL, "sys3pll_clk1", "sys3pll_div1", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, - 12, 0, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys3pll_clk2", "sys3pll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, - 13, 0, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys3pll_clk3", "sys3pll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, - 14, 0, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register(NULL, &clk_audio_dto.hw); - BUG_ON(!clk); - - clk = clk_register(NULL, &clk_disp0_dto.hw); - BUG_ON(!clk); - - clk = clk_register(NULL, &clk_disp1_dto.hw); - BUG_ON(!clk); - - for (i = 0; i < ARRAY_SIZE(divider_list); i++) { - div = ÷r_list[i]; - clk = clk_register_divider(NULL, div->div_name, - div->parent_name, div->divider_flags, sirfsoc_clk_vbase + div->div_offset, - div->shift, div->width, 0, div->lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, div->gate_name, div->div_name, - div->gate_flags, sirfsoc_clk_vbase + div->gate_offset, - div->gate_bit, 0, div->lock); - BUG_ON(!clk); - } - /* ignore selector status register check */ - for (i = 0; i < ARRAY_SIZE(mux_list); i++) { - mux = &mux_list[i]; - clk = clk_register_mux(NULL, mux->mux_name, mux->parent_names, - mux->parent_num, mux->flags, - sirfsoc_clk_vbase + mux->mux_offset, - mux->shift, mux->width, - mux->mux_flags, NULL); - atlas7_clks[ARRAY_SIZE(unit_list) + i] = clk; - BUG_ON(!clk); - } - - for (i = 0; i < ARRAY_SIZE(unit_list); i++) { - unit = &unit_list[i]; - atlas7_clks[i] = atlas7_unit_clk_register(NULL, unit->unit_name, unit->parent_name, - unit->flags, unit->regofs, unit->bit, unit->type, unit->idle_bit, unit->lock); - BUG_ON(!atlas7_clks[i]); - } - - clk_data.clks = atlas7_clks; - clk_data.clk_num = ARRAY_SIZE(unit_list) + ARRAY_SIZE(mux_list); - - ret = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); - BUG_ON(ret); - - atlas7_rst_ctlr.of_node = np; - atlas7_rst_ctlr.nr_resets = ARRAY_SIZE(atlas7_reset_unit); - reset_controller_register(&atlas7_rst_ctlr); -} -CLK_OF_DECLARE(atlas7_clk, "sirf,atlas7-car", atlas7_clk_init); diff --git a/drivers/clk/sirf/clk-common.c b/drivers/clk/sirf/clk-common.c deleted file mode 100644 index dcf4e25a0216..000000000000 --- a/drivers/clk/sirf/clk-common.c +++ /dev/null @@ -1,1037 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * common clks module for all SiRF SoCs - * - * Copyright (c) 2011 - 2014 Cambridge Silicon Radio Limited, a CSR plc group - * company. - */ - -#include - -#define KHZ 1000 -#define MHZ (KHZ * KHZ) - -static void __iomem *sirfsoc_clk_vbase; -static void __iomem *sirfsoc_rsc_vbase; -static struct clk_onecell_data clk_data; - -/* - * SiRFprimaII clock controller - * - 2 oscillators: osc-26MHz, rtc-32.768KHz - * - 3 standard configurable plls: pll1, pll2 & pll3 - * - 2 exclusive plls: usb phy pll and sata phy pll - * - 8 clock domains: cpu/cpudiv, mem/memdiv, sys/io, dsp, graphic, multimedia, - * display and sdphy. - * Each clock domain can select its own clock source from five clock sources, - * X_XIN, X_XINW, PLL1, PLL2 and PLL3. The domain clock is used as the source - * clock of the group clock. - * - dsp domain: gps, mf - * - io domain: dmac, nand, audio, uart, i2c, spi, usp, pwm, pulse - * - sys domain: security - */ - -struct clk_pll { - struct clk_hw hw; - unsigned short regofs; /* register offset */ -}; - -#define to_pllclk(_hw) container_of(_hw, struct clk_pll, hw) - -struct clk_dmn { - struct clk_hw hw; - signed char enable_bit; /* enable bit: 0 ~ 63 */ - unsigned short regofs; /* register offset */ -}; - -#define to_dmnclk(_hw) container_of(_hw, struct clk_dmn, hw) - -struct clk_std { - struct clk_hw hw; - signed char enable_bit; /* enable bit: 0 ~ 63 */ -}; - -#define to_stdclk(_hw) container_of(_hw, struct clk_std, hw) - -static int std_clk_is_enabled(struct clk_hw *hw); -static int std_clk_enable(struct clk_hw *hw); -static void std_clk_disable(struct clk_hw *hw); - -static inline unsigned long clkc_readl(unsigned reg) -{ - return readl(sirfsoc_clk_vbase + reg); -} - -static inline void clkc_writel(u32 val, unsigned reg) -{ - writel(val, sirfsoc_clk_vbase + reg); -} - -/* - * std pll - */ - -static unsigned long pll_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - unsigned long fin = parent_rate; - struct clk_pll *clk = to_pllclk(hw); - u32 regcfg2 = clk->regofs + SIRFSOC_CLKC_PLL1_CFG2 - - SIRFSOC_CLKC_PLL1_CFG0; - - if (clkc_readl(regcfg2) & BIT(2)) { - /* pll bypass mode */ - return fin; - } else { - /* fout = fin * nf / nr / od */ - u32 cfg0 = clkc_readl(clk->regofs); - u32 nf = (cfg0 & (BIT(13) - 1)) + 1; - u32 nr = ((cfg0 >> 13) & (BIT(6) - 1)) + 1; - u32 od = ((cfg0 >> 19) & (BIT(4) - 1)) + 1; - WARN_ON(fin % MHZ); - return fin / MHZ * nf / nr / od * MHZ; - } -} - -static long pll_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate) -{ - unsigned long fin, nf, nr, od; - u64 dividend; - - /* - * fout = fin * nf / (nr * od); - * set od = 1, nr = fin/MHz, so fout = nf * MHz - */ - rate = rate - rate % MHZ; - - nf = rate / MHZ; - if (nf > BIT(13)) - nf = BIT(13); - if (nf < 1) - nf = 1; - - fin = *parent_rate; - - nr = fin / MHZ; - if (nr > BIT(6)) - nr = BIT(6); - od = 1; - - dividend = (u64)fin * nf; - do_div(dividend, nr * od); - - return (long)dividend; -} - -static int pll_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct clk_pll *clk = to_pllclk(hw); - unsigned long fin, nf, nr, od, reg; - - /* - * fout = fin * nf / (nr * od); - * set od = 1, nr = fin/MHz, so fout = nf * MHz - */ - - nf = rate / MHZ; - if (unlikely((rate % MHZ) || nf > BIT(13) || nf < 1)) - return -EINVAL; - - fin = parent_rate; - BUG_ON(fin < MHZ); - - nr = fin / MHZ; - BUG_ON((fin % MHZ) || nr > BIT(6)); - - od = 1; - - reg = (nf - 1) | ((nr - 1) << 13) | ((od - 1) << 19); - clkc_writel(reg, clk->regofs); - - reg = clk->regofs + SIRFSOC_CLKC_PLL1_CFG1 - SIRFSOC_CLKC_PLL1_CFG0; - clkc_writel((nf >> 1) - 1, reg); - - reg = clk->regofs + SIRFSOC_CLKC_PLL1_CFG2 - SIRFSOC_CLKC_PLL1_CFG0; - while (!(clkc_readl(reg) & BIT(6))) - cpu_relax(); - - return 0; -} - -static long cpu_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate) -{ - /* - * SiRF SoC has not cpu clock control, - * So bypass to it's parent pll. - */ - struct clk_hw *parent_clk = clk_hw_get_parent(hw); - struct clk_hw *pll_parent_clk = clk_hw_get_parent(parent_clk); - unsigned long pll_parent_rate = clk_hw_get_rate(pll_parent_clk); - return pll_clk_round_rate(parent_clk, rate, &pll_parent_rate); -} - -static unsigned long cpu_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - /* - * SiRF SoC has not cpu clock control, - * So return the parent pll rate. - */ - struct clk_hw *parent_clk = clk_hw_get_parent(hw); - return clk_hw_get_rate(parent_clk); -} - -static const struct clk_ops std_pll_ops = { - .recalc_rate = pll_clk_recalc_rate, - .round_rate = pll_clk_round_rate, - .set_rate = pll_clk_set_rate, -}; - -static const char * const pll_clk_parents[] = { - "osc", -}; - -static const struct clk_init_data clk_pll1_init = { - .name = "pll1", - .ops = &std_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static const struct clk_init_data clk_pll2_init = { - .name = "pll2", - .ops = &std_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static const struct clk_init_data clk_pll3_init = { - .name = "pll3", - .ops = &std_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_pll1 = { - .regofs = SIRFSOC_CLKC_PLL1_CFG0, - .hw = { - .init = &clk_pll1_init, - }, -}; - -static struct clk_pll clk_pll2 = { - .regofs = SIRFSOC_CLKC_PLL2_CFG0, - .hw = { - .init = &clk_pll2_init, - }, -}; - -static struct clk_pll clk_pll3 = { - .regofs = SIRFSOC_CLKC_PLL3_CFG0, - .hw = { - .init = &clk_pll3_init, - }, -}; - -/* - * usb uses specified pll - */ - -static int usb_pll_clk_enable(struct clk_hw *hw) -{ - u32 reg = readl(sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL); - reg &= ~(SIRFSOC_USBPHY_PLL_POWERDOWN | SIRFSOC_USBPHY_PLL_BYPASS); - writel(reg, sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL); - while (!(readl(sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL) & - SIRFSOC_USBPHY_PLL_LOCK)) - cpu_relax(); - - return 0; -} - -static void usb_pll_clk_disable(struct clk_hw *clk) -{ - u32 reg = readl(sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL); - reg |= (SIRFSOC_USBPHY_PLL_POWERDOWN | SIRFSOC_USBPHY_PLL_BYPASS); - writel(reg, sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL); -} - -static unsigned long usb_pll_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) -{ - u32 reg = readl(sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL); - return (reg & SIRFSOC_USBPHY_PLL_BYPASS) ? parent_rate : 48*MHZ; -} - -static const struct clk_ops usb_pll_ops = { - .enable = usb_pll_clk_enable, - .disable = usb_pll_clk_disable, - .recalc_rate = usb_pll_clk_recalc_rate, -}; - -static const struct clk_init_data clk_usb_pll_init = { - .name = "usb_pll", - .ops = &usb_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_hw usb_pll_clk_hw = { - .init = &clk_usb_pll_init, -}; - -/* - * clock domains - cpu, mem, sys/io, dsp, gfx - */ - -static const char * const dmn_clk_parents[] = { - "rtc", - "osc", - "pll1", - "pll2", - "pll3", -}; - -static u8 dmn_clk_get_parent(struct clk_hw *hw) -{ - struct clk_dmn *clk = to_dmnclk(hw); - u32 cfg = clkc_readl(clk->regofs); - const char *name = clk_hw_get_name(hw); - - /* parent of io domain can only be pll3 */ - if (strcmp(name, "io") == 0) - return 4; - - WARN_ON((cfg & (BIT(3) - 1)) > 4); - - return cfg & (BIT(3) - 1); -} - -static int dmn_clk_set_parent(struct clk_hw *hw, u8 parent) -{ - struct clk_dmn *clk = to_dmnclk(hw); - u32 cfg = clkc_readl(clk->regofs); - const char *name = clk_hw_get_name(hw); - - /* parent of io domain can only be pll3 */ - if (strcmp(name, "io") == 0) - return -EINVAL; - - cfg &= ~(BIT(3) - 1); - clkc_writel(cfg | parent, clk->regofs); - /* BIT(3) - switching status: 1 - busy, 0 - done */ - while (clkc_readl(clk->regofs) & BIT(3)) - cpu_relax(); - - return 0; -} - -static unsigned long dmn_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) - -{ - unsigned long fin = parent_rate; - struct clk_dmn *clk = to_dmnclk(hw); - - u32 cfg = clkc_readl(clk->regofs); - - if (cfg & BIT(24)) { - /* fcd bypass mode */ - return fin; - } else { - /* - * wait count: bit[19:16], hold count: bit[23:20] - */ - u32 wait = (cfg >> 16) & (BIT(4) - 1); - u32 hold = (cfg >> 20) & (BIT(4) - 1); - - return fin / (wait + hold + 2); - } -} - -static long dmn_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate) -{ - unsigned long fin; - unsigned ratio, wait, hold; - const char *name = clk_hw_get_name(hw); - unsigned bits = (strcmp(name, "mem") == 0) ? 3 : 4; - - fin = *parent_rate; - ratio = fin / rate; - - if (ratio < 2) - ratio = 2; - if (ratio > BIT(bits + 1)) - ratio = BIT(bits + 1); - - wait = (ratio >> 1) - 1; - hold = ratio - wait - 2; - - return fin / (wait + hold + 2); -} - -static int dmn_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct clk_dmn *clk = to_dmnclk(hw); - unsigned long fin; - unsigned ratio, wait, hold, reg; - const char *name = clk_hw_get_name(hw); - unsigned bits = (strcmp(name, "mem") == 0) ? 3 : 4; - - fin = parent_rate; - ratio = fin / rate; - - if (unlikely(ratio < 2 || ratio > BIT(bits + 1))) - return -EINVAL; - - WARN_ON(fin % rate); - - wait = (ratio >> 1) - 1; - hold = ratio - wait - 2; - - reg = clkc_readl(clk->regofs); - reg &= ~(((BIT(bits) - 1) << 16) | ((BIT(bits) - 1) << 20)); - reg |= (wait << 16) | (hold << 20) | BIT(25); - clkc_writel(reg, clk->regofs); - - /* waiting FCD been effective */ - while (clkc_readl(clk->regofs) & BIT(25)) - cpu_relax(); - - return 0; -} - -static int cpu_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - int ret1, ret2; - struct clk *cur_parent; - - if (rate == clk_get_rate(clk_pll1.hw.clk)) { - ret1 = clk_set_parent(hw->clk, clk_pll1.hw.clk); - return ret1; - } - - if (rate == clk_get_rate(clk_pll2.hw.clk)) { - ret1 = clk_set_parent(hw->clk, clk_pll2.hw.clk); - return ret1; - } - - if (rate == clk_get_rate(clk_pll3.hw.clk)) { - ret1 = clk_set_parent(hw->clk, clk_pll3.hw.clk); - return ret1; - } - - cur_parent = clk_get_parent(hw->clk); - - /* switch to tmp pll before setting parent clock's rate */ - if (cur_parent == clk_pll1.hw.clk) { - ret1 = clk_set_parent(hw->clk, clk_pll2.hw.clk); - BUG_ON(ret1); - } - - ret2 = clk_set_rate(clk_pll1.hw.clk, rate); - - ret1 = clk_set_parent(hw->clk, clk_pll1.hw.clk); - - return ret2 ? ret2 : ret1; -} - -static const struct clk_ops msi_ops = { - .set_rate = dmn_clk_set_rate, - .round_rate = dmn_clk_round_rate, - .recalc_rate = dmn_clk_recalc_rate, - .set_parent = dmn_clk_set_parent, - .get_parent = dmn_clk_get_parent, -}; - -static const struct clk_init_data clk_mem_init = { - .name = "mem", - .ops = &msi_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_mem = { - .regofs = SIRFSOC_CLKC_MEM_CFG, - .hw = { - .init = &clk_mem_init, - }, -}; - -static const struct clk_init_data clk_sys_init = { - .name = "sys", - .ops = &msi_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), - .flags = CLK_SET_RATE_GATE, -}; - -static struct clk_dmn clk_sys = { - .regofs = SIRFSOC_CLKC_SYS_CFG, - .hw = { - .init = &clk_sys_init, - }, -}; - -static const struct clk_init_data clk_io_init = { - .name = "io", - .ops = &msi_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_io = { - .regofs = SIRFSOC_CLKC_IO_CFG, - .hw = { - .init = &clk_io_init, - }, -}; - -static const struct clk_ops cpu_ops = { - .set_parent = dmn_clk_set_parent, - .get_parent = dmn_clk_get_parent, - .set_rate = cpu_clk_set_rate, - .round_rate = cpu_clk_round_rate, - .recalc_rate = cpu_clk_recalc_rate, -}; - -static const struct clk_init_data clk_cpu_init = { - .name = "cpu", - .ops = &cpu_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), - .flags = CLK_SET_RATE_PARENT, -}; - -static struct clk_dmn clk_cpu = { - .regofs = SIRFSOC_CLKC_CPU_CFG, - .hw = { - .init = &clk_cpu_init, - }, -}; - -static const struct clk_ops dmn_ops = { - .is_enabled = std_clk_is_enabled, - .enable = std_clk_enable, - .disable = std_clk_disable, - .set_rate = dmn_clk_set_rate, - .round_rate = dmn_clk_round_rate, - .recalc_rate = dmn_clk_recalc_rate, - .set_parent = dmn_clk_set_parent, - .get_parent = dmn_clk_get_parent, -}; - -/* dsp, gfx, mm, lcd and vpp domain */ - -static const struct clk_init_data clk_dsp_init = { - .name = "dsp", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_dsp = { - .regofs = SIRFSOC_CLKC_DSP_CFG, - .enable_bit = 0, - .hw = { - .init = &clk_dsp_init, - }, -}; - -static const struct clk_init_data clk_gfx_init = { - .name = "gfx", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_gfx = { - .regofs = SIRFSOC_CLKC_GFX_CFG, - .enable_bit = 8, - .hw = { - .init = &clk_gfx_init, - }, -}; - -static const struct clk_init_data clk_mm_init = { - .name = "mm", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_mm = { - .regofs = SIRFSOC_CLKC_MM_CFG, - .enable_bit = 9, - .hw = { - .init = &clk_mm_init, - }, -}; - -/* - * for atlas6, gfx2d holds the bit of prima2's clk_mm - */ -#define clk_gfx2d clk_mm - -static const struct clk_init_data clk_lcd_init = { - .name = "lcd", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_lcd = { - .regofs = SIRFSOC_CLKC_LCD_CFG, - .enable_bit = 10, - .hw = { - .init = &clk_lcd_init, - }, -}; - -static const struct clk_init_data clk_vpp_init = { - .name = "vpp", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_vpp = { - .regofs = SIRFSOC_CLKC_LCD_CFG, - .enable_bit = 11, - .hw = { - .init = &clk_vpp_init, - }, -}; - -static const struct clk_init_data clk_mmc01_init = { - .name = "mmc01", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static const struct clk_init_data clk_mmc23_init = { - .name = "mmc23", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static const struct clk_init_data clk_mmc45_init = { - .name = "mmc45", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -/* - * peripheral controllers in io domain - */ - -static int std_clk_is_enabled(struct clk_hw *hw) -{ - u32 reg; - int bit; - struct clk_std *clk = to_stdclk(hw); - - bit = clk->enable_bit % 32; - reg = clk->enable_bit / 32; - reg = SIRFSOC_CLKC_CLK_EN0 + reg * sizeof(reg); - - return !!(clkc_readl(reg) & BIT(bit)); -} - -static int std_clk_enable(struct clk_hw *hw) -{ - u32 val, reg; - int bit; - struct clk_std *clk = to_stdclk(hw); - - BUG_ON(clk->enable_bit < 0 || clk->enable_bit > 63); - - bit = clk->enable_bit % 32; - reg = clk->enable_bit / 32; - reg = SIRFSOC_CLKC_CLK_EN0 + reg * sizeof(reg); - - val = clkc_readl(reg) | BIT(bit); - clkc_writel(val, reg); - return 0; -} - -static void std_clk_disable(struct clk_hw *hw) -{ - u32 val, reg; - int bit; - struct clk_std *clk = to_stdclk(hw); - - BUG_ON(clk->enable_bit < 0 || clk->enable_bit > 63); - - bit = clk->enable_bit % 32; - reg = clk->enable_bit / 32; - reg = SIRFSOC_CLKC_CLK_EN0 + reg * sizeof(reg); - - val = clkc_readl(reg) & ~BIT(bit); - clkc_writel(val, reg); -} - -static const char * const std_clk_io_parents[] = { - "io", -}; - -static const struct clk_ops ios_ops = { - .is_enabled = std_clk_is_enabled, - .enable = std_clk_enable, - .disable = std_clk_disable, -}; - -static const struct clk_init_data clk_cphif_init = { - .name = "cphif", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_cphif = { - .enable_bit = 20, - .hw = { - .init = &clk_cphif_init, - }, -}; - -static const struct clk_init_data clk_dmac0_init = { - .name = "dmac0", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_dmac0 = { - .enable_bit = 32, - .hw = { - .init = &clk_dmac0_init, - }, -}; - -static const struct clk_init_data clk_dmac1_init = { - .name = "dmac1", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_dmac1 = { - .enable_bit = 33, - .hw = { - .init = &clk_dmac1_init, - }, -}; - -static const struct clk_init_data clk_audio_init = { - .name = "audio", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_audio = { - .enable_bit = 35, - .hw = { - .init = &clk_audio_init, - }, -}; - -static const struct clk_init_data clk_uart0_init = { - .name = "uart0", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_uart0 = { - .enable_bit = 36, - .hw = { - .init = &clk_uart0_init, - }, -}; - -static const struct clk_init_data clk_uart1_init = { - .name = "uart1", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_uart1 = { - .enable_bit = 37, - .hw = { - .init = &clk_uart1_init, - }, -}; - -static const struct clk_init_data clk_uart2_init = { - .name = "uart2", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_uart2 = { - .enable_bit = 38, - .hw = { - .init = &clk_uart2_init, - }, -}; - -static const struct clk_init_data clk_usp0_init = { - .name = "usp0", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_usp0 = { - .enable_bit = 39, - .hw = { - .init = &clk_usp0_init, - }, -}; - -static const struct clk_init_data clk_usp1_init = { - .name = "usp1", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_usp1 = { - .enable_bit = 40, - .hw = { - .init = &clk_usp1_init, - }, -}; - -static const struct clk_init_data clk_usp2_init = { - .name = "usp2", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_usp2 = { - .enable_bit = 41, - .hw = { - .init = &clk_usp2_init, - }, -}; - -static const struct clk_init_data clk_vip_init = { - .name = "vip", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_vip = { - .enable_bit = 42, - .hw = { - .init = &clk_vip_init, - }, -}; - -static const struct clk_init_data clk_spi0_init = { - .name = "spi0", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_spi0 = { - .enable_bit = 43, - .hw = { - .init = &clk_spi0_init, - }, -}; - -static const struct clk_init_data clk_spi1_init = { - .name = "spi1", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_spi1 = { - .enable_bit = 44, - .hw = { - .init = &clk_spi1_init, - }, -}; - -static const struct clk_init_data clk_tsc_init = { - .name = "tsc", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_tsc = { - .enable_bit = 45, - .hw = { - .init = &clk_tsc_init, - }, -}; - -static const struct clk_init_data clk_i2c0_init = { - .name = "i2c0", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_i2c0 = { - .enable_bit = 46, - .hw = { - .init = &clk_i2c0_init, - }, -}; - -static const struct clk_init_data clk_i2c1_init = { - .name = "i2c1", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_i2c1 = { - .enable_bit = 47, - .hw = { - .init = &clk_i2c1_init, - }, -}; - -static const struct clk_init_data clk_pwmc_init = { - .name = "pwmc", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_pwmc = { - .enable_bit = 48, - .hw = { - .init = &clk_pwmc_init, - }, -}; - -static const struct clk_init_data clk_efuse_init = { - .name = "efuse", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_efuse = { - .enable_bit = 49, - .hw = { - .init = &clk_efuse_init, - }, -}; - -static const struct clk_init_data clk_pulse_init = { - .name = "pulse", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_pulse = { - .enable_bit = 50, - .hw = { - .init = &clk_pulse_init, - }, -}; - -static const char * const std_clk_dsp_parents[] = { - "dsp", -}; - -static const struct clk_init_data clk_gps_init = { - .name = "gps", - .ops = &ios_ops, - .parent_names = std_clk_dsp_parents, - .num_parents = ARRAY_SIZE(std_clk_dsp_parents), -}; - -static struct clk_std clk_gps = { - .enable_bit = 1, - .hw = { - .init = &clk_gps_init, - }, -}; - -static const struct clk_init_data clk_mf_init = { - .name = "mf", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_mf = { - .enable_bit = 2, - .hw = { - .init = &clk_mf_init, - }, -}; - -static const char * const std_clk_sys_parents[] = { - "sys", -}; - -static const struct clk_init_data clk_security_init = { - .name = "security", - .ops = &ios_ops, - .parent_names = std_clk_sys_parents, - .num_parents = ARRAY_SIZE(std_clk_sys_parents), -}; - -static struct clk_std clk_security = { - .enable_bit = 19, - .hw = { - .init = &clk_security_init, - }, -}; - -static const char * const std_clk_usb_parents[] = { - "usb_pll", -}; - -static const struct clk_init_data clk_usb0_init = { - .name = "usb0", - .ops = &ios_ops, - .parent_names = std_clk_usb_parents, - .num_parents = ARRAY_SIZE(std_clk_usb_parents), -}; - -static struct clk_std clk_usb0 = { - .enable_bit = 16, - .hw = { - .init = &clk_usb0_init, - }, -}; - -static const struct clk_init_data clk_usb1_init = { - .name = "usb1", - .ops = &ios_ops, - .parent_names = std_clk_usb_parents, - .num_parents = ARRAY_SIZE(std_clk_usb_parents), -}; - -static struct clk_std clk_usb1 = { - .enable_bit = 17, - .hw = { - .init = &clk_usb1_init, - }, -}; diff --git a/drivers/clk/sirf/clk-prima2.c b/drivers/clk/sirf/clk-prima2.c deleted file mode 100644 index d17b345f4d2d..000000000000 --- a/drivers/clk/sirf/clk-prima2.c +++ /dev/null @@ -1,149 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Clock tree for CSR SiRFprimaII - * - * Copyright (c) 2011 - 2014 Cambridge Silicon Radio Limited, a CSR plc group - * company. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "prima2.h" -#include "clk-common.c" - -static struct clk_dmn clk_mmc01 = { - .regofs = SIRFSOC_CLKC_MMC_CFG, - .enable_bit = 59, - .hw = { - .init = &clk_mmc01_init, - }, -}; - -static struct clk_dmn clk_mmc23 = { - .regofs = SIRFSOC_CLKC_MMC_CFG, - .enable_bit = 60, - .hw = { - .init = &clk_mmc23_init, - }, -}; - -static struct clk_dmn clk_mmc45 = { - .regofs = SIRFSOC_CLKC_MMC_CFG, - .enable_bit = 61, - .hw = { - .init = &clk_mmc45_init, - }, -}; - -static const struct clk_init_data clk_nand_init = { - .name = "nand", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_nand = { - .enable_bit = 34, - .hw = { - .init = &clk_nand_init, - }, -}; - -enum prima2_clk_index { - /* 0 1 2 3 4 5 6 7 8 9 */ - rtc, osc, pll1, pll2, pll3, mem, sys, security, dsp, gps, - mf, io, cpu, uart0, uart1, uart2, tsc, i2c0, i2c1, spi0, - spi1, pwmc, efuse, pulse, dmac0, dmac1, nand, audio, usp0, usp1, - usp2, vip, gfx, mm, lcd, vpp, mmc01, mmc23, mmc45, usbpll, - usb0, usb1, cphif, maxclk, -}; - -static __initdata struct clk_hw *prima2_clk_hw_array[maxclk] = { - NULL, /* dummy */ - NULL, - &clk_pll1.hw, - &clk_pll2.hw, - &clk_pll3.hw, - &clk_mem.hw, - &clk_sys.hw, - &clk_security.hw, - &clk_dsp.hw, - &clk_gps.hw, - &clk_mf.hw, - &clk_io.hw, - &clk_cpu.hw, - &clk_uart0.hw, - &clk_uart1.hw, - &clk_uart2.hw, - &clk_tsc.hw, - &clk_i2c0.hw, - &clk_i2c1.hw, - &clk_spi0.hw, - &clk_spi1.hw, - &clk_pwmc.hw, - &clk_efuse.hw, - &clk_pulse.hw, - &clk_dmac0.hw, - &clk_dmac1.hw, - &clk_nand.hw, - &clk_audio.hw, - &clk_usp0.hw, - &clk_usp1.hw, - &clk_usp2.hw, - &clk_vip.hw, - &clk_gfx.hw, - &clk_mm.hw, - &clk_lcd.hw, - &clk_vpp.hw, - &clk_mmc01.hw, - &clk_mmc23.hw, - &clk_mmc45.hw, - &usb_pll_clk_hw, - &clk_usb0.hw, - &clk_usb1.hw, - &clk_cphif.hw, -}; - -static struct clk *prima2_clks[maxclk]; - -static void __init prima2_clk_init(struct device_node *np) -{ - struct device_node *rscnp; - int i; - - rscnp = of_find_compatible_node(NULL, NULL, "sirf,prima2-rsc"); - sirfsoc_rsc_vbase = of_iomap(rscnp, 0); - if (!sirfsoc_rsc_vbase) - panic("unable to map rsc registers\n"); - of_node_put(rscnp); - - sirfsoc_clk_vbase = of_iomap(np, 0); - if (!sirfsoc_clk_vbase) - panic("unable to map clkc registers\n"); - - /* These are always available (RTC and 26MHz OSC)*/ - prima2_clks[rtc] = clk_register_fixed_rate(NULL, "rtc", NULL, 0, 32768); - prima2_clks[osc] = clk_register_fixed_rate(NULL, "osc", NULL, 0, - 26000000); - - for (i = pll1; i < maxclk; i++) { - prima2_clks[i] = clk_register(NULL, prima2_clk_hw_array[i]); - BUG_ON(IS_ERR(prima2_clks[i])); - } - clk_register_clkdev(prima2_clks[cpu], NULL, "cpu"); - clk_register_clkdev(prima2_clks[io], NULL, "io"); - clk_register_clkdev(prima2_clks[mem], NULL, "mem"); - clk_register_clkdev(prima2_clks[mem], NULL, "osc"); - - clk_data.clks = prima2_clks; - clk_data.clk_num = maxclk; - - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); -} -CLK_OF_DECLARE(prima2_clk, "sirf,prima2-clkc", prima2_clk_init); diff --git a/drivers/clk/sirf/prima2.h b/drivers/clk/sirf/prima2.h deleted file mode 100644 index 2fb56941795d..000000000000 --- a/drivers/clk/sirf/prima2.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#define SIRFSOC_CLKC_CLK_EN0 0x0000 -#define SIRFSOC_CLKC_CLK_EN1 0x0004 -#define SIRFSOC_CLKC_REF_CFG 0x0014 -#define SIRFSOC_CLKC_CPU_CFG 0x0018 -#define SIRFSOC_CLKC_MEM_CFG 0x001c -#define SIRFSOC_CLKC_SYS_CFG 0x0020 -#define SIRFSOC_CLKC_IO_CFG 0x0024 -#define SIRFSOC_CLKC_DSP_CFG 0x0028 -#define SIRFSOC_CLKC_GFX_CFG 0x002c -#define SIRFSOC_CLKC_MM_CFG 0x0030 -#define SIRFSOC_CLKC_LCD_CFG 0x0034 -#define SIRFSOC_CLKC_MMC_CFG 0x0038 -#define SIRFSOC_CLKC_PLL1_CFG0 0x0040 -#define SIRFSOC_CLKC_PLL2_CFG0 0x0044 -#define SIRFSOC_CLKC_PLL3_CFG0 0x0048 -#define SIRFSOC_CLKC_PLL1_CFG1 0x004c -#define SIRFSOC_CLKC_PLL2_CFG1 0x0050 -#define SIRFSOC_CLKC_PLL3_CFG1 0x0054 -#define SIRFSOC_CLKC_PLL1_CFG2 0x0058 -#define SIRFSOC_CLKC_PLL2_CFG2 0x005c -#define SIRFSOC_CLKC_PLL3_CFG2 0x0060 -#define SIRFSOC_USBPHY_PLL_CTRL 0x0008 -#define SIRFSOC_USBPHY_PLL_POWERDOWN BIT(1) -#define SIRFSOC_USBPHY_PLL_BYPASS BIT(2) -#define SIRFSOC_USBPHY_PLL_LOCK BIT(3) diff --git a/drivers/clk/socfpga/clk-agilex.c b/drivers/clk/socfpga/clk-agilex.c index bb3e80928ebe..7689bdd0a914 100644 --- a/drivers/clk/socfpga/clk-agilex.c +++ b/drivers/clk/socfpga/clk-agilex.c @@ -196,6 +196,17 @@ static const struct stratix10_pll_clock agilex_pll_clks[] = { 0, 0x9c}, }; +static const struct n5x_perip_c_clock n5x_main_perip_c_clks[] = { + { AGILEX_MAIN_PLL_C0_CLK, "main_pll_c0", "main_pll", NULL, 1, 0, 0x54, 0}, + { AGILEX_MAIN_PLL_C1_CLK, "main_pll_c1", "main_pll", NULL, 1, 0, 0x54, 8}, + { AGILEX_MAIN_PLL_C2_CLK, "main_pll_c2", "main_pll", NULL, 1, 0, 0x54, 16}, + { AGILEX_MAIN_PLL_C3_CLK, "main_pll_c3", "main_pll", NULL, 1, 0, 0x54, 24}, + { AGILEX_PERIPH_PLL_C0_CLK, "peri_pll_c0", "periph_pll", NULL, 1, 0, 0xA8, 0}, + { AGILEX_PERIPH_PLL_C1_CLK, "peri_pll_c1", "periph_pll", NULL, 1, 0, 0xA8, 8}, + { AGILEX_PERIPH_PLL_C2_CLK, "peri_pll_c2", "periph_pll", NULL, 1, 0, 0xA8, 16}, + { AGILEX_PERIPH_PLL_C3_CLK, "peri_pll_c3", "periph_pll", NULL, 1, 0, 0xA8, 24}, +}; + static const struct stratix10_perip_c_clock agilex_main_perip_c_clks[] = { { AGILEX_MAIN_PLL_C0_CLK, "main_pll_c0", "main_pll", NULL, 1, 0, 0x58}, { AGILEX_MAIN_PLL_C1_CLK, "main_pll_c1", "main_pll", NULL, 1, 0, 0x5C}, @@ -289,6 +300,25 @@ static const struct stratix10_gate_clock agilex_gate_clks[] = { 10, 0, 0, 0, 0, 0, 4}, }; +static int n5x_clk_register_c_perip(const struct n5x_perip_c_clock *clks, + int nums, struct stratix10_clock_data *data) +{ + struct clk *clk; + void __iomem *base = data->base; + int i; + + for (i = 0; i < nums; i++) { + clk = n5x_register_periph(&clks[i], base); + if (IS_ERR(clk)) { + pr_err("%s: failed to register clock %s\n", + __func__, clks[i].name); + continue; + } + data->clk_data.clks[clks[i].id] = clk; + } + return 0; +} + static int agilex_clk_register_c_perip(const struct stratix10_perip_c_clock *clks, int nums, struct stratix10_clock_data *data) { @@ -367,6 +397,26 @@ static int agilex_clk_register_pll(const struct stratix10_pll_clock *clks, return 0; } +static int n5x_clk_register_pll(const struct stratix10_pll_clock *clks, + int nums, struct stratix10_clock_data *data) +{ + struct clk *clk; + void __iomem *base = data->base; + int i; + + for (i = 0; i < nums; i++) { + clk = n5x_register_pll(&clks[i], base); + if (IS_ERR(clk)) { + pr_err("%s: failed to register clock %s\n", + __func__, clks[i].name); + continue; + } + data->clk_data.clks[clks[i].id] = clk; + } + + return 0; +} + static struct stratix10_clock_data *__socfpga_agilex_clk_init(struct platform_device *pdev, int nr_clks) { @@ -401,7 +451,7 @@ static struct stratix10_clock_data *__socfpga_agilex_clk_init(struct platform_de return clk_data; } -static int agilex_clkmgr_probe(struct platform_device *pdev) +static int agilex_clkmgr_init(struct platform_device *pdev) { struct stratix10_clock_data *clk_data; @@ -423,9 +473,43 @@ static int agilex_clkmgr_probe(struct platform_device *pdev) return 0; } +static int n5x_clkmgr_init(struct platform_device *pdev) +{ + struct stratix10_clock_data *clk_data; + + clk_data = __socfpga_agilex_clk_init(pdev, AGILEX_NUM_CLKS); + if (IS_ERR(clk_data)) + return PTR_ERR(clk_data); + + n5x_clk_register_pll(agilex_pll_clks, ARRAY_SIZE(agilex_pll_clks), clk_data); + + n5x_clk_register_c_perip(n5x_main_perip_c_clks, + ARRAY_SIZE(n5x_main_perip_c_clks), clk_data); + + agilex_clk_register_cnt_perip(agilex_main_perip_cnt_clks, + ARRAY_SIZE(agilex_main_perip_cnt_clks), + clk_data); + + agilex_clk_register_gate(agilex_gate_clks, ARRAY_SIZE(agilex_gate_clks), + clk_data); + return 0; +} + +static int agilex_clkmgr_probe(struct platform_device *pdev) +{ + int (*probe_func)(struct platform_device *init_func); + + probe_func = of_device_get_match_data(&pdev->dev); + if (!probe_func) + return -ENODEV; + return probe_func(pdev); +} + static const struct of_device_id agilex_clkmgr_match_table[] = { { .compatible = "intel,agilex-clkmgr", - .data = agilex_clkmgr_probe }, + .data = agilex_clkmgr_init }, + { .compatible = "intel,easic-n5x-clkmgr", + .data = n5x_clkmgr_init }, { } }; diff --git a/drivers/clk/socfpga/clk-periph-s10.c b/drivers/clk/socfpga/clk-periph-s10.c index 397b77b89b16..0ff2b9d24035 100644 --- a/drivers/clk/socfpga/clk-periph-s10.c +++ b/drivers/clk/socfpga/clk-periph-s10.c @@ -15,6 +15,21 @@ #define to_periph_clk(p) container_of(p, struct socfpga_periph_clk, hw.hw) +static unsigned long n5x_clk_peri_c_clk_recalc_rate(struct clk_hw *hwclk, + unsigned long parent_rate) +{ + struct socfpga_periph_clk *socfpgaclk = to_periph_clk(hwclk); + unsigned long div; + unsigned long shift = socfpgaclk->shift; + u32 val; + + val = readl(socfpgaclk->hw.reg); + val &= (0x1f << shift); + div = (val >> shift) + 1; + + return parent_rate / div; +} + static unsigned long clk_peri_c_clk_recalc_rate(struct clk_hw *hwclk, unsigned long parent_rate) { @@ -63,6 +78,11 @@ static u8 clk_periclk_get_parent(struct clk_hw *hwclk) return parent; } +static const struct clk_ops n5x_peri_c_clk_ops = { + .recalc_rate = n5x_clk_peri_c_clk_recalc_rate, + .get_parent = clk_periclk_get_parent, +}; + static const struct clk_ops peri_c_clk_ops = { .recalc_rate = clk_peri_c_clk_recalc_rate, .get_parent = clk_periclk_get_parent, @@ -107,6 +127,39 @@ struct clk *s10_register_periph(const struct stratix10_perip_c_clock *clks, return clk; } +struct clk *n5x_register_periph(const struct n5x_perip_c_clock *clks, + void __iomem *regbase) +{ + struct clk *clk; + struct socfpga_periph_clk *periph_clk; + struct clk_init_data init; + const char *name = clks->name; + const char *parent_name = clks->parent_name; + + periph_clk = kzalloc(sizeof(*periph_clk), GFP_KERNEL); + if (WARN_ON(!periph_clk)) + return NULL; + + periph_clk->hw.reg = regbase + clks->offset; + periph_clk->shift = clks->shift; + + init.name = name; + init.ops = &n5x_peri_c_clk_ops; + init.flags = clks->flags; + + init.num_parents = clks->num_parents; + init.parent_names = parent_name ? &parent_name : NULL; + + periph_clk->hw.hw.init = &init; + + clk = clk_register(NULL, &periph_clk->hw.hw); + if (WARN_ON(IS_ERR(clk))) { + kfree(periph_clk); + return NULL; + } + return clk; +} + struct clk *s10_register_cnt_periph(const struct stratix10_perip_cnt_clock *clks, void __iomem *regbase) { diff --git a/drivers/clk/socfpga/clk-pll-a10.c b/drivers/clk/socfpga/clk-pll-a10.c index db54f7d806a0..3338f054fe98 100644 --- a/drivers/clk/socfpga/clk-pll-a10.c +++ b/drivers/clk/socfpga/clk-pll-a10.c @@ -73,7 +73,6 @@ static struct clk * __init __socfpga_pll_init(struct device_node *node, const char *parent_name[SOCFGPA_MAX_PARENTS]; struct clk_init_data init; struct device_node *clkmgr_np; - int rc; int i = 0; of_property_read_u32(node, "reg", ®); @@ -108,7 +107,7 @@ static struct clk * __init __socfpga_pll_init(struct device_node *node, kfree(pll_clk); return NULL; } - rc = of_clk_add_provider(node, of_clk_src_simple_get, clk); + of_clk_add_provider(node, of_clk_src_simple_get, clk); return clk; } diff --git a/drivers/clk/socfpga/clk-pll-s10.c b/drivers/clk/socfpga/clk-pll-s10.c index 4e268953b7da..f6f66e08e1f4 100644 --- a/drivers/clk/socfpga/clk-pll-s10.c +++ b/drivers/clk/socfpga/clk-pll-s10.c @@ -27,10 +27,37 @@ #define SWCTRLBTCLKSEL_MASK 0x200 #define SWCTRLBTCLKSEL_SHIFT 9 +#define SOCFPGA_N5X_PLLDIV_FDIV_MASK GENMASK(16, 8) +#define SOCFPGA_N5X_PLLDIV_FDIV_SHIFT 8 +#define SOCFPGA_N5X_PLLDIV_RDIV_MASK GENMASK(5, 0) +#define SOCFPGA_N5X_PLLDIV_QDIV_MASK GENMASK(26, 24) +#define SOCFPGA_N5X_PLLDIV_QDIV_SHIFT 24 + #define SOCFPGA_BOOT_CLK "boot_clk" #define to_socfpga_clk(p) container_of(p, struct socfpga_pll, hw.hw) +static unsigned long n5x_clk_pll_recalc_rate(struct clk_hw *hwclk, + unsigned long parent_rate) +{ + struct socfpga_pll *socfpgaclk = to_socfpga_clk(hwclk); + unsigned long fdiv, reg, rdiv, qdiv; + u32 power = 1; + + /* read VCO1 reg for numerator and denominator */ + reg = readl(socfpgaclk->hw.reg + 0x8); + fdiv = (reg & SOCFPGA_N5X_PLLDIV_FDIV_MASK) >> SOCFPGA_N5X_PLLDIV_FDIV_SHIFT; + rdiv = (reg & SOCFPGA_N5X_PLLDIV_RDIV_MASK); + qdiv = (reg & SOCFPGA_N5X_PLLDIV_QDIV_MASK) >> SOCFPGA_N5X_PLLDIV_QDIV_SHIFT; + + while (qdiv) { + power *= 2; + qdiv--; + } + + return ((parent_rate * 2 * (fdiv + 1)) / ((rdiv + 1) * power)); +} + static unsigned long agilex_clk_pll_recalc_rate(struct clk_hw *hwclk, unsigned long parent_rate) { @@ -123,6 +150,25 @@ static int clk_pll_prepare(struct clk_hw *hwclk) return 0; } +static int n5x_clk_pll_prepare(struct clk_hw *hwclk) +{ + struct socfpga_pll *socfpgaclk = to_socfpga_clk(hwclk); + u32 reg; + + /* Bring PLL out of reset */ + reg = readl(socfpgaclk->hw.reg + 0x4); + reg |= SOCFPGA_PLL_RESET_MASK; + writel(reg, socfpgaclk->hw.reg + 0x4); + + return 0; +} + +static const struct clk_ops n5x_clk_pll_ops = { + .recalc_rate = n5x_clk_pll_recalc_rate, + .get_parent = clk_pll_get_parent, + .prepare = n5x_clk_pll_prepare, +}; + static const struct clk_ops agilex_clk_pll_ops = { .recalc_rate = agilex_clk_pll_recalc_rate, .get_parent = clk_pll_get_parent, @@ -214,3 +260,40 @@ struct clk *agilex_register_pll(const struct stratix10_pll_clock *clks, } return clk; } + +struct clk *n5x_register_pll(const struct stratix10_pll_clock *clks, + void __iomem *reg) +{ + struct clk *clk; + struct socfpga_pll *pll_clk; + struct clk_init_data init; + const char *name = clks->name; + + pll_clk = kzalloc(sizeof(*pll_clk), GFP_KERNEL); + if (WARN_ON(!pll_clk)) + return NULL; + + pll_clk->hw.reg = reg + clks->offset; + + if (streq(name, SOCFPGA_BOOT_CLK)) + init.ops = &clk_boot_ops; + else + init.ops = &n5x_clk_pll_ops; + + init.name = name; + init.flags = clks->flags; + + init.num_parents = clks->num_parents; + init.parent_names = NULL; + init.parent_data = clks->parent_data; + pll_clk->hw.hw.init = &init; + + pll_clk->hw.bit_idx = SOCFPGA_PLL_POWER; + + clk = clk_register(NULL, &pll_clk->hw.hw); + if (WARN_ON(IS_ERR(clk))) { + kfree(pll_clk); + return NULL; + } + return clk; +} diff --git a/drivers/clk/socfpga/clk-pll.c b/drivers/clk/socfpga/clk-pll.c index e5fb786843f3..3cf99df7d005 100644 --- a/drivers/clk/socfpga/clk-pll.c +++ b/drivers/clk/socfpga/clk-pll.c @@ -80,7 +80,6 @@ static __init struct clk *__socfpga_pll_init(struct device_node *node, const char *parent_name[SOCFPGA_MAX_PARENTS]; struct clk_init_data init; struct device_node *clkmgr_np; - int rc; of_property_read_u32(node, "reg", ®); @@ -111,7 +110,7 @@ static __init struct clk *__socfpga_pll_init(struct device_node *node, kfree(pll_clk); return NULL; } - rc = of_clk_add_provider(node, of_clk_src_simple_get, clk); + of_clk_add_provider(node, of_clk_src_simple_get, clk); return clk; } diff --git a/drivers/clk/socfpga/stratix10-clk.h b/drivers/clk/socfpga/stratix10-clk.h index f9d5d724c694..420deed677ce 100644 --- a/drivers/clk/socfpga/stratix10-clk.h +++ b/drivers/clk/socfpga/stratix10-clk.h @@ -30,6 +30,17 @@ struct stratix10_perip_c_clock { unsigned long offset; }; +struct n5x_perip_c_clock { + unsigned int id; + const char *name; + const char *parent_name; + const char *const *parent_names; + u8 num_parents; + unsigned long flags; + unsigned long offset; + unsigned long shift; +}; + struct stratix10_perip_cnt_clock { unsigned int id; const char *name; @@ -64,8 +75,12 @@ struct clk *s10_register_pll(const struct stratix10_pll_clock *, void __iomem *); struct clk *agilex_register_pll(const struct stratix10_pll_clock *, void __iomem *); +struct clk *n5x_register_pll(const struct stratix10_pll_clock *clks, + void __iomem *reg); struct clk *s10_register_periph(const struct stratix10_perip_c_clock *, - void __iomem *); + void __iomem *reg); +struct clk *n5x_register_periph(const struct n5x_perip_c_clock *clks, + void __iomem *reg); struct clk *s10_register_cnt_periph(const struct stratix10_perip_cnt_clock *, void __iomem *); struct clk *s10_register_gate(const struct stratix10_gate_clock *, diff --git a/drivers/clk/spear/spear1310_clock.c b/drivers/clk/spear/spear1310_clock.c index 591248c9a88e..8c8974866789 100644 --- a/drivers/clk/spear/spear1310_clock.c +++ b/drivers/clk/spear/spear1310_clock.c @@ -12,6 +12,7 @@ */ #include +#include #include #include #include diff --git a/drivers/clk/spear/spear1340_clock.c b/drivers/clk/spear/spear1340_clock.c index 9163bbb46411..c0dc94355c87 100644 --- a/drivers/clk/spear/spear1340_clock.c +++ b/drivers/clk/spear/spear1340_clock.c @@ -12,6 +12,7 @@ */ #include +#include #include #include #include diff --git a/drivers/clk/st/clkgen-fsyn.c b/drivers/clk/st/clkgen-fsyn.c index f1adc858b590..dd6062e043e0 100644 --- a/drivers/clk/st/clkgen-fsyn.c +++ b/drivers/clk/st/clkgen-fsyn.c @@ -172,10 +172,10 @@ static const struct clkgen_quadfs_data st_fs660c32_D = { * ST quad channel frequency synthesizer block * * @hw: handle between common and hardware-specific interfaces. - * @ndiv: regmap field for the ndiv control. * @regs_base: base address of the configuration registers. * @lock: spinlock. - * + * @data: local driver data + * @ndiv: regmap field for the ndiv control. */ struct st_clk_quadfs_pll { struct clk_hw hw; @@ -426,7 +426,7 @@ static struct clk * __init st_clk_register_quadfs_pll( * parent - fixed parent. No clk_set_parent support */ -/** +/* * struct st_clk_quadfs_fsynth - One clock output from a four channel digital * frequency synthesizer (fsynth) block. * diff --git a/drivers/clk/st/clkgen-pll.c b/drivers/clk/st/clkgen-pll.c index c3952f2c42ba..119c5b33080c 100644 --- a/drivers/clk/st/clkgen-pll.c +++ b/drivers/clk/st/clkgen-pll.c @@ -130,12 +130,11 @@ static struct clkgen_pll_data st_pll4600c28_418_a9 = { * parent - fixed parent. No clk_set_parent support */ -/** +/* * PLL clock that is integrated in the ClockGenA instances on the STiH415 * and STiH416. * * @hw: handle between common and hardware-specific interfaces. - * @type: PLL instance type. * @regs_base: base of the PLL configuration register(s). * */ diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig index ce5f5847d5d3..cd46d8853876 100644 --- a/drivers/clk/sunxi-ng/Kconfig +++ b/drivers/clk/sunxi-ng/Kconfig @@ -32,8 +32,13 @@ config SUN50I_H6_CCU default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST +config SUN50I_H616_CCU + bool "Support for the Allwinner H616 CCU" + default ARM64 && ARCH_SUNXI + depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST + config SUN50I_H6_R_CCU - bool "Support for the Allwinner H6 PRCM CCU" + bool "Support for the Allwinner H6 and H616 PRCM CCU" default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile index 3eb5cff40eac..96c324306d97 100644 --- a/drivers/clk/sunxi-ng/Makefile +++ b/drivers/clk/sunxi-ng/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_SUN50I_A64_CCU) += ccu-sun50i-a64.o obj-$(CONFIG_SUN50I_A100_CCU) += ccu-sun50i-a100.o obj-$(CONFIG_SUN50I_A100_R_CCU) += ccu-sun50i-a100-r.o obj-$(CONFIG_SUN50I_H6_CCU) += ccu-sun50i-h6.o +obj-$(CONFIG_SUN50I_H616_CCU) += ccu-sun50i-h616.o obj-$(CONFIG_SUN50I_H6_R_CCU) += ccu-sun50i-h6-r.o obj-$(CONFIG_SUN4I_A10_CCU) += ccu-sun4i-a10.o obj-$(CONFIG_SUN5I_CCU) += ccu-sun5i.o diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c index 50f8d1bc7046..f8909a7ed553 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c @@ -91,6 +91,8 @@ static SUNXI_CCU_GATE(r_apb2_uart_clk, "r-apb2-uart", "r-apb2", 0x18c, BIT(0), 0); static SUNXI_CCU_GATE(r_apb2_i2c_clk, "r-apb2-i2c", "r-apb2", 0x19c, BIT(0), 0); +static SUNXI_CCU_GATE(r_apb2_rsb_clk, "r-apb2-rsb", "r-apb2", + 0x1bc, BIT(0), 0); static SUNXI_CCU_GATE(r_apb1_ir_clk, "r-apb1-ir", "r-apb1", 0x1cc, BIT(0), 0); static SUNXI_CCU_GATE(r_apb1_w1_clk, "r-apb1-w1", "r-apb1", @@ -130,12 +132,23 @@ static struct ccu_common *sun50i_h6_r_ccu_clks[] = { &r_apb1_pwm_clk.common, &r_apb2_uart_clk.common, &r_apb2_i2c_clk.common, + &r_apb2_rsb_clk.common, &r_apb1_ir_clk.common, &r_apb1_w1_clk.common, &ir_clk.common, &w1_clk.common, }; +static struct ccu_common *sun50i_h616_r_ccu_clks[] = { + &r_apb1_clk.common, + &r_apb2_clk.common, + &r_apb1_twd_clk.common, + &r_apb2_i2c_clk.common, + &r_apb2_rsb_clk.common, + &r_apb1_ir_clk.common, + &ir_clk.common, +}; + static struct clk_hw_onecell_data sun50i_h6_r_hw_clks = { .hws = { [CLK_AR100] = &ar100_clk.common.hw, @@ -147,6 +160,7 @@ static struct clk_hw_onecell_data sun50i_h6_r_hw_clks = { [CLK_R_APB1_PWM] = &r_apb1_pwm_clk.common.hw, [CLK_R_APB2_UART] = &r_apb2_uart_clk.common.hw, [CLK_R_APB2_I2C] = &r_apb2_i2c_clk.common.hw, + [CLK_R_APB2_RSB] = &r_apb2_rsb_clk.common.hw, [CLK_R_APB1_IR] = &r_apb1_ir_clk.common.hw, [CLK_R_APB1_W1] = &r_apb1_w1_clk.common.hw, [CLK_IR] = &ir_clk.common.hw, @@ -155,16 +169,38 @@ static struct clk_hw_onecell_data sun50i_h6_r_hw_clks = { .num = CLK_NUMBER, }; +static struct clk_hw_onecell_data sun50i_h616_r_hw_clks = { + .hws = { + [CLK_R_AHB] = &r_ahb_clk.hw, + [CLK_R_APB1] = &r_apb1_clk.common.hw, + [CLK_R_APB2] = &r_apb2_clk.common.hw, + [CLK_R_APB1_TWD] = &r_apb1_twd_clk.common.hw, + [CLK_R_APB2_I2C] = &r_apb2_i2c_clk.common.hw, + [CLK_R_APB2_RSB] = &r_apb2_rsb_clk.common.hw, + [CLK_R_APB1_IR] = &r_apb1_ir_clk.common.hw, + [CLK_IR] = &ir_clk.common.hw, + }, + .num = CLK_NUMBER, +}; + static struct ccu_reset_map sun50i_h6_r_ccu_resets[] = { [RST_R_APB1_TIMER] = { 0x11c, BIT(16) }, [RST_R_APB1_TWD] = { 0x12c, BIT(16) }, [RST_R_APB1_PWM] = { 0x13c, BIT(16) }, [RST_R_APB2_UART] = { 0x18c, BIT(16) }, [RST_R_APB2_I2C] = { 0x19c, BIT(16) }, + [RST_R_APB2_RSB] = { 0x1bc, BIT(16) }, [RST_R_APB1_IR] = { 0x1cc, BIT(16) }, [RST_R_APB1_W1] = { 0x1ec, BIT(16) }, }; +static struct ccu_reset_map sun50i_h616_r_ccu_resets[] = { + [RST_R_APB1_TWD] = { 0x12c, BIT(16) }, + [RST_R_APB2_I2C] = { 0x19c, BIT(16) }, + [RST_R_APB2_RSB] = { 0x1bc, BIT(16) }, + [RST_R_APB1_IR] = { 0x1cc, BIT(16) }, +}; + static const struct sunxi_ccu_desc sun50i_h6_r_ccu_desc = { .ccu_clks = sun50i_h6_r_ccu_clks, .num_ccu_clks = ARRAY_SIZE(sun50i_h6_r_ccu_clks), @@ -175,6 +211,16 @@ static const struct sunxi_ccu_desc sun50i_h6_r_ccu_desc = { .num_resets = ARRAY_SIZE(sun50i_h6_r_ccu_resets), }; +static const struct sunxi_ccu_desc sun50i_h616_r_ccu_desc = { + .ccu_clks = sun50i_h616_r_ccu_clks, + .num_ccu_clks = ARRAY_SIZE(sun50i_h616_r_ccu_clks), + + .hw_clks = &sun50i_h616_r_hw_clks, + + .resets = sun50i_h616_r_ccu_resets, + .num_resets = ARRAY_SIZE(sun50i_h616_r_ccu_resets), +}; + static void __init sunxi_r_ccu_init(struct device_node *node, const struct sunxi_ccu_desc *desc) { @@ -195,3 +241,10 @@ static void __init sun50i_h6_r_ccu_setup(struct device_node *node) } CLK_OF_DECLARE(sun50i_h6_r_ccu, "allwinner,sun50i-h6-r-ccu", sun50i_h6_r_ccu_setup); + +static void __init sun50i_h616_r_ccu_setup(struct device_node *node) +{ + sunxi_r_ccu_init(node, &sun50i_h616_r_ccu_desc); +} +CLK_OF_DECLARE(sun50i_h616_r_ccu, "allwinner,sun50i-h616-r-ccu", + sun50i_h616_r_ccu_setup); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h index 782117dc0b28..7e290b840803 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h @@ -14,6 +14,6 @@ #define CLK_R_APB2 3 -#define CLK_NUMBER (CLK_W1 + 1) +#define CLK_NUMBER (CLK_R_APB2_RSB + 1) #endif /* _CCU_SUN50I_H6_R_H */ diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c index f2497d0a4683..bff446b78290 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c @@ -237,7 +237,7 @@ static const char * const psi_ahb1_ahb2_parents[] = { "osc24M", "osc32k", static SUNXI_CCU_MP_WITH_MUX(psi_ahb1_ahb2_clk, "psi-ahb1-ahb2", psi_ahb1_ahb2_parents, 0x510, - 0, 5, /* M */ + 0, 2, /* M */ 8, 2, /* P */ 24, 2, /* mux */ 0); @@ -246,19 +246,19 @@ static const char * const ahb3_apb1_apb2_parents[] = { "osc24M", "osc32k", "psi-ahb1-ahb2", "pll-periph0" }; static SUNXI_CCU_MP_WITH_MUX(ahb3_clk, "ahb3", ahb3_apb1_apb2_parents, 0x51c, - 0, 5, /* M */ + 0, 2, /* M */ 8, 2, /* P */ 24, 2, /* mux */ 0); static SUNXI_CCU_MP_WITH_MUX(apb1_clk, "apb1", ahb3_apb1_apb2_parents, 0x520, - 0, 5, /* M */ + 0, 2, /* M */ 8, 2, /* P */ 24, 2, /* mux */ 0); static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", ahb3_apb1_apb2_parents, 0x524, - 0, 5, /* M */ + 0, 2, /* M */ 8, 2, /* P */ 24, 2, /* mux */ 0); @@ -682,7 +682,7 @@ static struct ccu_mux hdmi_cec_clk = { .common = { .reg = 0xb10, - .features = CCU_FEATURE_VARIABLE_PREDIV, + .features = CCU_FEATURE_FIXED_PREDIV, .hw.init = CLK_HW_INIT_PARENTS("hdmi-cec", hdmi_cec_parents, &ccu_mux_ops, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h616.c b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c new file mode 100644 index 000000000000..225307305880 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c @@ -0,0 +1,1150 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 Arm Ltd. + * Based on the H6 CCU driver, which is: + * Copyright (c) 2017 Icenowy Zheng + */ + +#include +#include +#include +#include + +#include "ccu_common.h" +#include "ccu_reset.h" + +#include "ccu_div.h" +#include "ccu_gate.h" +#include "ccu_mp.h" +#include "ccu_mult.h" +#include "ccu_nk.h" +#include "ccu_nkm.h" +#include "ccu_nkmp.h" +#include "ccu_nm.h" + +#include "ccu-sun50i-h616.h" + +/* + * The CPU PLL is actually NP clock, with P being /1, /2 or /4. However + * P should only be used for output frequencies lower than 288 MHz. + * + * For now we can just model it as a multiplier clock, and force P to /1. + * + * The M factor is present in the register's description, but not in the + * frequency formula, and it's documented as "M is only used for backdoor + * testing", so it's not modelled and then force to 0. + */ +#define SUN50I_H616_PLL_CPUX_REG 0x000 +static struct ccu_mult pll_cpux_clk = { + .enable = BIT(31), + .lock = BIT(28), + .mult = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .common = { + .reg = 0x000, + .hw.init = CLK_HW_INIT("pll-cpux", "osc24M", + &ccu_mult_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* Some PLLs are input * N / div1 / P. Model them as NKMP with no K */ +#define SUN50I_H616_PLL_DDR0_REG 0x010 +static struct ccu_nkmp pll_ddr0_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x010, + .hw.init = CLK_HW_INIT("pll-ddr0", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_DDR1_REG 0x018 +static struct ccu_nkmp pll_ddr1_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x018, + .hw.init = CLK_HW_INIT("pll-ddr1", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_PERIPH0_REG 0x020 +static struct ccu_nkmp pll_periph0_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .fixed_post_div = 2, + .common = { + .reg = 0x020, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-periph0", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_PERIPH1_REG 0x028 +static struct ccu_nkmp pll_periph1_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .fixed_post_div = 2, + .common = { + .reg = 0x028, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-periph1", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_GPU_REG 0x030 +static struct ccu_nkmp pll_gpu_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x030, + .hw.init = CLK_HW_INIT("pll-gpu", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* + * For Video PLLs, the output divider is described as "used for testing" + * in the user manual. So it's not modelled and forced to 0. + */ +#define SUN50I_H616_PLL_VIDEO0_REG 0x040 +static struct ccu_nm pll_video0_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .fixed_post_div = 4, + .min_rate = 288000000, + .max_rate = 2400000000UL, + .common = { + .reg = 0x040, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-video0", "osc24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_VIDEO1_REG 0x048 +static struct ccu_nm pll_video1_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .fixed_post_div = 4, + .min_rate = 288000000, + .max_rate = 2400000000UL, + .common = { + .reg = 0x048, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-video1", "osc24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_VIDEO2_REG 0x050 +static struct ccu_nm pll_video2_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .fixed_post_div = 4, + .min_rate = 288000000, + .max_rate = 2400000000UL, + .common = { + .reg = 0x050, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-video2", "osc24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_VE_REG 0x058 +static struct ccu_nkmp pll_ve_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x058, + .hw.init = CLK_HW_INIT("pll-ve", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_DE_REG 0x060 +static struct ccu_nkmp pll_de_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x060, + .hw.init = CLK_HW_INIT("pll-de", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* + * TODO: Determine SDM settings for the audio PLL. The manual suggests + * PLL_FACTOR_N=16, PLL_POST_DIV_P=2, OUTPUT_DIV=2, pattern=0xe000c49b + * for 24.576 MHz, and PLL_FACTOR_N=22, PLL_POST_DIV_P=3, OUTPUT_DIV=2, + * pattern=0xe001288c for 22.5792 MHz. + * This clashes with our fixed PLL_POST_DIV_P. + */ +#define SUN50I_H616_PLL_AUDIO_REG 0x078 +static struct ccu_nm pll_audio_hs_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .common = { + .reg = 0x078, + .hw.init = CLK_HW_INIT("pll-audio-hs", "osc24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +static const char * const cpux_parents[] = { "osc24M", "osc32k", + "iosc", "pll-cpux", "pll-periph0" }; +static SUNXI_CCU_MUX(cpux_clk, "cpux", cpux_parents, + 0x500, 24, 3, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); +static SUNXI_CCU_M(axi_clk, "axi", "cpux", 0x500, 0, 2, 0); +static SUNXI_CCU_M(cpux_apb_clk, "cpux-apb", "cpux", 0x500, 8, 2, 0); + +static const char * const psi_ahb1_ahb2_parents[] = { "osc24M", "osc32k", + "iosc", "pll-periph0" }; +static SUNXI_CCU_MP_WITH_MUX(psi_ahb1_ahb2_clk, "psi-ahb1-ahb2", + psi_ahb1_ahb2_parents, + 0x510, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static const char * const ahb3_apb1_apb2_parents[] = { "osc24M", "osc32k", + "psi-ahb1-ahb2", + "pll-periph0" }; +static SUNXI_CCU_MP_WITH_MUX(ahb3_clk, "ahb3", ahb3_apb1_apb2_parents, 0x51c, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static SUNXI_CCU_MP_WITH_MUX(apb1_clk, "apb1", ahb3_apb1_apb2_parents, 0x520, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", ahb3_apb1_apb2_parents, 0x524, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static const char * const mbus_parents[] = { "osc24M", "pll-periph0-2x", + "pll-ddr0", "pll-ddr1" }; +static SUNXI_CCU_M_WITH_MUX_GATE(mbus_clk, "mbus", mbus_parents, 0x540, + 0, 3, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + CLK_IS_CRITICAL); + +static const char * const de_parents[] = { "pll-de", "pll-periph0-2x" }; +static SUNXI_CCU_M_WITH_MUX_GATE(de_clk, "de", de_parents, 0x600, + 0, 4, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_de_clk, "bus-de", "psi-ahb1-ahb2", + 0x60c, BIT(0), 0); + +static SUNXI_CCU_M_WITH_MUX_GATE(deinterlace_clk, "deinterlace", + de_parents, + 0x620, + 0, 4, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_deinterlace_clk, "bus-deinterlace", "psi-ahb1-ahb2", + 0x62c, BIT(0), 0); + +static SUNXI_CCU_M_WITH_MUX_GATE(g2d_clk, "g2d", de_parents, 0x630, + 0, 4, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_g2d_clk, "bus-g2d", "psi-ahb1-ahb2", + 0x63c, BIT(0), 0); + +static const char * const gpu0_parents[] = { "pll-gpu", "gpu1" }; +static SUNXI_CCU_M_WITH_MUX_GATE(gpu0_clk, "gpu0", gpu0_parents, 0x670, + 0, 2, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); +static SUNXI_CCU_M_WITH_GATE(gpu1_clk, "gpu1", "pll-periph0-2x", 0x674, + 0, 2, /* M */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_GATE(bus_gpu_clk, "bus-gpu", "psi-ahb1-ahb2", + 0x67c, BIT(0), 0); + +static const char * const ce_parents[] = { "osc24M", "pll-periph0-2x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(ce_clk, "ce", ce_parents, 0x680, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 1, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_GATE(bus_ce_clk, "bus-ce", "psi-ahb1-ahb2", + 0x68c, BIT(0), 0); + +static const char * const ve_parents[] = { "pll-ve" }; +static SUNXI_CCU_M_WITH_MUX_GATE(ve_clk, "ve", ve_parents, 0x690, + 0, 3, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_ve_clk, "bus-ve", "psi-ahb1-ahb2", + 0x69c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_dma_clk, "bus-dma", "psi-ahb1-ahb2", + 0x70c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_hstimer_clk, "bus-hstimer", "psi-ahb1-ahb2", + 0x73c, BIT(0), 0); + +static SUNXI_CCU_GATE(avs_clk, "avs", "osc24M", 0x740, BIT(31), 0); + +static SUNXI_CCU_GATE(bus_dbg_clk, "bus-dbg", "psi-ahb1-ahb2", + 0x78c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_psi_clk, "bus-psi", "psi-ahb1-ahb2", + 0x79c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_pwm_clk, "bus-pwm", "apb1", 0x7ac, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_iommu_clk, "bus-iommu", "apb1", 0x7bc, BIT(0), 0); + +static const char * const dram_parents[] = { "pll-ddr0", "pll-ddr1" }; +static struct ccu_div dram_clk = { + .div = _SUNXI_CCU_DIV(0, 2), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0x800, + .hw.init = CLK_HW_INIT_PARENTS("dram", + dram_parents, + &ccu_div_ops, + CLK_IS_CRITICAL), + }, +}; + +static SUNXI_CCU_GATE(mbus_dma_clk, "mbus-dma", "mbus", + 0x804, BIT(0), 0); +static SUNXI_CCU_GATE(mbus_ve_clk, "mbus-ve", "mbus", + 0x804, BIT(1), 0); +static SUNXI_CCU_GATE(mbus_ce_clk, "mbus-ce", "mbus", + 0x804, BIT(2), 0); +static SUNXI_CCU_GATE(mbus_ts_clk, "mbus-ts", "mbus", + 0x804, BIT(3), 0); +static SUNXI_CCU_GATE(mbus_nand_clk, "mbus-nand", "mbus", + 0x804, BIT(5), 0); +static SUNXI_CCU_GATE(mbus_g2d_clk, "mbus-g2d", "mbus", + 0x804, BIT(10), 0); + +static SUNXI_CCU_GATE(bus_dram_clk, "bus-dram", "psi-ahb1-ahb2", + 0x80c, BIT(0), CLK_IS_CRITICAL); + +static const char * const nand_spi_parents[] = { "osc24M", "pll-periph0", + "pll-periph1", "pll-periph0-2x", + "pll-periph1-2x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(nand0_clk, "nand0", nand_spi_parents, 0x810, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 3, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(nand1_clk, "nand1", nand_spi_parents, 0x814, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 3, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_GATE(bus_nand_clk, "bus-nand", "ahb3", 0x82c, BIT(0), 0); + +static const char * const mmc_parents[] = { "osc24M", "pll-periph0-2x", + "pll-periph1-2x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc0_clk, "mmc0", mmc_parents, 0x830, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc1_clk, "mmc1", mmc_parents, 0x834, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc2_clk, "mmc2", mmc_parents, 0x838, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + 0); + +static SUNXI_CCU_GATE(bus_mmc0_clk, "bus-mmc0", "ahb3", 0x84c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_mmc1_clk, "bus-mmc1", "ahb3", 0x84c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_mmc2_clk, "bus-mmc2", "ahb3", 0x84c, BIT(2), 0); + +static SUNXI_CCU_GATE(bus_uart0_clk, "bus-uart0", "apb2", 0x90c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_uart1_clk, "bus-uart1", "apb2", 0x90c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_uart2_clk, "bus-uart2", "apb2", 0x90c, BIT(2), 0); +static SUNXI_CCU_GATE(bus_uart3_clk, "bus-uart3", "apb2", 0x90c, BIT(3), 0); +static SUNXI_CCU_GATE(bus_uart4_clk, "bus-uart4", "apb2", 0x90c, BIT(4), 0); +static SUNXI_CCU_GATE(bus_uart5_clk, "bus-uart5", "apb2", 0x90c, BIT(5), 0); + +static SUNXI_CCU_GATE(bus_i2c0_clk, "bus-i2c0", "apb2", 0x91c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_i2c1_clk, "bus-i2c1", "apb2", 0x91c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_i2c2_clk, "bus-i2c2", "apb2", 0x91c, BIT(2), 0); +static SUNXI_CCU_GATE(bus_i2c3_clk, "bus-i2c3", "apb2", 0x91c, BIT(3), 0); +static SUNXI_CCU_GATE(bus_i2c4_clk, "bus-i2c4", "apb2", 0x91c, BIT(4), 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(spi0_clk, "spi0", nand_spi_parents, 0x940, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 3, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", nand_spi_parents, 0x944, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 3, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_GATE(bus_spi0_clk, "bus-spi0", "ahb3", 0x96c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_spi1_clk, "bus-spi1", "ahb3", 0x96c, BIT(1), 0); + +static SUNXI_CCU_GATE(emac_25m_clk, "emac-25m", "ahb3", 0x970, + BIT(31) | BIT(30), 0); + +static SUNXI_CCU_GATE(bus_emac0_clk, "bus-emac0", "ahb3", 0x97c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_emac1_clk, "bus-emac1", "ahb3", 0x97c, BIT(1), 0); + +static const char * const ts_parents[] = { "osc24M", "pll-periph0" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(ts_clk, "ts", ts_parents, 0x9b0, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 1, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_GATE(bus_ts_clk, "bus-ts", "ahb3", 0x9bc, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_ths_clk, "bus-ths", "apb1", 0x9fc, BIT(0), 0); + +static const char * const audio_parents[] = { "pll-audio-1x", "pll-audio-2x", + "pll-audio-4x", "pll-audio-hs" }; +static struct ccu_div spdif_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa20, + .hw.init = CLK_HW_INIT_PARENTS("spdif", + audio_parents, + &ccu_div_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_spdif_clk, "bus-spdif", "apb1", 0xa2c, BIT(0), 0); + +static struct ccu_div dmic_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa40, + .hw.init = CLK_HW_INIT_PARENTS("dmic", + audio_parents, + &ccu_div_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_dmic_clk, "bus-dmic", "apb1", 0xa4c, BIT(0), 0); + +static SUNXI_CCU_M_WITH_MUX_GATE(audio_codec_1x_clk, "audio-codec-1x", + audio_parents, 0xa50, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); +static SUNXI_CCU_M_WITH_MUX_GATE(audio_codec_4x_clk, "audio-codec-4x", + audio_parents, 0xa54, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_audio_codec_clk, "bus-audio-codec", "apb1", 0xa5c, + BIT(0), 0); + +static struct ccu_div audio_hub_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa60, + .hw.init = CLK_HW_INIT_PARENTS("audio-hub", + audio_parents, + &ccu_div_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_audio_hub_clk, "bus-audio-hub", "apb1", 0xa6c, BIT(0), 0); + +/* + * There are OHCI 12M clock source selection bits for the four USB 2.0 ports. + * We will force them to 0 (12M divided from 48M). + */ +#define SUN50I_H616_USB0_CLK_REG 0xa70 +#define SUN50I_H616_USB1_CLK_REG 0xa74 +#define SUN50I_H616_USB2_CLK_REG 0xa78 +#define SUN50I_H616_USB3_CLK_REG 0xa7c + +static SUNXI_CCU_GATE(usb_ohci0_clk, "usb-ohci0", "osc12M", 0xa70, BIT(31), 0); +static SUNXI_CCU_GATE(usb_phy0_clk, "usb-phy0", "osc24M", 0xa70, BIT(29), 0); + +static SUNXI_CCU_GATE(usb_ohci1_clk, "usb-ohci1", "osc12M", 0xa74, BIT(31), 0); +static SUNXI_CCU_GATE(usb_phy1_clk, "usb-phy1", "osc24M", 0xa74, BIT(29), 0); + +static SUNXI_CCU_GATE(usb_ohci2_clk, "usb-ohci2", "osc12M", 0xa78, BIT(31), 0); +static SUNXI_CCU_GATE(usb_phy2_clk, "usb-phy2", "osc24M", 0xa78, BIT(29), 0); + +static SUNXI_CCU_GATE(usb_ohci3_clk, "usb-ohci3", "osc12M", 0xa7c, BIT(31), 0); +static SUNXI_CCU_GATE(usb_phy3_clk, "usb-phy3", "osc24M", 0xa7c, BIT(29), 0); + +static SUNXI_CCU_GATE(bus_ohci0_clk, "bus-ohci0", "ahb3", 0xa8c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_ohci1_clk, "bus-ohci1", "ahb3", 0xa8c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_ohci2_clk, "bus-ohci2", "ahb3", 0xa8c, BIT(2), 0); +static SUNXI_CCU_GATE(bus_ohci3_clk, "bus-ohci3", "ahb3", 0xa8c, BIT(3), 0); +static SUNXI_CCU_GATE(bus_ehci0_clk, "bus-ehci0", "ahb3", 0xa8c, BIT(4), 0); +static SUNXI_CCU_GATE(bus_ehci1_clk, "bus-ehci1", "ahb3", 0xa8c, BIT(5), 0); +static SUNXI_CCU_GATE(bus_ehci2_clk, "bus-ehci2", "ahb3", 0xa8c, BIT(6), 0); +static SUNXI_CCU_GATE(bus_ehci3_clk, "bus-ehci3", "ahb3", 0xa8c, BIT(7), 0); +static SUNXI_CCU_GATE(bus_otg_clk, "bus-otg", "ahb3", 0xa8c, BIT(8), 0); + +static SUNXI_CCU_GATE(bus_keyadc_clk, "bus-keyadc", "apb1", 0xa9c, BIT(0), 0); + +static const char * const hdmi_parents[] = { "pll-video0", "pll-video0-4x", + "pll-video2", "pll-video2-4x" }; +static SUNXI_CCU_M_WITH_MUX_GATE(hdmi_clk, "hdmi", hdmi_parents, 0xb00, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(hdmi_slow_clk, "hdmi-slow", "osc24M", 0xb04, BIT(31), 0); + +static const char * const hdmi_cec_parents[] = { "osc32k", "pll-periph0-2x" }; +static const struct ccu_mux_fixed_prediv hdmi_cec_predivs[] = { + { .index = 1, .div = 36621 }, +}; + +#define SUN50I_H616_HDMI_CEC_CLK_REG 0xb10 +static struct ccu_mux hdmi_cec_clk = { + .enable = BIT(31) | BIT(30), + + .mux = { + .shift = 24, + .width = 2, + + .fixed_predivs = hdmi_cec_predivs, + .n_predivs = ARRAY_SIZE(hdmi_cec_predivs), + }, + + .common = { + .reg = 0xb10, + .features = CCU_FEATURE_FIXED_PREDIV, + .hw.init = CLK_HW_INIT_PARENTS("hdmi-cec", + hdmi_cec_parents, + &ccu_mux_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_hdmi_clk, "bus-hdmi", "ahb3", 0xb1c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_tcon_top_clk, "bus-tcon-top", "ahb3", + 0xb5c, BIT(0), 0); + +static const char * const tcon_tv_parents[] = { "pll-video0", + "pll-video0-4x", + "pll-video1", + "pll-video1-4x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(tcon_tv0_clk, "tcon-tv0", + tcon_tv_parents, 0xb80, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); +static SUNXI_CCU_MP_WITH_MUX_GATE(tcon_tv1_clk, "tcon-tv1", + tcon_tv_parents, 0xb84, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_tcon_tv0_clk, "bus-tcon-tv0", "ahb3", + 0xb9c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_tcon_tv1_clk, "bus-tcon-tv1", "ahb3", + 0xb9c, BIT(1), 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(tve0_clk, "tve0", + tcon_tv_parents, 0xbb0, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_tve_top_clk, "bus-tve-top", "ahb3", + 0xbbc, BIT(0), 0); +static SUNXI_CCU_GATE(bus_tve0_clk, "bus-tve0", "ahb3", + 0xbbc, BIT(1), 0); + +static const char * const hdcp_parents[] = { "pll-periph0", "pll-periph1" }; +static SUNXI_CCU_M_WITH_MUX_GATE(hdcp_clk, "hdcp", hdcp_parents, 0xc40, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_hdcp_clk, "bus-hdcp", "ahb3", 0xc4c, BIT(0), 0); + +/* Fixed factor clocks */ +static CLK_FIXED_FACTOR_FW_NAME(osc12M_clk, "osc12M", "hosc", 2, 1, 0); + +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_hs_clk.common.hw +}; + +/* + * The divider of pll-audio is fixed to 24 for now, so 24576000 and 22579200 + * rates can be set exactly in conjunction with sigma-delta modulation. + */ +static CLK_FIXED_FACTOR_HWS(pll_audio_1x_clk, "pll-audio-1x", + clk_parent_pll_audio, + 96, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 48, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 24, 1, CLK_SET_RATE_PARENT); + +static const struct clk_hw *pll_periph0_parents[] = { + &pll_periph0_clk.common.hw +}; + +static CLK_FIXED_FACTOR_HWS(pll_periph0_2x_clk, "pll-periph0-2x", + pll_periph0_parents, + 1, 2, 0); + +static const struct clk_hw *pll_periph1_parents[] = { + &pll_periph1_clk.common.hw +}; + +static CLK_FIXED_FACTOR_HWS(pll_periph1_2x_clk, "pll-periph1-2x", + pll_periph1_parents, + 1, 2, 0); + +static CLK_FIXED_FACTOR_HW(pll_video0_4x_clk, "pll-video0-4x", + &pll_video0_clk.common.hw, + 1, 4, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video1_4x_clk, "pll-video1-4x", + &pll_video1_clk.common.hw, + 1, 4, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video2_4x_clk, "pll-video2-4x", + &pll_video2_clk.common.hw, + 1, 4, CLK_SET_RATE_PARENT); + +static struct ccu_common *sun50i_h616_ccu_clks[] = { + &pll_cpux_clk.common, + &pll_ddr0_clk.common, + &pll_ddr1_clk.common, + &pll_periph0_clk.common, + &pll_periph1_clk.common, + &pll_gpu_clk.common, + &pll_video0_clk.common, + &pll_video1_clk.common, + &pll_video2_clk.common, + &pll_ve_clk.common, + &pll_de_clk.common, + &pll_audio_hs_clk.common, + &cpux_clk.common, + &axi_clk.common, + &cpux_apb_clk.common, + &psi_ahb1_ahb2_clk.common, + &ahb3_clk.common, + &apb1_clk.common, + &apb2_clk.common, + &mbus_clk.common, + &de_clk.common, + &bus_de_clk.common, + &deinterlace_clk.common, + &bus_deinterlace_clk.common, + &g2d_clk.common, + &bus_g2d_clk.common, + &gpu0_clk.common, + &bus_gpu_clk.common, + &gpu1_clk.common, + &ce_clk.common, + &bus_ce_clk.common, + &ve_clk.common, + &bus_ve_clk.common, + &bus_dma_clk.common, + &bus_hstimer_clk.common, + &avs_clk.common, + &bus_dbg_clk.common, + &bus_psi_clk.common, + &bus_pwm_clk.common, + &bus_iommu_clk.common, + &dram_clk.common, + &mbus_dma_clk.common, + &mbus_ve_clk.common, + &mbus_ce_clk.common, + &mbus_ts_clk.common, + &mbus_nand_clk.common, + &mbus_g2d_clk.common, + &bus_dram_clk.common, + &nand0_clk.common, + &nand1_clk.common, + &bus_nand_clk.common, + &mmc0_clk.common, + &mmc1_clk.common, + &mmc2_clk.common, + &bus_mmc0_clk.common, + &bus_mmc1_clk.common, + &bus_mmc2_clk.common, + &bus_uart0_clk.common, + &bus_uart1_clk.common, + &bus_uart2_clk.common, + &bus_uart3_clk.common, + &bus_uart4_clk.common, + &bus_uart5_clk.common, + &bus_i2c0_clk.common, + &bus_i2c1_clk.common, + &bus_i2c2_clk.common, + &bus_i2c3_clk.common, + &bus_i2c4_clk.common, + &spi0_clk.common, + &spi1_clk.common, + &bus_spi0_clk.common, + &bus_spi1_clk.common, + &emac_25m_clk.common, + &bus_emac0_clk.common, + &bus_emac1_clk.common, + &ts_clk.common, + &bus_ts_clk.common, + &bus_ths_clk.common, + &spdif_clk.common, + &bus_spdif_clk.common, + &dmic_clk.common, + &bus_dmic_clk.common, + &audio_codec_1x_clk.common, + &audio_codec_4x_clk.common, + &bus_audio_codec_clk.common, + &audio_hub_clk.common, + &bus_audio_hub_clk.common, + &usb_ohci0_clk.common, + &usb_phy0_clk.common, + &usb_ohci1_clk.common, + &usb_phy1_clk.common, + &usb_ohci2_clk.common, + &usb_phy2_clk.common, + &usb_ohci3_clk.common, + &usb_phy3_clk.common, + &bus_ohci0_clk.common, + &bus_ohci1_clk.common, + &bus_ohci2_clk.common, + &bus_ohci3_clk.common, + &bus_ehci0_clk.common, + &bus_ehci1_clk.common, + &bus_ehci2_clk.common, + &bus_ehci3_clk.common, + &bus_otg_clk.common, + &bus_keyadc_clk.common, + &hdmi_clk.common, + &hdmi_slow_clk.common, + &hdmi_cec_clk.common, + &bus_hdmi_clk.common, + &bus_tcon_top_clk.common, + &tcon_tv0_clk.common, + &tcon_tv1_clk.common, + &bus_tcon_tv0_clk.common, + &bus_tcon_tv1_clk.common, + &tve0_clk.common, + &bus_tve_top_clk.common, + &bus_tve0_clk.common, + &hdcp_clk.common, + &bus_hdcp_clk.common, +}; + +static struct clk_hw_onecell_data sun50i_h616_hw_clks = { + .hws = { + [CLK_OSC12M] = &osc12M_clk.hw, + [CLK_PLL_CPUX] = &pll_cpux_clk.common.hw, + [CLK_PLL_DDR0] = &pll_ddr0_clk.common.hw, + [CLK_PLL_DDR1] = &pll_ddr1_clk.common.hw, + [CLK_PLL_PERIPH0] = &pll_periph0_clk.common.hw, + [CLK_PLL_PERIPH0_2X] = &pll_periph0_2x_clk.hw, + [CLK_PLL_PERIPH1] = &pll_periph1_clk.common.hw, + [CLK_PLL_PERIPH1_2X] = &pll_periph1_2x_clk.hw, + [CLK_PLL_GPU] = &pll_gpu_clk.common.hw, + [CLK_PLL_VIDEO0] = &pll_video0_clk.common.hw, + [CLK_PLL_VIDEO0_4X] = &pll_video0_4x_clk.hw, + [CLK_PLL_VIDEO1] = &pll_video1_clk.common.hw, + [CLK_PLL_VIDEO1_4X] = &pll_video1_4x_clk.hw, + [CLK_PLL_VIDEO2] = &pll_video2_clk.common.hw, + [CLK_PLL_VIDEO2_4X] = &pll_video2_4x_clk.hw, + [CLK_PLL_VE] = &pll_ve_clk.common.hw, + [CLK_PLL_DE] = &pll_de_clk.common.hw, + [CLK_PLL_AUDIO_HS] = &pll_audio_hs_clk.common.hw, + [CLK_PLL_AUDIO_1X] = &pll_audio_1x_clk.hw, + [CLK_PLL_AUDIO_2X] = &pll_audio_2x_clk.hw, + [CLK_PLL_AUDIO_4X] = &pll_audio_4x_clk.hw, + [CLK_CPUX] = &cpux_clk.common.hw, + [CLK_AXI] = &axi_clk.common.hw, + [CLK_CPUX_APB] = &cpux_apb_clk.common.hw, + [CLK_PSI_AHB1_AHB2] = &psi_ahb1_ahb2_clk.common.hw, + [CLK_AHB3] = &ahb3_clk.common.hw, + [CLK_APB1] = &apb1_clk.common.hw, + [CLK_APB2] = &apb2_clk.common.hw, + [CLK_MBUS] = &mbus_clk.common.hw, + [CLK_DE] = &de_clk.common.hw, + [CLK_BUS_DE] = &bus_de_clk.common.hw, + [CLK_DEINTERLACE] = &deinterlace_clk.common.hw, + [CLK_BUS_DEINTERLACE] = &bus_deinterlace_clk.common.hw, + [CLK_G2D] = &g2d_clk.common.hw, + [CLK_BUS_G2D] = &bus_g2d_clk.common.hw, + [CLK_GPU0] = &gpu0_clk.common.hw, + [CLK_BUS_GPU] = &bus_gpu_clk.common.hw, + [CLK_GPU1] = &gpu1_clk.common.hw, + [CLK_CE] = &ce_clk.common.hw, + [CLK_BUS_CE] = &bus_ce_clk.common.hw, + [CLK_VE] = &ve_clk.common.hw, + [CLK_BUS_VE] = &bus_ve_clk.common.hw, + [CLK_BUS_DMA] = &bus_dma_clk.common.hw, + [CLK_BUS_HSTIMER] = &bus_hstimer_clk.common.hw, + [CLK_AVS] = &avs_clk.common.hw, + [CLK_BUS_DBG] = &bus_dbg_clk.common.hw, + [CLK_BUS_PSI] = &bus_psi_clk.common.hw, + [CLK_BUS_PWM] = &bus_pwm_clk.common.hw, + [CLK_BUS_IOMMU] = &bus_iommu_clk.common.hw, + [CLK_DRAM] = &dram_clk.common.hw, + [CLK_MBUS_DMA] = &mbus_dma_clk.common.hw, + [CLK_MBUS_VE] = &mbus_ve_clk.common.hw, + [CLK_MBUS_CE] = &mbus_ce_clk.common.hw, + [CLK_MBUS_TS] = &mbus_ts_clk.common.hw, + [CLK_MBUS_NAND] = &mbus_nand_clk.common.hw, + [CLK_MBUS_G2D] = &mbus_g2d_clk.common.hw, + [CLK_BUS_DRAM] = &bus_dram_clk.common.hw, + [CLK_NAND0] = &nand0_clk.common.hw, + [CLK_NAND1] = &nand1_clk.common.hw, + [CLK_BUS_NAND] = &bus_nand_clk.common.hw, + [CLK_MMC0] = &mmc0_clk.common.hw, + [CLK_MMC1] = &mmc1_clk.common.hw, + [CLK_MMC2] = &mmc2_clk.common.hw, + [CLK_BUS_MMC0] = &bus_mmc0_clk.common.hw, + [CLK_BUS_MMC1] = &bus_mmc1_clk.common.hw, + [CLK_BUS_MMC2] = &bus_mmc2_clk.common.hw, + [CLK_BUS_UART0] = &bus_uart0_clk.common.hw, + [CLK_BUS_UART1] = &bus_uart1_clk.common.hw, + [CLK_BUS_UART2] = &bus_uart2_clk.common.hw, + [CLK_BUS_UART3] = &bus_uart3_clk.common.hw, + [CLK_BUS_UART4] = &bus_uart4_clk.common.hw, + [CLK_BUS_UART5] = &bus_uart5_clk.common.hw, + [CLK_BUS_I2C0] = &bus_i2c0_clk.common.hw, + [CLK_BUS_I2C1] = &bus_i2c1_clk.common.hw, + [CLK_BUS_I2C2] = &bus_i2c2_clk.common.hw, + [CLK_BUS_I2C3] = &bus_i2c3_clk.common.hw, + [CLK_BUS_I2C4] = &bus_i2c4_clk.common.hw, + [CLK_SPI0] = &spi0_clk.common.hw, + [CLK_SPI1] = &spi1_clk.common.hw, + [CLK_BUS_SPI0] = &bus_spi0_clk.common.hw, + [CLK_BUS_SPI1] = &bus_spi1_clk.common.hw, + [CLK_EMAC_25M] = &emac_25m_clk.common.hw, + [CLK_BUS_EMAC0] = &bus_emac0_clk.common.hw, + [CLK_BUS_EMAC1] = &bus_emac1_clk.common.hw, + [CLK_TS] = &ts_clk.common.hw, + [CLK_BUS_TS] = &bus_ts_clk.common.hw, + [CLK_BUS_THS] = &bus_ths_clk.common.hw, + [CLK_SPDIF] = &spdif_clk.common.hw, + [CLK_BUS_SPDIF] = &bus_spdif_clk.common.hw, + [CLK_DMIC] = &dmic_clk.common.hw, + [CLK_BUS_DMIC] = &bus_dmic_clk.common.hw, + [CLK_AUDIO_CODEC_1X] = &audio_codec_1x_clk.common.hw, + [CLK_AUDIO_CODEC_4X] = &audio_codec_4x_clk.common.hw, + [CLK_BUS_AUDIO_CODEC] = &bus_audio_codec_clk.common.hw, + [CLK_AUDIO_HUB] = &audio_hub_clk.common.hw, + [CLK_BUS_AUDIO_HUB] = &bus_audio_hub_clk.common.hw, + [CLK_USB_OHCI0] = &usb_ohci0_clk.common.hw, + [CLK_USB_PHY0] = &usb_phy0_clk.common.hw, + [CLK_USB_OHCI1] = &usb_ohci1_clk.common.hw, + [CLK_USB_PHY1] = &usb_phy1_clk.common.hw, + [CLK_USB_OHCI2] = &usb_ohci2_clk.common.hw, + [CLK_USB_PHY2] = &usb_phy2_clk.common.hw, + [CLK_USB_OHCI3] = &usb_ohci3_clk.common.hw, + [CLK_USB_PHY3] = &usb_phy3_clk.common.hw, + [CLK_BUS_OHCI0] = &bus_ohci0_clk.common.hw, + [CLK_BUS_OHCI1] = &bus_ohci1_clk.common.hw, + [CLK_BUS_OHCI2] = &bus_ohci2_clk.common.hw, + [CLK_BUS_OHCI3] = &bus_ohci3_clk.common.hw, + [CLK_BUS_EHCI0] = &bus_ehci0_clk.common.hw, + [CLK_BUS_EHCI1] = &bus_ehci1_clk.common.hw, + [CLK_BUS_EHCI2] = &bus_ehci2_clk.common.hw, + [CLK_BUS_EHCI3] = &bus_ehci3_clk.common.hw, + [CLK_BUS_OTG] = &bus_otg_clk.common.hw, + [CLK_BUS_KEYADC] = &bus_keyadc_clk.common.hw, + [CLK_HDMI] = &hdmi_clk.common.hw, + [CLK_HDMI_SLOW] = &hdmi_slow_clk.common.hw, + [CLK_HDMI_CEC] = &hdmi_cec_clk.common.hw, + [CLK_BUS_HDMI] = &bus_hdmi_clk.common.hw, + [CLK_BUS_TCON_TOP] = &bus_tcon_top_clk.common.hw, + [CLK_TCON_TV0] = &tcon_tv0_clk.common.hw, + [CLK_TCON_TV1] = &tcon_tv1_clk.common.hw, + [CLK_BUS_TCON_TV0] = &bus_tcon_tv0_clk.common.hw, + [CLK_BUS_TCON_TV1] = &bus_tcon_tv1_clk.common.hw, + [CLK_TVE0] = &tve0_clk.common.hw, + [CLK_BUS_TVE_TOP] = &bus_tve_top_clk.common.hw, + [CLK_BUS_TVE0] = &bus_tve0_clk.common.hw, + [CLK_HDCP] = &hdcp_clk.common.hw, + [CLK_BUS_HDCP] = &bus_hdcp_clk.common.hw, + }, + .num = CLK_NUMBER, +}; + +static struct ccu_reset_map sun50i_h616_ccu_resets[] = { + [RST_MBUS] = { 0x540, BIT(30) }, + + [RST_BUS_DE] = { 0x60c, BIT(16) }, + [RST_BUS_DEINTERLACE] = { 0x62c, BIT(16) }, + [RST_BUS_GPU] = { 0x67c, BIT(16) }, + [RST_BUS_CE] = { 0x68c, BIT(16) }, + [RST_BUS_VE] = { 0x69c, BIT(16) }, + [RST_BUS_DMA] = { 0x70c, BIT(16) }, + [RST_BUS_HSTIMER] = { 0x73c, BIT(16) }, + [RST_BUS_DBG] = { 0x78c, BIT(16) }, + [RST_BUS_PSI] = { 0x79c, BIT(16) }, + [RST_BUS_PWM] = { 0x7ac, BIT(16) }, + [RST_BUS_IOMMU] = { 0x7bc, BIT(16) }, + [RST_BUS_DRAM] = { 0x80c, BIT(16) }, + [RST_BUS_NAND] = { 0x82c, BIT(16) }, + [RST_BUS_MMC0] = { 0x84c, BIT(16) }, + [RST_BUS_MMC1] = { 0x84c, BIT(17) }, + [RST_BUS_MMC2] = { 0x84c, BIT(18) }, + [RST_BUS_UART0] = { 0x90c, BIT(16) }, + [RST_BUS_UART1] = { 0x90c, BIT(17) }, + [RST_BUS_UART2] = { 0x90c, BIT(18) }, + [RST_BUS_UART3] = { 0x90c, BIT(19) }, + [RST_BUS_UART4] = { 0x90c, BIT(20) }, + [RST_BUS_UART5] = { 0x90c, BIT(21) }, + [RST_BUS_I2C0] = { 0x91c, BIT(16) }, + [RST_BUS_I2C1] = { 0x91c, BIT(17) }, + [RST_BUS_I2C2] = { 0x91c, BIT(18) }, + [RST_BUS_I2C3] = { 0x91c, BIT(19) }, + [RST_BUS_I2C4] = { 0x91c, BIT(20) }, + [RST_BUS_SPI0] = { 0x96c, BIT(16) }, + [RST_BUS_SPI1] = { 0x96c, BIT(17) }, + [RST_BUS_EMAC0] = { 0x97c, BIT(16) }, + [RST_BUS_EMAC1] = { 0x97c, BIT(17) }, + [RST_BUS_TS] = { 0x9bc, BIT(16) }, + [RST_BUS_THS] = { 0x9fc, BIT(16) }, + [RST_BUS_SPDIF] = { 0xa2c, BIT(16) }, + [RST_BUS_DMIC] = { 0xa4c, BIT(16) }, + [RST_BUS_AUDIO_CODEC] = { 0xa5c, BIT(16) }, + [RST_BUS_AUDIO_HUB] = { 0xa6c, BIT(16) }, + + [RST_USB_PHY0] = { 0xa70, BIT(30) }, + [RST_USB_PHY1] = { 0xa74, BIT(30) }, + [RST_USB_PHY2] = { 0xa78, BIT(30) }, + [RST_USB_PHY3] = { 0xa7c, BIT(30) }, + [RST_BUS_OHCI0] = { 0xa8c, BIT(16) }, + [RST_BUS_OHCI1] = { 0xa8c, BIT(17) }, + [RST_BUS_OHCI2] = { 0xa8c, BIT(18) }, + [RST_BUS_OHCI3] = { 0xa8c, BIT(19) }, + [RST_BUS_EHCI0] = { 0xa8c, BIT(20) }, + [RST_BUS_EHCI1] = { 0xa8c, BIT(21) }, + [RST_BUS_EHCI2] = { 0xa8c, BIT(22) }, + [RST_BUS_EHCI3] = { 0xa8c, BIT(23) }, + [RST_BUS_OTG] = { 0xa8c, BIT(24) }, + [RST_BUS_KEYADC] = { 0xa9c, BIT(16) }, + + [RST_BUS_HDMI] = { 0xb1c, BIT(16) }, + [RST_BUS_HDMI_SUB] = { 0xb1c, BIT(17) }, + [RST_BUS_TCON_TOP] = { 0xb5c, BIT(16) }, + [RST_BUS_TCON_TV0] = { 0xb9c, BIT(16) }, + [RST_BUS_TCON_TV1] = { 0xb9c, BIT(17) }, + [RST_BUS_TVE_TOP] = { 0xbbc, BIT(16) }, + [RST_BUS_TVE0] = { 0xbbc, BIT(17) }, + [RST_BUS_HDCP] = { 0xc4c, BIT(16) }, +}; + +static const struct sunxi_ccu_desc sun50i_h616_ccu_desc = { + .ccu_clks = sun50i_h616_ccu_clks, + .num_ccu_clks = ARRAY_SIZE(sun50i_h616_ccu_clks), + + .hw_clks = &sun50i_h616_hw_clks, + + .resets = sun50i_h616_ccu_resets, + .num_resets = ARRAY_SIZE(sun50i_h616_ccu_resets), +}; + +static const u32 pll_regs[] = { + SUN50I_H616_PLL_CPUX_REG, + SUN50I_H616_PLL_DDR0_REG, + SUN50I_H616_PLL_DDR1_REG, + SUN50I_H616_PLL_PERIPH0_REG, + SUN50I_H616_PLL_PERIPH1_REG, + SUN50I_H616_PLL_GPU_REG, + SUN50I_H616_PLL_VIDEO0_REG, + SUN50I_H616_PLL_VIDEO1_REG, + SUN50I_H616_PLL_VIDEO2_REG, + SUN50I_H616_PLL_VE_REG, + SUN50I_H616_PLL_DE_REG, + SUN50I_H616_PLL_AUDIO_REG, +}; + +static const u32 pll_video_regs[] = { + SUN50I_H616_PLL_VIDEO0_REG, + SUN50I_H616_PLL_VIDEO1_REG, + SUN50I_H616_PLL_VIDEO2_REG, +}; + +static const u32 usb2_clk_regs[] = { + SUN50I_H616_USB0_CLK_REG, + SUN50I_H616_USB1_CLK_REG, + SUN50I_H616_USB2_CLK_REG, + SUN50I_H616_USB3_CLK_REG, +}; + +static void __init sun50i_h616_ccu_setup(struct device_node *node) +{ + void __iomem *reg; + u32 val; + int i; + + reg = of_io_request_and_map(node, 0, of_node_full_name(node)); + if (IS_ERR(reg)) { + pr_err("%pOF: Could not map clock registers\n", node); + return; + } + + /* Enable the lock bits and the output enable bits on all PLLs */ + for (i = 0; i < ARRAY_SIZE(pll_regs); i++) { + val = readl(reg + pll_regs[i]); + val |= BIT(29) | BIT(27); + writel(val, reg + pll_regs[i]); + } + + /* + * Force the output divider of video PLLs to 0. + * + * See the comment before pll-video0 definition for the reason. + */ + for (i = 0; i < ARRAY_SIZE(pll_video_regs); i++) { + val = readl(reg + pll_video_regs[i]); + val &= ~BIT(0); + writel(val, reg + pll_video_regs[i]); + } + + /* + * Force OHCI 12M clock sources to 00 (12MHz divided from 48MHz) + * + * This clock mux is still mysterious, and the code just enforces + * it to have a valid clock parent. + */ + for (i = 0; i < ARRAY_SIZE(usb2_clk_regs); i++) { + val = readl(reg + usb2_clk_regs[i]); + val &= ~GENMASK(25, 24); + writel(val, reg + usb2_clk_regs[i]); + } + + /* + * Force the post-divider of pll-audio to 12 and the output divider + * of it to 2, so 24576000 and 22579200 rates can be set exactly. + */ + val = readl(reg + SUN50I_H616_PLL_AUDIO_REG); + val &= ~(GENMASK(21, 16) | BIT(0)); + writel(val | (11 << 16) | BIT(0), reg + SUN50I_H616_PLL_AUDIO_REG); + + /* + * First clock parent (osc32K) is unusable for CEC. But since there + * is no good way to force parent switch (both run with same frequency), + * just set second clock parent here. + */ + val = readl(reg + SUN50I_H616_HDMI_CEC_CLK_REG); + val |= BIT(24); + writel(val, reg + SUN50I_H616_HDMI_CEC_CLK_REG); + + i = sunxi_ccu_probe(node, reg, &sun50i_h616_ccu_desc); + if (i) + pr_err("%pOF: probing clocks fails: %d\n", node, i); +} + +CLK_OF_DECLARE(sun50i_h616_ccu, "allwinner,sun50i-h616-ccu", + sun50i_h616_ccu_setup); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h616.h b/drivers/clk/sunxi-ng/ccu-sun50i-h616.h new file mode 100644 index 000000000000..dd671b413f22 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h616.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2020 Arm Ltd. + */ + +#ifndef _CCU_SUN50I_H616_H_ +#define _CCU_SUN50I_H616_H_ + +#include +#include + +#define CLK_OSC12M 0 +#define CLK_PLL_CPUX 1 +#define CLK_PLL_DDR0 2 +#define CLK_PLL_DDR1 3 + +/* PLL_PERIPH0 exported for PRCM */ + +#define CLK_PLL_PERIPH0_2X 5 +#define CLK_PLL_PERIPH1 6 +#define CLK_PLL_PERIPH1_2X 7 +#define CLK_PLL_GPU 8 +#define CLK_PLL_VIDEO0 9 +#define CLK_PLL_VIDEO0_4X 10 +#define CLK_PLL_VIDEO1 11 +#define CLK_PLL_VIDEO1_4X 12 +#define CLK_PLL_VIDEO2 13 +#define CLK_PLL_VIDEO2_4X 14 +#define CLK_PLL_VE 15 +#define CLK_PLL_DE 16 +#define CLK_PLL_AUDIO_HS 17 +#define CLK_PLL_AUDIO_1X 18 +#define CLK_PLL_AUDIO_2X 19 +#define CLK_PLL_AUDIO_4X 20 + +/* CPUX clock exported for DVFS */ + +#define CLK_AXI 22 +#define CLK_CPUX_APB 23 +#define CLK_PSI_AHB1_AHB2 24 +#define CLK_AHB3 25 + +/* APB1 clock exported for PIO */ + +#define CLK_APB2 27 +#define CLK_MBUS 28 + +/* All module clocks and bus gates are exported except DRAM */ + +#define CLK_DRAM 49 + +#define CLK_BUS_DRAM 56 + +#define CLK_NUMBER (CLK_BUS_HDCP + 1) + +#endif /* _CCU_SUN50I_H616_H_ */ diff --git a/drivers/clk/sunxi/clk-a10-ve.c b/drivers/clk/sunxi/clk-a10-ve.c index cb5daa4b37db..65810937a13a 100644 --- a/drivers/clk/sunxi/clk-a10-ve.c +++ b/drivers/clk/sunxi/clk-a10-ve.c @@ -20,7 +20,7 @@ static DEFINE_SPINLOCK(ve_lock); #define SUN4I_VE_DIVIDER_WIDTH 3 #define SUN4I_VE_RESET 0 -/** +/* * sunxi_ve_reset... - reset bit in ve clk registers handling */ diff --git a/drivers/clk/sunxi/clk-mod0.c b/drivers/clk/sunxi/clk-mod0.c index 0cca91e075a5..f9d715ec9908 100644 --- a/drivers/clk/sunxi/clk-mod0.c +++ b/drivers/clk/sunxi/clk-mod0.c @@ -14,7 +14,7 @@ #include "clk-factors.h" -/** +/* * sun4i_a10_get_mod0_factors() - calculates m, n factors for MOD0-style clocks * MOD0 rate is calculated as follows * rate = (parent_rate >> p) / (m + 1); diff --git a/drivers/clk/sunxi/clk-sunxi.c b/drivers/clk/sunxi/clk-sunxi.c index e1aa1fbac48a..5fe7049ea693 100644 --- a/drivers/clk/sunxi/clk-sunxi.c +++ b/drivers/clk/sunxi/clk-sunxi.c @@ -23,7 +23,7 @@ static DEFINE_SPINLOCK(clk_lock); /* Maximum number of parents our clocks have */ #define SUNXI_MAX_PARENTS 5 -/** +/* * sun4i_get_pll1_factors() - calculates n, k, m, p factors for PLL1 * PLL1 rate is calculated as follows * rate = (parent_rate * n * (k + 1) >> p) / (m + 1); @@ -71,7 +71,7 @@ static void sun4i_get_pll1_factors(struct factors_request *req) req->n = div / 4; } -/** +/* * sun6i_a31_get_pll1_factors() - calculates n, k and m factors for PLL1 * PLL1 rate is calculated as follows * rate = parent_rate * (n + 1) * (k + 1) / (m + 1); @@ -147,7 +147,7 @@ static void sun6i_a31_get_pll1_factors(struct factors_request *req) } } -/** +/* * sun8i_a23_get_pll1_factors() - calculates n, k, m, p factors for PLL1 * PLL1 rate is calculated as follows * rate = (parent_rate * (n + 1) * (k + 1) >> p) / (m + 1); @@ -191,7 +191,7 @@ static void sun8i_a23_get_pll1_factors(struct factors_request *req) req->n = div / 4 - 1; } -/** +/* * sun4i_get_pll5_factors() - calculates n, k factors for PLL5 * PLL5 rate is calculated as follows * rate = parent_rate * n * (k + 1) @@ -218,7 +218,7 @@ static void sun4i_get_pll5_factors(struct factors_request *req) req->n = DIV_ROUND_UP(div, (req->k + 1)); } -/** +/* * sun6i_a31_get_pll6_factors() - calculates n, k factors for A31 PLL6x2 * PLL6x2 rate is calculated as follows * rate = parent_rate * (n + 1) * (k + 1) @@ -240,7 +240,7 @@ static void sun6i_a31_get_pll6_factors(struct factors_request *req) req->n = DIV_ROUND_UP(div, (req->k + 1)) - 1; } -/** +/* * sun5i_a13_get_ahb_factors() - calculates m, p factors for AHB * AHB rate is calculated as follows * rate = parent_rate >> p @@ -276,7 +276,7 @@ static void sun5i_a13_get_ahb_factors(struct factors_request *req) #define SUN6I_AHB1_PARENT_PLL6 3 -/** +/* * sun6i_a31_get_ahb_factors() - calculates m, p factors for AHB * AHB rate is calculated as follows * rate = parent_rate >> p @@ -320,7 +320,7 @@ static void sun6i_get_ahb1_factors(struct factors_request *req) req->m = calcm - 1; } -/** +/* * sun6i_ahb1_recalc() - calculates AHB clock rate from m, p factors and * parent index */ @@ -336,7 +336,7 @@ static void sun6i_ahb1_recalc(struct factors_request *req) req->rate >>= req->p; } -/** +/* * sun4i_get_apb1_factors() - calculates m, p factors for APB1 * APB1 rate is calculated as follows * rate = (parent_rate >> p) / (m + 1); @@ -375,7 +375,7 @@ static void sun4i_get_apb1_factors(struct factors_request *req) -/** +/* * sun7i_a20_get_out_factors() - calculates m, p factors for CLK_OUT_A/B * CLK_OUT rate is calculated as follows * rate = (parent_rate >> p) / (m + 1); @@ -408,7 +408,7 @@ static void sun7i_a20_get_out_factors(struct factors_request *req) req->p = calcp; } -/** +/* * sunxi_factors_clk_setup() - Setup function for factor clocks */ @@ -625,7 +625,7 @@ CLK_OF_DECLARE(sun7i_out, "allwinner,sun7i-a20-out-clk", sun7i_out_clk_setup); -/** +/* * sunxi_mux_clk_setup() - Setup function for muxes */ @@ -717,7 +717,7 @@ CLK_OF_DECLARE(sun8i_ahb2, "allwinner,sun8i-h3-ahb2-clk", sun8i_ahb2_clk_setup); -/** +/* * sunxi_divider_clk_setup() - Setup function for simple divider clocks */ @@ -853,7 +853,7 @@ CLK_OF_DECLARE(sun8i_axi, "allwinner,sun8i-a23-axi-clk", -/** +/* * sunxi_gates_clk_setup() - Setup function for leaf gates on clocks */ @@ -863,7 +863,7 @@ struct gates_data { DECLARE_BITMAP(mask, SUNXI_GATES_MAX_SIZE); }; -/** +/* * sunxi_divs_clk_setup() helper data */ @@ -929,7 +929,7 @@ static const struct divs_data sun6i_a31_pll6_divs_data __initconst = { } }; -/** +/* * sunxi_divs_clk_setup() - Setup function for leaf divisors on clocks * * These clocks look something like this diff --git a/drivers/clk/tegra/clk-tegra30.c b/drivers/clk/tegra/clk-tegra30.c index 9cf249c344d9..16dbf83d2f62 100644 --- a/drivers/clk/tegra/clk-tegra30.c +++ b/drivers/clk/tegra/clk-tegra30.c @@ -1104,12 +1104,9 @@ static void tegra30_cpu_out_of_reset(u32 cpu) static void tegra30_enable_cpu_clock(u32 cpu) { - unsigned int reg; - writel(CPU_CLOCK(cpu), clk_base + TEGRA30_CLK_RST_CONTROLLER_CLK_CPU_CMPLX_CLR); - reg = readl(clk_base + - TEGRA30_CLK_RST_CONTROLLER_CLK_CPU_CMPLX_CLR); + readl(clk_base + TEGRA30_CLK_RST_CONTROLLER_CLK_CPU_CMPLX_CLR); } static void tegra30_disable_cpu_clock(u32 cpu) diff --git a/drivers/clk/tegra/cvb.c b/drivers/clk/tegra/cvb.c index 21115c4e5d3a..a7fdc7622913 100644 --- a/drivers/clk/tegra/cvb.c +++ b/drivers/clk/tegra/cvb.c @@ -86,6 +86,7 @@ static int build_opp_table(struct device *dev, const struct cvb_table *table, * @dev: the struct device * for which the OPP table is built * @tables: array of CVB tables * @count: size of the previously mentioned array + * @align: parameters of the regulator step and offset * @process_id: process id of the HW module * @speedo_id: speedo id of the HW module * @speedo_value: speedo value of the HW module diff --git a/drivers/clk/ti/clkt_dpll.c b/drivers/clk/ti/clkt_dpll.c index 87ece6cd4226..dfaa4d1f0b64 100644 --- a/drivers/clk/ti/clkt_dpll.c +++ b/drivers/clk/ti/clkt_dpll.c @@ -269,8 +269,9 @@ unsigned long omap2_get_dpll_rate(struct clk_hw_omap *clk) /** * omap2_dpll_round_rate - round a target rate for an OMAP DPLL - * @clk: struct clk * for a DPLL + * @hw: struct clk_hw containing the struct clk * for a DPLL * @target_rate: desired DPLL clock rate + * @parent_rate: parent's DPLL clock rate * * Given a DPLL and a desired target rate, round the target rate to a * possible, programmable rate for this DPLL. Attempts to select the diff --git a/drivers/clk/ti/clockdomain.c b/drivers/clk/ti/clockdomain.c index 700b7f44f671..74831b2752b3 100644 --- a/drivers/clk/ti/clockdomain.c +++ b/drivers/clk/ti/clockdomain.c @@ -97,7 +97,7 @@ void omap2_clkops_disable_clkdm(struct clk_hw *hw) /** * omap2_init_clk_clkdm - look up a clockdomain name, store pointer in clk - * @clk: OMAP clock struct ptr to use + * @hw: Pointer to clk_hw_omap used to obtain OMAP clock struct ptr to use * * Convert a clockdomain name stored in a struct clk 'clk' into a * clockdomain pointer, and save it into the struct clk. Intended to be diff --git a/drivers/clk/ti/dpll.c b/drivers/clk/ti/dpll.c index 247510e306e2..d6f1ac5b53e1 100644 --- a/drivers/clk/ti/dpll.c +++ b/drivers/clk/ti/dpll.c @@ -151,7 +151,7 @@ static const struct clk_ops dpll_x2_ck_ops = { /** * _register_dpll - low level registration of a DPLL clock - * @hw: hardware clock definition for the clock + * @user: pointer to the hardware clock definition for the clock * @node: device node for the clock * * Finalizes DPLL registration process. In case a failure (clk-ref or diff --git a/drivers/clk/ti/dpll3xxx.c b/drivers/clk/ti/dpll3xxx.c index 2490026948b4..6097b099a5df 100644 --- a/drivers/clk/ti/dpll3xxx.c +++ b/drivers/clk/ti/dpll3xxx.c @@ -125,7 +125,7 @@ static u16 _omap3_dpll_compute_freqsel(struct clk_hw_omap *clk, u8 n) return f; } -/* +/** * _omap3_noncore_dpll_lock - instruct a DPLL to lock and wait for readiness * @clk: pointer to a DPLL struct clk * @@ -168,7 +168,7 @@ done: return r; } -/* +/** * _omap3_noncore_dpll_bypass - instruct a DPLL to bypass and wait for readiness * @clk: pointer to a DPLL struct clk * @@ -204,7 +204,7 @@ static int _omap3_noncore_dpll_bypass(struct clk_hw_omap *clk) return r; } -/* +/** * _omap3_noncore_dpll_stop - instruct a DPLL to stop * @clk: pointer to a DPLL struct clk * @@ -291,7 +291,7 @@ static void _lookup_sddiv(struct clk_hw_omap *clk, u8 *sd_div, u16 m, u8 n) *sd_div = sd; } -/* +/** * _omap3_noncore_dpll_program - set non-core DPLL M,N values directly * @clk: struct clk * of DPLL to set * @freqsel: FREQSEL value to set @@ -406,7 +406,8 @@ static int omap3_noncore_dpll_program(struct clk_hw_omap *clk, u16 freqsel) /** * omap3_dpll_recalc - recalculate DPLL rate - * @clk: DPLL struct clk + * @hw: struct clk_hw containing the DPLL struct clk + * @parent_rate: clock rate of the DPLL parent * * Recalculate and propagate the DPLL rate. */ @@ -421,7 +422,7 @@ unsigned long omap3_dpll_recalc(struct clk_hw *hw, unsigned long parent_rate) /** * omap3_noncore_dpll_enable - instruct a DPLL to enter bypass or lock mode - * @clk: pointer to a DPLL struct clk + * @hw: struct clk_hw containing then pointer to a DPLL struct clk * * Instructs a non-CORE DPLL to enable, e.g., to enter bypass or lock. * The choice of modes depends on the DPLL's programmed rate: if it is @@ -470,7 +471,7 @@ int omap3_noncore_dpll_enable(struct clk_hw *hw) /** * omap3_noncore_dpll_disable - instruct a DPLL to enter low-power stop - * @clk: pointer to a DPLL struct clk + * @hw: struct clk_hw containing then pointer to a DPLL struct clk * * Instructs a non-CORE DPLL to enter low-power stop. This function is * intended for use in struct clkops. No return value. @@ -745,7 +746,8 @@ static struct clk_hw_omap *omap3_find_clkoutx2_dpll(struct clk_hw *hw) /** * omap3_clkoutx2_recalc - recalculate DPLL X2 output virtual clock rate - * @clk: DPLL output struct clk + * @hw: pointer struct clk_hw + * @parent_rate: clock rate of the DPLL parent * * Using parent clock DPLL data, look up DPLL state. If locked, set our * rate to the dpll_clk * 2; otherwise, just use dpll_clk. @@ -913,7 +915,7 @@ const struct clk_hw_omap_ops clkhwops_omap3_dpll = { * omap3_dpll4_set_rate - set rate for omap3 per-dpll * @hw: clock to change * @rate: target rate for clock - * @parent_rate: rate of the parent clock + * @parent_rate: clock rate of the DPLL parent * * Check if the current SoC supports the per-dpll reprogram operation * or not, and then do the rate change if supported. Returns -EINVAL diff --git a/drivers/clk/ti/dpll44xx.c b/drivers/clk/ti/dpll44xx.c index 89c3ed1a24b8..3fc2cab69a3f 100644 --- a/drivers/clk/ti/dpll44xx.c +++ b/drivers/clk/ti/dpll44xx.c @@ -102,7 +102,8 @@ static void omap4_dpll_lpmode_recalc(struct dpll_data *dd) /** * omap4_dpll_regm4xen_recalc - compute DPLL rate, considering REGM4XEN bit - * @clk: struct clk * of the DPLL to compute the rate for + * @hw: pointer to the clock to compute the rate for + * @parent_rate: clock rate of the DPLL parent * * Compute the output rate for the OMAP4 DPLL represented by @clk. * Takes the REGM4XEN bit into consideration, which is needed for the @@ -134,8 +135,9 @@ unsigned long omap4_dpll_regm4xen_recalc(struct clk_hw *hw, /** * omap4_dpll_regm4xen_round_rate - round DPLL rate, considering REGM4XEN bit - * @clk: struct clk * of the DPLL to round a rate for + * @hw: struct hw_clk containing the struct clk * of the DPLL to round a rate for * @target_rate: the desired rate of the DPLL + * @parent_rate: clock rate of the DPLL parent * * Compute the rate that would be programmed into the DPLL hardware * for @clk if set_rate() were to be provided with the rate diff --git a/drivers/clk/ti/gate.c b/drivers/clk/ti/gate.c index 42389558418c..b1d0fdb40a75 100644 --- a/drivers/clk/ti/gate.c +++ b/drivers/clk/ti/gate.c @@ -55,7 +55,7 @@ static const struct clk_ops omap_gate_clk_hsdiv_restore_ops = { /** * omap36xx_gate_clk_enable_with_hsdiv_restore - enable clocks suffering * from HSDivider PWRDN problem Implements Errata ID: i556. - * @clk: DPLL output struct clk + * @hw: DPLL output struct clk_hw * * 3630 only: dpll3_m3_ck, dpll4_m2_ck, dpll4_m3_ck, dpll4_m4_ck, * dpll4_m5_ck & dpll4_m6_ck dividers gets loaded with reset diff --git a/drivers/clk/versatile/clk-icst.c b/drivers/clk/versatile/clk-icst.c index 692be2fd9261..fdd6aa3cb1fc 100644 --- a/drivers/clk/versatile/clk-icst.c +++ b/drivers/clk/versatile/clk-icst.c @@ -36,8 +36,9 @@ /** * struct clk_icst - ICST VCO clock wrapper * @hw: corresponding clock hardware entry - * @vcoreg: VCO register address - * @lockreg: VCO lock register address + * @map: register map + * @vcoreg_off: VCO register address + * @lockreg_off: VCO lock register address * @params: parameters for this ICST instance * @rate: current rate * @ctype: the type of control register for the ICST @@ -428,7 +429,7 @@ static const struct icst_params icst307_params = { .idx2s = icst307_idx2s, }; -/** +/* * The core modules on the Integrator/AP and Integrator/CP have * especially crippled ICST525 control. */ diff --git a/drivers/clk/xilinx/Kconfig b/drivers/clk/xilinx/Kconfig new file mode 100644 index 000000000000..5224114176ed --- /dev/null +++ b/drivers/clk/xilinx/Kconfig @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0 + +config XILINX_VCU + tristate "Xilinx VCU logicoreIP Init" + depends on HAS_IOMEM && COMMON_CLK + select REGMAP_MMIO + help + Provides the driver to enable and disable the isolation between the + processing system and programmable logic part by using the logicoreIP + register set. This driver also configures the frequency based on the + clock information from the logicoreIP register set. + + If you say yes here you get support for the logicoreIP. + + If unsure, say N. + + To compile this driver as a module, choose M here: the + module will be called xlnx_vcu. + diff --git a/drivers/clk/xilinx/Makefile b/drivers/clk/xilinx/Makefile new file mode 100644 index 000000000000..dee8fd51e303 --- /dev/null +++ b/drivers/clk/xilinx/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_XILINX_VCU) += xlnx_vcu.o diff --git a/drivers/clk/xilinx/xlnx_vcu.c b/drivers/clk/xilinx/xlnx_vcu.c new file mode 100644 index 000000000000..d66b1315114e --- /dev/null +++ b/drivers/clk/xilinx/xlnx_vcu.c @@ -0,0 +1,743 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Xilinx VCU Init + * + * Copyright (C) 2016 - 2017 Xilinx, Inc. + * + * Contacts Dhaval Shah + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define VCU_PLL_CTRL 0x24 +#define VCU_PLL_CTRL_RESET BIT(0) +#define VCU_PLL_CTRL_POR_IN BIT(1) +#define VCU_PLL_CTRL_PWR_POR BIT(2) +#define VCU_PLL_CTRL_BYPASS BIT(3) +#define VCU_PLL_CTRL_FBDIV GENMASK(14, 8) +#define VCU_PLL_CTRL_CLKOUTDIV GENMASK(18, 16) + +#define VCU_PLL_CFG 0x28 +#define VCU_PLL_CFG_RES GENMASK(3, 0) +#define VCU_PLL_CFG_CP GENMASK(8, 5) +#define VCU_PLL_CFG_LFHF GENMASK(12, 10) +#define VCU_PLL_CFG_LOCK_CNT GENMASK(22, 13) +#define VCU_PLL_CFG_LOCK_DLY GENMASK(31, 25) +#define VCU_ENC_CORE_CTRL 0x30 +#define VCU_ENC_MCU_CTRL 0x34 +#define VCU_DEC_CORE_CTRL 0x38 +#define VCU_DEC_MCU_CTRL 0x3c +#define VCU_PLL_STATUS 0x60 +#define VCU_PLL_STATUS_LOCK_STATUS BIT(0) + +#define MHZ 1000000 +#define FVCO_MIN (1500U * MHZ) +#define FVCO_MAX (3000U * MHZ) + +/** + * struct xvcu_device - Xilinx VCU init device structure + * @dev: Platform device + * @pll_ref: pll ref clock source + * @aclk: axi clock source + * @logicore_reg_ba: logicore reg base address + * @vcu_slcr_ba: vcu_slcr Register base address + * @pll: handle for the VCU PLL + * @pll_post: handle for the VCU PLL post divider + * @clk_data: clocks provided by the vcu clock provider + */ +struct xvcu_device { + struct device *dev; + struct clk *pll_ref; + struct clk *aclk; + struct regmap *logicore_reg_ba; + void __iomem *vcu_slcr_ba; + struct clk_hw *pll; + struct clk_hw *pll_post; + struct clk_hw_onecell_data *clk_data; +}; + +static struct regmap_config vcu_settings_regmap_config = { + .name = "regmap", + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .max_register = 0xfff, + .cache_type = REGCACHE_NONE, +}; + +/** + * struct xvcu_pll_cfg - Helper data + * @fbdiv: The integer portion of the feedback divider to the PLL + * @cp: PLL charge pump control + * @res: PLL loop filter resistor control + * @lfhf: PLL loop filter high frequency capacitor control + * @lock_dly: Lock circuit configuration settings for lock windowsize + * @lock_cnt: Lock circuit counter setting + */ +struct xvcu_pll_cfg { + u32 fbdiv; + u32 cp; + u32 res; + u32 lfhf; + u32 lock_dly; + u32 lock_cnt; +}; + +static const struct xvcu_pll_cfg xvcu_pll_cfg[] = { + { 25, 3, 10, 3, 63, 1000 }, + { 26, 3, 10, 3, 63, 1000 }, + { 27, 4, 6, 3, 63, 1000 }, + { 28, 4, 6, 3, 63, 1000 }, + { 29, 4, 6, 3, 63, 1000 }, + { 30, 4, 6, 3, 63, 1000 }, + { 31, 6, 1, 3, 63, 1000 }, + { 32, 6, 1, 3, 63, 1000 }, + { 33, 4, 10, 3, 63, 1000 }, + { 34, 5, 6, 3, 63, 1000 }, + { 35, 5, 6, 3, 63, 1000 }, + { 36, 5, 6, 3, 63, 1000 }, + { 37, 5, 6, 3, 63, 1000 }, + { 38, 5, 6, 3, 63, 975 }, + { 39, 3, 12, 3, 63, 950 }, + { 40, 3, 12, 3, 63, 925 }, + { 41, 3, 12, 3, 63, 900 }, + { 42, 3, 12, 3, 63, 875 }, + { 43, 3, 12, 3, 63, 850 }, + { 44, 3, 12, 3, 63, 850 }, + { 45, 3, 12, 3, 63, 825 }, + { 46, 3, 12, 3, 63, 800 }, + { 47, 3, 12, 3, 63, 775 }, + { 48, 3, 12, 3, 63, 775 }, + { 49, 3, 12, 3, 63, 750 }, + { 50, 3, 12, 3, 63, 750 }, + { 51, 3, 2, 3, 63, 725 }, + { 52, 3, 2, 3, 63, 700 }, + { 53, 3, 2, 3, 63, 700 }, + { 54, 3, 2, 3, 63, 675 }, + { 55, 3, 2, 3, 63, 675 }, + { 56, 3, 2, 3, 63, 650 }, + { 57, 3, 2, 3, 63, 650 }, + { 58, 3, 2, 3, 63, 625 }, + { 59, 3, 2, 3, 63, 625 }, + { 60, 3, 2, 3, 63, 625 }, + { 61, 3, 2, 3, 63, 600 }, + { 62, 3, 2, 3, 63, 600 }, + { 63, 3, 2, 3, 63, 600 }, + { 64, 3, 2, 3, 63, 600 }, + { 65, 3, 2, 3, 63, 600 }, + { 66, 3, 2, 3, 63, 600 }, + { 67, 3, 2, 3, 63, 600 }, + { 68, 3, 2, 3, 63, 600 }, + { 69, 3, 2, 3, 63, 600 }, + { 70, 3, 2, 3, 63, 600 }, + { 71, 3, 2, 3, 63, 600 }, + { 72, 3, 2, 3, 63, 600 }, + { 73, 3, 2, 3, 63, 600 }, + { 74, 3, 2, 3, 63, 600 }, + { 75, 3, 2, 3, 63, 600 }, + { 76, 3, 2, 3, 63, 600 }, + { 77, 3, 2, 3, 63, 600 }, + { 78, 3, 2, 3, 63, 600 }, + { 79, 3, 2, 3, 63, 600 }, + { 80, 3, 2, 3, 63, 600 }, + { 81, 3, 2, 3, 63, 600 }, + { 82, 3, 2, 3, 63, 600 }, + { 83, 4, 2, 3, 63, 600 }, + { 84, 4, 2, 3, 63, 600 }, + { 85, 4, 2, 3, 63, 600 }, + { 86, 4, 2, 3, 63, 600 }, + { 87, 4, 2, 3, 63, 600 }, + { 88, 4, 2, 3, 63, 600 }, + { 89, 4, 2, 3, 63, 600 }, + { 90, 4, 2, 3, 63, 600 }, + { 91, 4, 2, 3, 63, 600 }, + { 92, 4, 2, 3, 63, 600 }, + { 93, 4, 2, 3, 63, 600 }, + { 94, 4, 2, 3, 63, 600 }, + { 95, 4, 2, 3, 63, 600 }, + { 96, 4, 2, 3, 63, 600 }, + { 97, 4, 2, 3, 63, 600 }, + { 98, 4, 2, 3, 63, 600 }, + { 99, 4, 2, 3, 63, 600 }, + { 100, 4, 2, 3, 63, 600 }, + { 101, 4, 2, 3, 63, 600 }, + { 102, 4, 2, 3, 63, 600 }, + { 103, 5, 2, 3, 63, 600 }, + { 104, 5, 2, 3, 63, 600 }, + { 105, 5, 2, 3, 63, 600 }, + { 106, 5, 2, 3, 63, 600 }, + { 107, 3, 4, 3, 63, 600 }, + { 108, 3, 4, 3, 63, 600 }, + { 109, 3, 4, 3, 63, 600 }, + { 110, 3, 4, 3, 63, 600 }, + { 111, 3, 4, 3, 63, 600 }, + { 112, 3, 4, 3, 63, 600 }, + { 113, 3, 4, 3, 63, 600 }, + { 114, 3, 4, 3, 63, 600 }, + { 115, 3, 4, 3, 63, 600 }, + { 116, 3, 4, 3, 63, 600 }, + { 117, 3, 4, 3, 63, 600 }, + { 118, 3, 4, 3, 63, 600 }, + { 119, 3, 4, 3, 63, 600 }, + { 120, 3, 4, 3, 63, 600 }, + { 121, 3, 4, 3, 63, 600 }, + { 122, 3, 4, 3, 63, 600 }, + { 123, 3, 4, 3, 63, 600 }, + { 124, 3, 4, 3, 63, 600 }, + { 125, 3, 4, 3, 63, 600 }, +}; + +/** + * xvcu_read - Read from the VCU register space + * @iomem: vcu reg space base address + * @offset: vcu reg offset from base + * + * Return: Returns 32bit value from VCU register specified + * + */ +static inline u32 xvcu_read(void __iomem *iomem, u32 offset) +{ + return ioread32(iomem + offset); +} + +/** + * xvcu_write - Write to the VCU register space + * @iomem: vcu reg space base address + * @offset: vcu reg offset from base + * @value: Value to write + */ +static inline void xvcu_write(void __iomem *iomem, u32 offset, u32 value) +{ + iowrite32(value, iomem + offset); +} + +#define to_vcu_pll(_hw) container_of(_hw, struct vcu_pll, hw) + +struct vcu_pll { + struct clk_hw hw; + void __iomem *reg_base; + unsigned long fvco_min; + unsigned long fvco_max; +}; + +static int xvcu_pll_wait_for_lock(struct vcu_pll *pll) +{ + void __iomem *base = pll->reg_base; + unsigned long timeout; + u32 lock_status; + + timeout = jiffies + msecs_to_jiffies(2000); + do { + lock_status = xvcu_read(base, VCU_PLL_STATUS); + if (lock_status & VCU_PLL_STATUS_LOCK_STATUS) + return 0; + } while (!time_after(jiffies, timeout)); + + return -ETIMEDOUT; +} + +static struct clk_hw *xvcu_register_pll_post(struct device *dev, + const char *name, + const struct clk_hw *parent_hw, + void __iomem *reg_base) +{ + u32 div; + u32 vcu_pll_ctrl; + + /* + * The output divider of the PLL must be set to 1/2 to meet the + * timing in the design. + */ + vcu_pll_ctrl = xvcu_read(reg_base, VCU_PLL_CTRL); + div = FIELD_GET(VCU_PLL_CTRL_CLKOUTDIV, vcu_pll_ctrl); + if (div != 1) + return ERR_PTR(-EINVAL); + + return clk_hw_register_fixed_factor(dev, "vcu_pll_post", + clk_hw_get_name(parent_hw), + CLK_SET_RATE_PARENT, 1, 2); +} + +static const struct xvcu_pll_cfg *xvcu_find_cfg(int div) +{ + const struct xvcu_pll_cfg *cfg = NULL; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(xvcu_pll_cfg) - 1; i++) + if (xvcu_pll_cfg[i].fbdiv == div) + cfg = &xvcu_pll_cfg[i]; + + return cfg; +} + +static int xvcu_pll_set_div(struct vcu_pll *pll, int div) +{ + void __iomem *base = pll->reg_base; + const struct xvcu_pll_cfg *cfg = NULL; + u32 vcu_pll_ctrl; + u32 cfg_val; + + cfg = xvcu_find_cfg(div); + if (!cfg) + return -EINVAL; + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + vcu_pll_ctrl &= ~VCU_PLL_CTRL_FBDIV; + vcu_pll_ctrl |= FIELD_PREP(VCU_PLL_CTRL_FBDIV, cfg->fbdiv); + xvcu_write(base, VCU_PLL_CTRL, vcu_pll_ctrl); + + cfg_val = FIELD_PREP(VCU_PLL_CFG_RES, cfg->res) | + FIELD_PREP(VCU_PLL_CFG_CP, cfg->cp) | + FIELD_PREP(VCU_PLL_CFG_LFHF, cfg->lfhf) | + FIELD_PREP(VCU_PLL_CFG_LOCK_CNT, cfg->lock_cnt) | + FIELD_PREP(VCU_PLL_CFG_LOCK_DLY, cfg->lock_dly); + xvcu_write(base, VCU_PLL_CFG, cfg_val); + + return 0; +} + +static long xvcu_pll_round_rate(struct clk_hw *hw, + unsigned long rate, unsigned long *parent_rate) +{ + struct vcu_pll *pll = to_vcu_pll(hw); + unsigned int feedback_div; + + rate = clamp_t(unsigned long, rate, pll->fvco_min, pll->fvco_max); + + feedback_div = DIV_ROUND_CLOSEST_ULL(rate, *parent_rate); + feedback_div = clamp_t(unsigned int, feedback_div, 25, 125); + + return *parent_rate * feedback_div; +} + +static unsigned long xvcu_pll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct vcu_pll *pll = to_vcu_pll(hw); + void __iomem *base = pll->reg_base; + unsigned int div; + u32 vcu_pll_ctrl; + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + div = FIELD_GET(VCU_PLL_CTRL_FBDIV, vcu_pll_ctrl); + + return div * parent_rate; +} + +static int xvcu_pll_set_rate(struct clk_hw *hw, + unsigned long rate, unsigned long parent_rate) +{ + struct vcu_pll *pll = to_vcu_pll(hw); + + return xvcu_pll_set_div(pll, rate / parent_rate); +} + +static int xvcu_pll_enable(struct clk_hw *hw) +{ + struct vcu_pll *pll = to_vcu_pll(hw); + void __iomem *base = pll->reg_base; + u32 vcu_pll_ctrl; + int ret; + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + vcu_pll_ctrl |= VCU_PLL_CTRL_BYPASS; + xvcu_write(base, VCU_PLL_CTRL, vcu_pll_ctrl); + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + vcu_pll_ctrl &= ~VCU_PLL_CTRL_POR_IN; + vcu_pll_ctrl &= ~VCU_PLL_CTRL_PWR_POR; + vcu_pll_ctrl &= ~VCU_PLL_CTRL_RESET; + xvcu_write(base, VCU_PLL_CTRL, vcu_pll_ctrl); + + ret = xvcu_pll_wait_for_lock(pll); + if (ret) { + pr_err("VCU PLL is not locked\n"); + goto err; + } + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + vcu_pll_ctrl &= ~VCU_PLL_CTRL_BYPASS; + xvcu_write(base, VCU_PLL_CTRL, vcu_pll_ctrl); + +err: + return ret; +} + +static void xvcu_pll_disable(struct clk_hw *hw) +{ + struct vcu_pll *pll = to_vcu_pll(hw); + void __iomem *base = pll->reg_base; + u32 vcu_pll_ctrl; + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + vcu_pll_ctrl |= VCU_PLL_CTRL_POR_IN; + vcu_pll_ctrl |= VCU_PLL_CTRL_PWR_POR; + vcu_pll_ctrl |= VCU_PLL_CTRL_RESET; + xvcu_write(base, VCU_PLL_CTRL, vcu_pll_ctrl); +} + +static const struct clk_ops vcu_pll_ops = { + .enable = xvcu_pll_enable, + .disable = xvcu_pll_disable, + .round_rate = xvcu_pll_round_rate, + .recalc_rate = xvcu_pll_recalc_rate, + .set_rate = xvcu_pll_set_rate, +}; + +static struct clk_hw *xvcu_register_pll(struct device *dev, + void __iomem *reg_base, + const char *name, const char *parent, + unsigned long flags) +{ + struct vcu_pll *pll; + struct clk_hw *hw; + struct clk_init_data init; + int ret; + + init.name = name; + init.parent_names = &parent; + init.ops = &vcu_pll_ops; + init.num_parents = 1; + init.flags = flags; + + pll = devm_kmalloc(dev, sizeof(*pll), GFP_KERNEL); + if (!pll) + return ERR_PTR(-ENOMEM); + + pll->hw.init = &init; + pll->reg_base = reg_base; + pll->fvco_min = FVCO_MIN; + pll->fvco_max = FVCO_MAX; + + hw = &pll->hw; + ret = devm_clk_hw_register(dev, hw); + if (ret) + return ERR_PTR(ret); + + clk_hw_set_rate_range(hw, pll->fvco_min, pll->fvco_max); + + return hw; +} + +static struct clk_hw *xvcu_clk_hw_register_leaf(struct device *dev, + const char *name, + const struct clk_parent_data *parent_data, + u8 num_parents, + void __iomem *reg) +{ + u8 mux_flags = CLK_MUX_ROUND_CLOSEST; + u8 divider_flags = CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO | + CLK_DIVIDER_ROUND_CLOSEST; + struct clk_hw *mux = NULL; + struct clk_hw *divider = NULL; + struct clk_hw *gate = NULL; + char *name_mux; + char *name_div; + int err; + /* Protect register shared by clocks */ + spinlock_t *lock; + + lock = devm_kzalloc(dev, sizeof(*lock), GFP_KERNEL); + if (!lock) + return ERR_PTR(-ENOMEM); + spin_lock_init(lock); + + name_mux = devm_kasprintf(dev, GFP_KERNEL, "%s%s", name, "_mux"); + if (!name_mux) + return ERR_PTR(-ENOMEM); + mux = clk_hw_register_mux_parent_data(dev, name_mux, + parent_data, num_parents, + CLK_SET_RATE_PARENT, + reg, 0, 1, mux_flags, lock); + if (IS_ERR(mux)) + return mux; + + name_div = devm_kasprintf(dev, GFP_KERNEL, "%s%s", name, "_div"); + if (!name_div) { + err = -ENOMEM; + goto unregister_mux; + } + divider = clk_hw_register_divider_parent_hw(dev, name_div, mux, + CLK_SET_RATE_PARENT, + reg, 4, 6, divider_flags, + lock); + if (IS_ERR(divider)) { + err = PTR_ERR(divider); + goto unregister_mux; + } + + gate = clk_hw_register_gate_parent_hw(dev, name, divider, + CLK_SET_RATE_PARENT, reg, 12, 0, + lock); + if (IS_ERR(gate)) { + err = PTR_ERR(gate); + goto unregister_divider; + } + + return gate; + +unregister_divider: + clk_hw_unregister_divider(divider); +unregister_mux: + clk_hw_unregister_mux(mux); + + return ERR_PTR(err); +} + +static void xvcu_clk_hw_unregister_leaf(struct clk_hw *hw) +{ + struct clk_hw *gate = hw; + struct clk_hw *divider; + struct clk_hw *mux; + + if (!gate) + return; + + divider = clk_hw_get_parent(gate); + clk_hw_unregister_gate(gate); + if (!divider) + return; + + mux = clk_hw_get_parent(divider); + clk_hw_unregister_mux(mux); + if (!divider) + return; + + clk_hw_unregister_divider(divider); +} + +static int xvcu_register_clock_provider(struct xvcu_device *xvcu) +{ + struct device *dev = xvcu->dev; + struct clk_parent_data parent_data[2] = { 0 }; + struct clk_hw_onecell_data *data; + struct clk_hw **hws; + struct clk_hw *hw; + void __iomem *reg_base = xvcu->vcu_slcr_ba; + + data = devm_kzalloc(dev, struct_size(data, hws, CLK_XVCU_NUM_CLOCKS), GFP_KERNEL); + if (!data) + return -ENOMEM; + data->num = CLK_XVCU_NUM_CLOCKS; + hws = data->hws; + + xvcu->clk_data = data; + + hw = xvcu_register_pll(dev, reg_base, + "vcu_pll", __clk_get_name(xvcu->pll_ref), + CLK_SET_RATE_NO_REPARENT | CLK_OPS_PARENT_ENABLE); + if (IS_ERR(hw)) + return PTR_ERR(hw); + xvcu->pll = hw; + + hw = xvcu_register_pll_post(dev, "vcu_pll_post", xvcu->pll, reg_base); + if (IS_ERR(hw)) + return PTR_ERR(hw); + xvcu->pll_post = hw; + + parent_data[0].fw_name = "pll_ref"; + parent_data[1].hw = xvcu->pll_post; + + hws[CLK_XVCU_ENC_CORE] = + xvcu_clk_hw_register_leaf(dev, "venc_core_clk", + parent_data, + ARRAY_SIZE(parent_data), + reg_base + VCU_ENC_CORE_CTRL); + hws[CLK_XVCU_ENC_MCU] = + xvcu_clk_hw_register_leaf(dev, "venc_mcu_clk", + parent_data, + ARRAY_SIZE(parent_data), + reg_base + VCU_ENC_MCU_CTRL); + hws[CLK_XVCU_DEC_CORE] = + xvcu_clk_hw_register_leaf(dev, "vdec_core_clk", + parent_data, + ARRAY_SIZE(parent_data), + reg_base + VCU_DEC_CORE_CTRL); + hws[CLK_XVCU_DEC_MCU] = + xvcu_clk_hw_register_leaf(dev, "vdec_mcu_clk", + parent_data, + ARRAY_SIZE(parent_data), + reg_base + VCU_DEC_MCU_CTRL); + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, data); +} + +static void xvcu_unregister_clock_provider(struct xvcu_device *xvcu) +{ + struct clk_hw_onecell_data *data = xvcu->clk_data; + struct clk_hw **hws = data->hws; + + if (!IS_ERR_OR_NULL(hws[CLK_XVCU_DEC_MCU])) + xvcu_clk_hw_unregister_leaf(hws[CLK_XVCU_DEC_MCU]); + if (!IS_ERR_OR_NULL(hws[CLK_XVCU_DEC_CORE])) + xvcu_clk_hw_unregister_leaf(hws[CLK_XVCU_DEC_CORE]); + if (!IS_ERR_OR_NULL(hws[CLK_XVCU_ENC_MCU])) + xvcu_clk_hw_unregister_leaf(hws[CLK_XVCU_ENC_MCU]); + if (!IS_ERR_OR_NULL(hws[CLK_XVCU_ENC_CORE])) + xvcu_clk_hw_unregister_leaf(hws[CLK_XVCU_ENC_CORE]); + + clk_hw_unregister_fixed_factor(xvcu->pll_post); +} + +/** + * xvcu_probe - Probe existence of the logicoreIP + * and initialize PLL + * + * @pdev: Pointer to the platform_device structure + * + * Return: Returns 0 on success + * Negative error code otherwise + */ +static int xvcu_probe(struct platform_device *pdev) +{ + struct resource *res; + struct xvcu_device *xvcu; + void __iomem *regs; + int ret; + + xvcu = devm_kzalloc(&pdev->dev, sizeof(*xvcu), GFP_KERNEL); + if (!xvcu) + return -ENOMEM; + + xvcu->dev = &pdev->dev; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vcu_slcr"); + if (!res) { + dev_err(&pdev->dev, "get vcu_slcr memory resource failed.\n"); + return -ENODEV; + } + + xvcu->vcu_slcr_ba = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!xvcu->vcu_slcr_ba) { + dev_err(&pdev->dev, "vcu_slcr register mapping failed.\n"); + return -ENOMEM; + } + + xvcu->logicore_reg_ba = + syscon_regmap_lookup_by_compatible("xlnx,vcu-settings"); + if (IS_ERR(xvcu->logicore_reg_ba)) { + dev_info(&pdev->dev, + "could not find xlnx,vcu-settings: trying direct register access\n"); + + res = platform_get_resource_byname(pdev, + IORESOURCE_MEM, "logicore"); + if (!res) { + dev_err(&pdev->dev, "get logicore memory resource failed.\n"); + return -ENODEV; + } + + regs = devm_ioremap(&pdev->dev, res->start, resource_size(res)); + if (!regs) { + dev_err(&pdev->dev, "logicore register mapping failed.\n"); + return -ENOMEM; + } + + xvcu->logicore_reg_ba = + devm_regmap_init_mmio(&pdev->dev, regs, + &vcu_settings_regmap_config); + if (IS_ERR(xvcu->logicore_reg_ba)) { + dev_err(&pdev->dev, "failed to init regmap\n"); + return PTR_ERR(xvcu->logicore_reg_ba); + } + } + + xvcu->aclk = devm_clk_get(&pdev->dev, "aclk"); + if (IS_ERR(xvcu->aclk)) { + dev_err(&pdev->dev, "Could not get aclk clock\n"); + return PTR_ERR(xvcu->aclk); + } + + xvcu->pll_ref = devm_clk_get(&pdev->dev, "pll_ref"); + if (IS_ERR(xvcu->pll_ref)) { + dev_err(&pdev->dev, "Could not get pll_ref clock\n"); + return PTR_ERR(xvcu->pll_ref); + } + + ret = clk_prepare_enable(xvcu->aclk); + if (ret) { + dev_err(&pdev->dev, "aclk clock enable failed\n"); + return ret; + } + + /* + * Do the Gasket isolation and put the VCU out of reset + * Bit 0 : Gasket isolation + * Bit 1 : put VCU out of reset + */ + regmap_write(xvcu->logicore_reg_ba, VCU_GASKET_INIT, VCU_GASKET_VALUE); + + ret = xvcu_register_clock_provider(xvcu); + if (ret) { + dev_err(&pdev->dev, "failed to register clock provider\n"); + goto error_clk_provider; + } + + dev_set_drvdata(&pdev->dev, xvcu); + + return 0; + +error_clk_provider: + xvcu_unregister_clock_provider(xvcu); + clk_disable_unprepare(xvcu->aclk); + return ret; +} + +/** + * xvcu_remove - Insert gasket isolation + * and disable the clock + * @pdev: Pointer to the platform_device structure + * + * Return: Returns 0 on success + * Negative error code otherwise + */ +static int xvcu_remove(struct platform_device *pdev) +{ + struct xvcu_device *xvcu; + + xvcu = platform_get_drvdata(pdev); + if (!xvcu) + return -ENODEV; + + xvcu_unregister_clock_provider(xvcu); + + /* Add the Gasket isolation and put the VCU in reset. */ + regmap_write(xvcu->logicore_reg_ba, VCU_GASKET_INIT, 0); + + clk_disable_unprepare(xvcu->aclk); + + return 0; +} + +static const struct of_device_id xvcu_of_id_table[] = { + { .compatible = "xlnx,vcu" }, + { .compatible = "xlnx,vcu-logicoreip-1.0" }, + { } +}; +MODULE_DEVICE_TABLE(of, xvcu_of_id_table); + +static struct platform_driver xvcu_driver = { + .driver = { + .name = "xilinx-vcu", + .of_match_table = xvcu_of_id_table, + }, + .probe = xvcu_probe, + .remove = xvcu_remove, +}; + +module_platform_driver(xvcu_driver); + +MODULE_AUTHOR("Dhaval Shah "); +MODULE_DESCRIPTION("Xilinx VCU init Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/zte/Makefile b/drivers/clk/zte/Makefile deleted file mode 100644 index f130643b695d..000000000000 --- a/drivers/clk/zte/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-y := clk.o -obj-$(CONFIG_SOC_ZX296702) += clk-zx296702.o -obj-$(CONFIG_ARCH_ZX) += clk-zx296718.o diff --git a/drivers/clk/zte/clk-zx296702.c b/drivers/clk/zte/clk-zx296702.c deleted file mode 100644 index e846f2a34feb..000000000000 --- a/drivers/clk/zte/clk-zx296702.c +++ /dev/null @@ -1,741 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2014 Linaro Ltd. - * Copyright (C) 2014 ZTE Corporation. - */ - -#include -#include -#include -#include "clk.h" - -static DEFINE_SPINLOCK(reg_lock); - -static void __iomem *topcrm_base; -static void __iomem *lsp0crpm_base; -static void __iomem *lsp1crpm_base; - -static struct clk *topclk[ZX296702_TOPCLK_END]; -static struct clk *lsp0clk[ZX296702_LSP0CLK_END]; -static struct clk *lsp1clk[ZX296702_LSP1CLK_END]; - -static struct clk_onecell_data topclk_data; -static struct clk_onecell_data lsp0clk_data; -static struct clk_onecell_data lsp1clk_data; - -#define CLK_MUX (topcrm_base + 0x04) -#define CLK_DIV (topcrm_base + 0x08) -#define CLK_EN0 (topcrm_base + 0x0c) -#define CLK_EN1 (topcrm_base + 0x10) -#define VOU_LOCAL_CLKEN (topcrm_base + 0x68) -#define VOU_LOCAL_CLKSEL (topcrm_base + 0x70) -#define VOU_LOCAL_DIV2_SET (topcrm_base + 0x74) -#define CLK_MUX1 (topcrm_base + 0x8c) - -#define CLK_SDMMC1 (lsp0crpm_base + 0x0c) -#define CLK_GPIO (lsp0crpm_base + 0x2c) -#define CLK_SPDIF0 (lsp0crpm_base + 0x10) -#define SPDIF0_DIV (lsp0crpm_base + 0x14) -#define CLK_I2S0 (lsp0crpm_base + 0x18) -#define I2S0_DIV (lsp0crpm_base + 0x1c) -#define CLK_I2S1 (lsp0crpm_base + 0x20) -#define I2S1_DIV (lsp0crpm_base + 0x24) -#define CLK_I2S2 (lsp0crpm_base + 0x34) -#define I2S2_DIV (lsp0crpm_base + 0x38) - -#define CLK_UART0 (lsp1crpm_base + 0x20) -#define CLK_UART1 (lsp1crpm_base + 0x24) -#define CLK_SDMMC0 (lsp1crpm_base + 0x2c) -#define CLK_SPDIF1 (lsp1crpm_base + 0x30) -#define SPDIF1_DIV (lsp1crpm_base + 0x34) - -static const struct zx_pll_config pll_a9_config[] = { - { .rate = 700000000, .cfg0 = 0x800405d1, .cfg1 = 0x04555555 }, - { .rate = 800000000, .cfg0 = 0x80040691, .cfg1 = 0x04aaaaaa }, - { .rate = 900000000, .cfg0 = 0x80040791, .cfg1 = 0x04000000 }, - { .rate = 1000000000, .cfg0 = 0x80040851, .cfg1 = 0x04555555 }, - { .rate = 1100000000, .cfg0 = 0x80040911, .cfg1 = 0x04aaaaaa }, - { .rate = 1200000000, .cfg0 = 0x80040a11, .cfg1 = 0x04000000 }, -}; - -static const struct clk_div_table main_hlk_div[] = { - { .val = 1, .div = 2, }, - { .val = 3, .div = 4, }, - { /* sentinel */ } -}; - -static const struct clk_div_table a9_as1_aclk_divider[] = { - { .val = 0, .div = 1, }, - { .val = 1, .div = 2, }, - { .val = 3, .div = 4, }, - { /* sentinel */ } -}; - -static const struct clk_div_table sec_wclk_divider[] = { - { .val = 0, .div = 1, }, - { .val = 1, .div = 2, }, - { .val = 3, .div = 4, }, - { .val = 5, .div = 6, }, - { .val = 7, .div = 8, }, - { /* sentinel */ } -}; - -static const char * const matrix_aclk_sel[] = { - "pll_mm0_198M", - "osc", - "clk_148M5", - "pll_lsp_104M", -}; - -static const char * const a9_wclk_sel[] = { - "pll_a9", - "osc", - "clk_500", - "clk_250", -}; - -static const char * const a9_as1_aclk_sel[] = { - "clk_250", - "osc", - "pll_mm0_396M", - "pll_mac_333M", -}; - -static const char * const a9_trace_clkin_sel[] = { - "clk_74M25", - "pll_mm1_108M", - "clk_125", - "clk_148M5", -}; - -static const char * const decppu_aclk_sel[] = { - "clk_250", - "pll_mm0_198M", - "pll_lsp_104M", - "pll_audio_294M912", -}; - -static const char * const vou_main_wclk_sel[] = { - "clk_148M5", - "clk_74M25", - "clk_27", - "pll_mm1_54M", -}; - -static const char * const vou_scaler_wclk_sel[] = { - "clk_250", - "pll_mac_333M", - "pll_audio_294M912", - "pll_mm0_198M", -}; - -static const char * const r2d_wclk_sel[] = { - "pll_audio_294M912", - "pll_mac_333M", - "pll_a9_350M", - "pll_mm0_396M", -}; - -static const char * const ddr_wclk_sel[] = { - "pll_mac_333M", - "pll_ddr_266M", - "pll_audio_294M912", - "pll_mm0_198M", -}; - -static const char * const nand_wclk_sel[] = { - "pll_lsp_104M", - "osc", -}; - -static const char * const lsp_26_wclk_sel[] = { - "pll_lsp_26M", - "osc", -}; - -static const char * const vl0_sel[] = { - "vou_main_channel_div", - "vou_aux_channel_div", -}; - -static const char * const hdmi_sel[] = { - "vou_main_channel_wclk", - "vou_aux_channel_wclk", -}; - -static const char * const sdmmc0_wclk_sel[] = { - "lsp1_104M_wclk", - "lsp1_26M_wclk", -}; - -static const char * const sdmmc1_wclk_sel[] = { - "lsp0_104M_wclk", - "lsp0_26M_wclk", -}; - -static const char * const uart_wclk_sel[] = { - "lsp1_104M_wclk", - "lsp1_26M_wclk", -}; - -static const char * const spdif0_wclk_sel[] = { - "lsp0_104M_wclk", - "lsp0_26M_wclk", -}; - -static const char * const spdif1_wclk_sel[] = { - "lsp1_104M_wclk", - "lsp1_26M_wclk", -}; - -static const char * const i2s_wclk_sel[] = { - "lsp0_104M_wclk", - "lsp0_26M_wclk", -}; - -static inline struct clk *zx_divtbl(const char *name, const char *parent, - void __iomem *reg, u8 shift, u8 width, - const struct clk_div_table *table) -{ - return clk_register_divider_table(NULL, name, parent, 0, reg, shift, - width, 0, table, ®_lock); -} - -static inline struct clk *zx_div(const char *name, const char *parent, - void __iomem *reg, u8 shift, u8 width) -{ - return clk_register_divider(NULL, name, parent, 0, - reg, shift, width, 0, ®_lock); -} - -static inline struct clk *zx_mux(const char *name, const char * const *parents, - int num_parents, void __iomem *reg, u8 shift, u8 width) -{ - return clk_register_mux(NULL, name, parents, num_parents, - 0, reg, shift, width, 0, ®_lock); -} - -static inline struct clk *zx_gate(const char *name, const char *parent, - void __iomem *reg, u8 shift) -{ - return clk_register_gate(NULL, name, parent, CLK_IGNORE_UNUSED, - reg, shift, CLK_SET_RATE_PARENT, ®_lock); -} - -static void __init zx296702_top_clocks_init(struct device_node *np) -{ - struct clk **clk = topclk; - int i; - - topcrm_base = of_iomap(np, 0); - WARN_ON(!topcrm_base); - - clk[ZX296702_OSC] = - clk_register_fixed_rate(NULL, "osc", NULL, 0, 30000000); - clk[ZX296702_PLL_A9] = - clk_register_zx_pll("pll_a9", "osc", 0, topcrm_base - + 0x01c, pll_a9_config, - ARRAY_SIZE(pll_a9_config), ®_lock); - - /* TODO: pll_a9_350M look like changeble follow a9 pll */ - clk[ZX296702_PLL_A9_350M] = - clk_register_fixed_rate(NULL, "pll_a9_350M", "osc", 0, - 350000000); - clk[ZX296702_PLL_MAC_1000M] = - clk_register_fixed_rate(NULL, "pll_mac_1000M", "osc", 0, - 1000000000); - clk[ZX296702_PLL_MAC_333M] = - clk_register_fixed_rate(NULL, "pll_mac_333M", "osc", 0, - 333000000); - clk[ZX296702_PLL_MM0_1188M] = - clk_register_fixed_rate(NULL, "pll_mm0_1188M", "osc", 0, - 1188000000); - clk[ZX296702_PLL_MM0_396M] = - clk_register_fixed_rate(NULL, "pll_mm0_396M", "osc", 0, - 396000000); - clk[ZX296702_PLL_MM0_198M] = - clk_register_fixed_rate(NULL, "pll_mm0_198M", "osc", 0, - 198000000); - clk[ZX296702_PLL_MM1_108M] = - clk_register_fixed_rate(NULL, "pll_mm1_108M", "osc", 0, - 108000000); - clk[ZX296702_PLL_MM1_72M] = - clk_register_fixed_rate(NULL, "pll_mm1_72M", "osc", 0, - 72000000); - clk[ZX296702_PLL_MM1_54M] = - clk_register_fixed_rate(NULL, "pll_mm1_54M", "osc", 0, - 54000000); - clk[ZX296702_PLL_LSP_104M] = - clk_register_fixed_rate(NULL, "pll_lsp_104M", "osc", 0, - 104000000); - clk[ZX296702_PLL_LSP_26M] = - clk_register_fixed_rate(NULL, "pll_lsp_26M", "osc", 0, - 26000000); - clk[ZX296702_PLL_DDR_266M] = - clk_register_fixed_rate(NULL, "pll_ddr_266M", "osc", 0, - 266000000); - clk[ZX296702_PLL_AUDIO_294M912] = - clk_register_fixed_rate(NULL, "pll_audio_294M912", "osc", 0, - 294912000); - - /* bus clock */ - clk[ZX296702_MATRIX_ACLK] = - zx_mux("matrix_aclk", matrix_aclk_sel, - ARRAY_SIZE(matrix_aclk_sel), CLK_MUX, 2, 2); - clk[ZX296702_MAIN_HCLK] = - zx_divtbl("main_hclk", "matrix_aclk", CLK_DIV, 0, 2, - main_hlk_div); - clk[ZX296702_MAIN_PCLK] = - zx_divtbl("main_pclk", "matrix_aclk", CLK_DIV, 2, 2, - main_hlk_div); - - /* cpu clock */ - clk[ZX296702_CLK_500] = - clk_register_fixed_factor(NULL, "clk_500", "pll_mac_1000M", 0, - 1, 2); - clk[ZX296702_CLK_250] = - clk_register_fixed_factor(NULL, "clk_250", "pll_mac_1000M", 0, - 1, 4); - clk[ZX296702_CLK_125] = - clk_register_fixed_factor(NULL, "clk_125", "clk_250", 0, 1, 2); - clk[ZX296702_CLK_148M5] = - clk_register_fixed_factor(NULL, "clk_148M5", "pll_mm0_1188M", 0, - 1, 8); - clk[ZX296702_CLK_74M25] = - clk_register_fixed_factor(NULL, "clk_74M25", "pll_mm0_1188M", 0, - 1, 16); - clk[ZX296702_A9_WCLK] = - zx_mux("a9_wclk", a9_wclk_sel, ARRAY_SIZE(a9_wclk_sel), CLK_MUX, - 0, 2); - clk[ZX296702_A9_AS1_ACLK_MUX] = - zx_mux("a9_as1_aclk_mux", a9_as1_aclk_sel, - ARRAY_SIZE(a9_as1_aclk_sel), CLK_MUX, 4, 2); - clk[ZX296702_A9_TRACE_CLKIN_MUX] = - zx_mux("a9_trace_clkin_mux", a9_trace_clkin_sel, - ARRAY_SIZE(a9_trace_clkin_sel), CLK_MUX1, 0, 2); - clk[ZX296702_A9_AS1_ACLK_DIV] = - zx_divtbl("a9_as1_aclk_div", "a9_as1_aclk_mux", CLK_DIV, 4, 2, - a9_as1_aclk_divider); - - /* multi-media clock */ - clk[ZX296702_CLK_2] = - clk_register_fixed_factor(NULL, "clk_2", "pll_mm1_72M", 0, - 1, 36); - clk[ZX296702_CLK_27] = - clk_register_fixed_factor(NULL, "clk_27", "pll_mm1_54M", 0, - 1, 2); - clk[ZX296702_DECPPU_ACLK_MUX] = - zx_mux("decppu_aclk_mux", decppu_aclk_sel, - ARRAY_SIZE(decppu_aclk_sel), CLK_MUX, 6, 2); - clk[ZX296702_PPU_ACLK_MUX] = - zx_mux("ppu_aclk_mux", decppu_aclk_sel, - ARRAY_SIZE(decppu_aclk_sel), CLK_MUX, 8, 2); - clk[ZX296702_MALI400_ACLK_MUX] = - zx_mux("mali400_aclk_mux", decppu_aclk_sel, - ARRAY_SIZE(decppu_aclk_sel), CLK_MUX, 12, 2); - clk[ZX296702_VOU_ACLK_MUX] = - zx_mux("vou_aclk_mux", decppu_aclk_sel, - ARRAY_SIZE(decppu_aclk_sel), CLK_MUX, 10, 2); - clk[ZX296702_VOU_MAIN_WCLK_MUX] = - zx_mux("vou_main_wclk_mux", vou_main_wclk_sel, - ARRAY_SIZE(vou_main_wclk_sel), CLK_MUX, 14, 2); - clk[ZX296702_VOU_AUX_WCLK_MUX] = - zx_mux("vou_aux_wclk_mux", vou_main_wclk_sel, - ARRAY_SIZE(vou_main_wclk_sel), CLK_MUX, 16, 2); - clk[ZX296702_VOU_SCALER_WCLK_MUX] = - zx_mux("vou_scaler_wclk_mux", vou_scaler_wclk_sel, - ARRAY_SIZE(vou_scaler_wclk_sel), CLK_MUX, - 18, 2); - clk[ZX296702_R2D_ACLK_MUX] = - zx_mux("r2d_aclk_mux", decppu_aclk_sel, - ARRAY_SIZE(decppu_aclk_sel), CLK_MUX, 20, 2); - clk[ZX296702_R2D_WCLK_MUX] = - zx_mux("r2d_wclk_mux", r2d_wclk_sel, - ARRAY_SIZE(r2d_wclk_sel), CLK_MUX, 22, 2); - - /* other clock */ - clk[ZX296702_CLK_50] = - clk_register_fixed_factor(NULL, "clk_50", "pll_mac_1000M", - 0, 1, 20); - clk[ZX296702_CLK_25] = - clk_register_fixed_factor(NULL, "clk_25", "pll_mac_1000M", - 0, 1, 40); - clk[ZX296702_CLK_12] = - clk_register_fixed_factor(NULL, "clk_12", "pll_mm1_72M", - 0, 1, 6); - clk[ZX296702_CLK_16M384] = - clk_register_fixed_factor(NULL, "clk_16M384", - "pll_audio_294M912", 0, 1, 18); - clk[ZX296702_CLK_32K768] = - clk_register_fixed_factor(NULL, "clk_32K768", "clk_16M384", - 0, 1, 500); - clk[ZX296702_SEC_WCLK_DIV] = - zx_divtbl("sec_wclk_div", "pll_lsp_104M", CLK_DIV, 6, 3, - sec_wclk_divider); - clk[ZX296702_DDR_WCLK_MUX] = - zx_mux("ddr_wclk_mux", ddr_wclk_sel, - ARRAY_SIZE(ddr_wclk_sel), CLK_MUX, 24, 2); - clk[ZX296702_NAND_WCLK_MUX] = - zx_mux("nand_wclk_mux", nand_wclk_sel, - ARRAY_SIZE(nand_wclk_sel), CLK_MUX, 24, 2); - clk[ZX296702_LSP_26_WCLK_MUX] = - zx_mux("lsp_26_wclk_mux", lsp_26_wclk_sel, - ARRAY_SIZE(lsp_26_wclk_sel), CLK_MUX, 27, 1); - - /* gates */ - clk[ZX296702_A9_AS0_ACLK] = - zx_gate("a9_as0_aclk", "matrix_aclk", CLK_EN0, 0); - clk[ZX296702_A9_AS1_ACLK] = - zx_gate("a9_as1_aclk", "a9_as1_aclk_div", CLK_EN0, 1); - clk[ZX296702_A9_TRACE_CLKIN] = - zx_gate("a9_trace_clkin", "a9_trace_clkin_mux", CLK_EN0, 2); - clk[ZX296702_DECPPU_AXI_M_ACLK] = - zx_gate("decppu_axi_m_aclk", "decppu_aclk_mux", CLK_EN0, 3); - clk[ZX296702_DECPPU_AHB_S_HCLK] = - zx_gate("decppu_ahb_s_hclk", "main_hclk", CLK_EN0, 4); - clk[ZX296702_PPU_AXI_M_ACLK] = - zx_gate("ppu_axi_m_aclk", "ppu_aclk_mux", CLK_EN0, 5); - clk[ZX296702_PPU_AHB_S_HCLK] = - zx_gate("ppu_ahb_s_hclk", "main_hclk", CLK_EN0, 6); - clk[ZX296702_VOU_AXI_M_ACLK] = - zx_gate("vou_axi_m_aclk", "vou_aclk_mux", CLK_EN0, 7); - clk[ZX296702_VOU_APB_PCLK] = - zx_gate("vou_apb_pclk", "main_pclk", CLK_EN0, 8); - clk[ZX296702_VOU_MAIN_CHANNEL_WCLK] = - zx_gate("vou_main_channel_wclk", "vou_main_wclk_mux", - CLK_EN0, 9); - clk[ZX296702_VOU_AUX_CHANNEL_WCLK] = - zx_gate("vou_aux_channel_wclk", "vou_aux_wclk_mux", - CLK_EN0, 10); - clk[ZX296702_VOU_HDMI_OSCLK_CEC] = - zx_gate("vou_hdmi_osclk_cec", "clk_2", CLK_EN0, 11); - clk[ZX296702_VOU_SCALER_WCLK] = - zx_gate("vou_scaler_wclk", "vou_scaler_wclk_mux", CLK_EN0, 12); - clk[ZX296702_MALI400_AXI_M_ACLK] = - zx_gate("mali400_axi_m_aclk", "mali400_aclk_mux", CLK_EN0, 13); - clk[ZX296702_MALI400_APB_PCLK] = - zx_gate("mali400_apb_pclk", "main_pclk", CLK_EN0, 14); - clk[ZX296702_R2D_WCLK] = - zx_gate("r2d_wclk", "r2d_wclk_mux", CLK_EN0, 15); - clk[ZX296702_R2D_AXI_M_ACLK] = - zx_gate("r2d_axi_m_aclk", "r2d_aclk_mux", CLK_EN0, 16); - clk[ZX296702_R2D_AHB_HCLK] = - zx_gate("r2d_ahb_hclk", "main_hclk", CLK_EN0, 17); - clk[ZX296702_DDR3_AXI_S0_ACLK] = - zx_gate("ddr3_axi_s0_aclk", "matrix_aclk", CLK_EN0, 18); - clk[ZX296702_DDR3_APB_PCLK] = - zx_gate("ddr3_apb_pclk", "main_pclk", CLK_EN0, 19); - clk[ZX296702_DDR3_WCLK] = - zx_gate("ddr3_wclk", "ddr_wclk_mux", CLK_EN0, 20); - clk[ZX296702_USB20_0_AHB_HCLK] = - zx_gate("usb20_0_ahb_hclk", "main_hclk", CLK_EN0, 21); - clk[ZX296702_USB20_0_EXTREFCLK] = - zx_gate("usb20_0_extrefclk", "clk_12", CLK_EN0, 22); - clk[ZX296702_USB20_1_AHB_HCLK] = - zx_gate("usb20_1_ahb_hclk", "main_hclk", CLK_EN0, 23); - clk[ZX296702_USB20_1_EXTREFCLK] = - zx_gate("usb20_1_extrefclk", "clk_12", CLK_EN0, 24); - clk[ZX296702_USB20_2_AHB_HCLK] = - zx_gate("usb20_2_ahb_hclk", "main_hclk", CLK_EN0, 25); - clk[ZX296702_USB20_2_EXTREFCLK] = - zx_gate("usb20_2_extrefclk", "clk_12", CLK_EN0, 26); - clk[ZX296702_GMAC_AXI_M_ACLK] = - zx_gate("gmac_axi_m_aclk", "matrix_aclk", CLK_EN0, 27); - clk[ZX296702_GMAC_APB_PCLK] = - zx_gate("gmac_apb_pclk", "main_pclk", CLK_EN0, 28); - clk[ZX296702_GMAC_125_CLKIN] = - zx_gate("gmac_125_clkin", "clk_125", CLK_EN0, 29); - clk[ZX296702_GMAC_RMII_CLKIN] = - zx_gate("gmac_rmii_clkin", "clk_50", CLK_EN0, 30); - clk[ZX296702_GMAC_25M_CLK] = - zx_gate("gmac_25M_clk", "clk_25", CLK_EN0, 31); - clk[ZX296702_NANDFLASH_AHB_HCLK] = - zx_gate("nandflash_ahb_hclk", "main_hclk", CLK_EN1, 0); - clk[ZX296702_NANDFLASH_WCLK] = - zx_gate("nandflash_wclk", "nand_wclk_mux", CLK_EN1, 1); - clk[ZX296702_LSP0_APB_PCLK] = - zx_gate("lsp0_apb_pclk", "main_pclk", CLK_EN1, 2); - clk[ZX296702_LSP0_AHB_HCLK] = - zx_gate("lsp0_ahb_hclk", "main_hclk", CLK_EN1, 3); - clk[ZX296702_LSP0_26M_WCLK] = - zx_gate("lsp0_26M_wclk", "lsp_26_wclk_mux", CLK_EN1, 4); - clk[ZX296702_LSP0_104M_WCLK] = - zx_gate("lsp0_104M_wclk", "pll_lsp_104M", CLK_EN1, 5); - clk[ZX296702_LSP0_16M384_WCLK] = - zx_gate("lsp0_16M384_wclk", "clk_16M384", CLK_EN1, 6); - clk[ZX296702_LSP1_APB_PCLK] = - zx_gate("lsp1_apb_pclk", "main_pclk", CLK_EN1, 7); - /* FIXME: wclk enable bit is bit8. We hack it as reserved 31 for - * UART does not work after parent clk is disabled/enabled */ - clk[ZX296702_LSP1_26M_WCLK] = - zx_gate("lsp1_26M_wclk", "lsp_26_wclk_mux", CLK_EN1, 31); - clk[ZX296702_LSP1_104M_WCLK] = - zx_gate("lsp1_104M_wclk", "pll_lsp_104M", CLK_EN1, 9); - clk[ZX296702_LSP1_32K_CLK] = - zx_gate("lsp1_32K_clk", "clk_32K768", CLK_EN1, 10); - clk[ZX296702_AON_HCLK] = - zx_gate("aon_hclk", "main_hclk", CLK_EN1, 11); - clk[ZX296702_SYS_CTRL_PCLK] = - zx_gate("sys_ctrl_pclk", "main_pclk", CLK_EN1, 12); - clk[ZX296702_DMA_PCLK] = - zx_gate("dma_pclk", "main_pclk", CLK_EN1, 13); - clk[ZX296702_DMA_ACLK] = - zx_gate("dma_aclk", "matrix_aclk", CLK_EN1, 14); - clk[ZX296702_SEC_HCLK] = - zx_gate("sec_hclk", "main_hclk", CLK_EN1, 15); - clk[ZX296702_AES_WCLK] = - zx_gate("aes_wclk", "sec_wclk_div", CLK_EN1, 16); - clk[ZX296702_DES_WCLK] = - zx_gate("des_wclk", "sec_wclk_div", CLK_EN1, 17); - clk[ZX296702_IRAM_ACLK] = - zx_gate("iram_aclk", "matrix_aclk", CLK_EN1, 18); - clk[ZX296702_IROM_ACLK] = - zx_gate("irom_aclk", "matrix_aclk", CLK_EN1, 19); - clk[ZX296702_BOOT_CTRL_HCLK] = - zx_gate("boot_ctrl_hclk", "main_hclk", CLK_EN1, 20); - clk[ZX296702_EFUSE_CLK_30] = - zx_gate("efuse_clk_30", "osc", CLK_EN1, 21); - - /* TODO: add VOU Local clocks */ - clk[ZX296702_VOU_MAIN_CHANNEL_DIV] = - zx_div("vou_main_channel_div", "vou_main_channel_wclk", - VOU_LOCAL_DIV2_SET, 1, 1); - clk[ZX296702_VOU_AUX_CHANNEL_DIV] = - zx_div("vou_aux_channel_div", "vou_aux_channel_wclk", - VOU_LOCAL_DIV2_SET, 0, 1); - clk[ZX296702_VOU_TV_ENC_HD_DIV] = - zx_div("vou_tv_enc_hd_div", "vou_tv_enc_hd_mux", - VOU_LOCAL_DIV2_SET, 3, 1); - clk[ZX296702_VOU_TV_ENC_SD_DIV] = - zx_div("vou_tv_enc_sd_div", "vou_tv_enc_sd_mux", - VOU_LOCAL_DIV2_SET, 2, 1); - clk[ZX296702_VL0_MUX] = - zx_mux("vl0_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 8, 1); - clk[ZX296702_VL1_MUX] = - zx_mux("vl1_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 9, 1); - clk[ZX296702_VL2_MUX] = - zx_mux("vl2_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 10, 1); - clk[ZX296702_GL0_MUX] = - zx_mux("gl0_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 5, 1); - clk[ZX296702_GL1_MUX] = - zx_mux("gl1_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 6, 1); - clk[ZX296702_GL2_MUX] = - zx_mux("gl2_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 7, 1); - clk[ZX296702_WB_MUX] = - zx_mux("wb_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 11, 1); - clk[ZX296702_HDMI_MUX] = - zx_mux("hdmi_mux", hdmi_sel, ARRAY_SIZE(hdmi_sel), - VOU_LOCAL_CLKSEL, 4, 1); - clk[ZX296702_VOU_TV_ENC_HD_MUX] = - zx_mux("vou_tv_enc_hd_mux", hdmi_sel, ARRAY_SIZE(hdmi_sel), - VOU_LOCAL_CLKSEL, 3, 1); - clk[ZX296702_VOU_TV_ENC_SD_MUX] = - zx_mux("vou_tv_enc_sd_mux", hdmi_sel, ARRAY_SIZE(hdmi_sel), - VOU_LOCAL_CLKSEL, 2, 1); - clk[ZX296702_VL0_CLK] = - zx_gate("vl0_clk", "vl0_mux", VOU_LOCAL_CLKEN, 8); - clk[ZX296702_VL1_CLK] = - zx_gate("vl1_clk", "vl1_mux", VOU_LOCAL_CLKEN, 9); - clk[ZX296702_VL2_CLK] = - zx_gate("vl2_clk", "vl2_mux", VOU_LOCAL_CLKEN, 10); - clk[ZX296702_GL0_CLK] = - zx_gate("gl0_clk", "gl0_mux", VOU_LOCAL_CLKEN, 5); - clk[ZX296702_GL1_CLK] = - zx_gate("gl1_clk", "gl1_mux", VOU_LOCAL_CLKEN, 6); - clk[ZX296702_GL2_CLK] = - zx_gate("gl2_clk", "gl2_mux", VOU_LOCAL_CLKEN, 7); - clk[ZX296702_WB_CLK] = - zx_gate("wb_clk", "wb_mux", VOU_LOCAL_CLKEN, 11); - clk[ZX296702_CL_CLK] = - zx_gate("cl_clk", "vou_main_channel_div", VOU_LOCAL_CLKEN, 12); - clk[ZX296702_MAIN_MIX_CLK] = - zx_gate("main_mix_clk", "vou_main_channel_div", - VOU_LOCAL_CLKEN, 4); - clk[ZX296702_AUX_MIX_CLK] = - zx_gate("aux_mix_clk", "vou_aux_channel_div", - VOU_LOCAL_CLKEN, 3); - clk[ZX296702_HDMI_CLK] = - zx_gate("hdmi_clk", "hdmi_mux", VOU_LOCAL_CLKEN, 2); - clk[ZX296702_VOU_TV_ENC_HD_DAC_CLK] = - zx_gate("vou_tv_enc_hd_dac_clk", "vou_tv_enc_hd_div", - VOU_LOCAL_CLKEN, 1); - clk[ZX296702_VOU_TV_ENC_SD_DAC_CLK] = - zx_gate("vou_tv_enc_sd_dac_clk", "vou_tv_enc_sd_div", - VOU_LOCAL_CLKEN, 0); - - /* CA9 PERIPHCLK = a9_wclk / 2 */ - clk[ZX296702_A9_PERIPHCLK] = - clk_register_fixed_factor(NULL, "a9_periphclk", "a9_wclk", - 0, 1, 2); - - for (i = 0; i < ARRAY_SIZE(topclk); i++) { - if (IS_ERR(clk[i])) { - pr_err("zx296702 clk %d: register failed with %ld\n", - i, PTR_ERR(clk[i])); - return; - } - } - - topclk_data.clks = topclk; - topclk_data.clk_num = ARRAY_SIZE(topclk); - of_clk_add_provider(np, of_clk_src_onecell_get, &topclk_data); -} -CLK_OF_DECLARE(zx296702_top_clk, "zte,zx296702-topcrm-clk", - zx296702_top_clocks_init); - -static void __init zx296702_lsp0_clocks_init(struct device_node *np) -{ - struct clk **clk = lsp0clk; - int i; - - lsp0crpm_base = of_iomap(np, 0); - WARN_ON(!lsp0crpm_base); - - /* SDMMC1 */ - clk[ZX296702_SDMMC1_WCLK_MUX] = - zx_mux("sdmmc1_wclk_mux", sdmmc1_wclk_sel, - ARRAY_SIZE(sdmmc1_wclk_sel), CLK_SDMMC1, 4, 1); - clk[ZX296702_SDMMC1_WCLK_DIV] = - zx_div("sdmmc1_wclk_div", "sdmmc1_wclk_mux", CLK_SDMMC1, 12, 4); - clk[ZX296702_SDMMC1_WCLK] = - zx_gate("sdmmc1_wclk", "sdmmc1_wclk_div", CLK_SDMMC1, 1); - clk[ZX296702_SDMMC1_PCLK] = - zx_gate("sdmmc1_pclk", "lsp0_apb_pclk", CLK_SDMMC1, 0); - - clk[ZX296702_GPIO_CLK] = - zx_gate("gpio_clk", "lsp0_apb_pclk", CLK_GPIO, 0); - - /* SPDIF */ - clk[ZX296702_SPDIF0_WCLK_MUX] = - zx_mux("spdif0_wclk_mux", spdif0_wclk_sel, - ARRAY_SIZE(spdif0_wclk_sel), CLK_SPDIF0, 4, 1); - clk[ZX296702_SPDIF0_WCLK] = - zx_gate("spdif0_wclk", "spdif0_wclk_mux", CLK_SPDIF0, 1); - clk[ZX296702_SPDIF0_PCLK] = - zx_gate("spdif0_pclk", "lsp0_apb_pclk", CLK_SPDIF0, 0); - - clk[ZX296702_SPDIF0_DIV] = - clk_register_zx_audio("spdif0_div", "spdif0_wclk", 0, - SPDIF0_DIV); - - /* I2S */ - clk[ZX296702_I2S0_WCLK_MUX] = - zx_mux("i2s0_wclk_mux", i2s_wclk_sel, - ARRAY_SIZE(i2s_wclk_sel), CLK_I2S0, 4, 1); - clk[ZX296702_I2S0_WCLK] = - zx_gate("i2s0_wclk", "i2s0_wclk_mux", CLK_I2S0, 1); - clk[ZX296702_I2S0_PCLK] = - zx_gate("i2s0_pclk", "lsp0_apb_pclk", CLK_I2S0, 0); - - clk[ZX296702_I2S0_DIV] = - clk_register_zx_audio("i2s0_div", "i2s0_wclk", 0, I2S0_DIV); - - clk[ZX296702_I2S1_WCLK_MUX] = - zx_mux("i2s1_wclk_mux", i2s_wclk_sel, - ARRAY_SIZE(i2s_wclk_sel), CLK_I2S1, 4, 1); - clk[ZX296702_I2S1_WCLK] = - zx_gate("i2s1_wclk", "i2s1_wclk_mux", CLK_I2S1, 1); - clk[ZX296702_I2S1_PCLK] = - zx_gate("i2s1_pclk", "lsp0_apb_pclk", CLK_I2S1, 0); - - clk[ZX296702_I2S1_DIV] = - clk_register_zx_audio("i2s1_div", "i2s1_wclk", 0, I2S1_DIV); - - clk[ZX296702_I2S2_WCLK_MUX] = - zx_mux("i2s2_wclk_mux", i2s_wclk_sel, - ARRAY_SIZE(i2s_wclk_sel), CLK_I2S2, 4, 1); - clk[ZX296702_I2S2_WCLK] = - zx_gate("i2s2_wclk", "i2s2_wclk_mux", CLK_I2S2, 1); - clk[ZX296702_I2S2_PCLK] = - zx_gate("i2s2_pclk", "lsp0_apb_pclk", CLK_I2S2, 0); - - clk[ZX296702_I2S2_DIV] = - clk_register_zx_audio("i2s2_div", "i2s2_wclk", 0, I2S2_DIV); - - for (i = 0; i < ARRAY_SIZE(lsp0clk); i++) { - if (IS_ERR(clk[i])) { - pr_err("zx296702 clk %d: register failed with %ld\n", - i, PTR_ERR(clk[i])); - return; - } - } - - lsp0clk_data.clks = lsp0clk; - lsp0clk_data.clk_num = ARRAY_SIZE(lsp0clk); - of_clk_add_provider(np, of_clk_src_onecell_get, &lsp0clk_data); -} -CLK_OF_DECLARE(zx296702_lsp0_clk, "zte,zx296702-lsp0crpm-clk", - zx296702_lsp0_clocks_init); - -static void __init zx296702_lsp1_clocks_init(struct device_node *np) -{ - struct clk **clk = lsp1clk; - int i; - - lsp1crpm_base = of_iomap(np, 0); - WARN_ON(!lsp1crpm_base); - - /* UART0 */ - clk[ZX296702_UART0_WCLK_MUX] = - zx_mux("uart0_wclk_mux", uart_wclk_sel, - ARRAY_SIZE(uart_wclk_sel), CLK_UART0, 4, 1); - /* FIXME: uart wclk enable bit is bit1 in. We hack it as reserved 31 for - * UART does not work after parent clk is disabled/enabled */ - clk[ZX296702_UART0_WCLK] = - zx_gate("uart0_wclk", "uart0_wclk_mux", CLK_UART0, 31); - clk[ZX296702_UART0_PCLK] = - zx_gate("uart0_pclk", "lsp1_apb_pclk", CLK_UART0, 0); - - /* UART1 */ - clk[ZX296702_UART1_WCLK_MUX] = - zx_mux("uart1_wclk_mux", uart_wclk_sel, - ARRAY_SIZE(uart_wclk_sel), CLK_UART1, 4, 1); - clk[ZX296702_UART1_WCLK] = - zx_gate("uart1_wclk", "uart1_wclk_mux", CLK_UART1, 1); - clk[ZX296702_UART1_PCLK] = - zx_gate("uart1_pclk", "lsp1_apb_pclk", CLK_UART1, 0); - - /* SDMMC0 */ - clk[ZX296702_SDMMC0_WCLK_MUX] = - zx_mux("sdmmc0_wclk_mux", sdmmc0_wclk_sel, - ARRAY_SIZE(sdmmc0_wclk_sel), CLK_SDMMC0, 4, 1); - clk[ZX296702_SDMMC0_WCLK_DIV] = - zx_div("sdmmc0_wclk_div", "sdmmc0_wclk_mux", CLK_SDMMC0, 12, 4); - clk[ZX296702_SDMMC0_WCLK] = - zx_gate("sdmmc0_wclk", "sdmmc0_wclk_div", CLK_SDMMC0, 1); - clk[ZX296702_SDMMC0_PCLK] = - zx_gate("sdmmc0_pclk", "lsp1_apb_pclk", CLK_SDMMC0, 0); - - clk[ZX296702_SPDIF1_WCLK_MUX] = - zx_mux("spdif1_wclk_mux", spdif1_wclk_sel, - ARRAY_SIZE(spdif1_wclk_sel), CLK_SPDIF1, 4, 1); - clk[ZX296702_SPDIF1_WCLK] = - zx_gate("spdif1_wclk", "spdif1_wclk_mux", CLK_SPDIF1, 1); - clk[ZX296702_SPDIF1_PCLK] = - zx_gate("spdif1_pclk", "lsp1_apb_pclk", CLK_SPDIF1, 0); - - clk[ZX296702_SPDIF1_DIV] = - clk_register_zx_audio("spdif1_div", "spdif1_wclk", 0, - SPDIF1_DIV); - - for (i = 0; i < ARRAY_SIZE(lsp1clk); i++) { - if (IS_ERR(clk[i])) { - pr_err("zx296702 clk %d: register failed with %ld\n", - i, PTR_ERR(clk[i])); - return; - } - } - - lsp1clk_data.clks = lsp1clk; - lsp1clk_data.clk_num = ARRAY_SIZE(lsp1clk); - of_clk_add_provider(np, of_clk_src_onecell_get, &lsp1clk_data); -} -CLK_OF_DECLARE(zx296702_lsp1_clk, "zte,zx296702-lsp1crpm-clk", - zx296702_lsp1_clocks_init); diff --git a/drivers/clk/zte/clk-zx296718.c b/drivers/clk/zte/clk-zx296718.c deleted file mode 100644 index dd7045bc48c1..000000000000 --- a/drivers/clk/zte/clk-zx296718.c +++ /dev/null @@ -1,1074 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2015 - 2016 ZTE Corporation. - * Copyright (C) 2016 Linaro Ltd. - */ -#include -#include -#include -#include -#include -#include - -#include -#include "clk.h" - -/* TOP CRM */ -#define TOP_CLK_MUX0 0x04 -#define TOP_CLK_MUX1 0x08 -#define TOP_CLK_MUX2 0x0c -#define TOP_CLK_MUX3 0x10 -#define TOP_CLK_MUX4 0x14 -#define TOP_CLK_MUX5 0x18 -#define TOP_CLK_MUX6 0x1c -#define TOP_CLK_MUX7 0x20 -#define TOP_CLK_MUX9 0x28 - - -#define TOP_CLK_GATE0 0x34 -#define TOP_CLK_GATE1 0x38 -#define TOP_CLK_GATE2 0x3c -#define TOP_CLK_GATE3 0x40 -#define TOP_CLK_GATE4 0x44 -#define TOP_CLK_GATE5 0x48 -#define TOP_CLK_GATE6 0x4c - -#define TOP_CLK_DIV0 0x58 - -#define PLL_CPU_REG 0x80 -#define PLL_VGA_REG 0xb0 -#define PLL_DDR_REG 0xa0 - -/* LSP0 CRM */ -#define LSP0_TIMER3_CLK 0x4 -#define LSP0_TIMER4_CLK 0x8 -#define LSP0_TIMER5_CLK 0xc -#define LSP0_UART3_CLK 0x10 -#define LSP0_UART1_CLK 0x14 -#define LSP0_UART2_CLK 0x18 -#define LSP0_SPIFC0_CLK 0x1c -#define LSP0_I2C4_CLK 0x20 -#define LSP0_I2C5_CLK 0x24 -#define LSP0_SSP0_CLK 0x28 -#define LSP0_SSP1_CLK 0x2c -#define LSP0_USIM0_CLK 0x30 -#define LSP0_GPIO_CLK 0x34 -#define LSP0_I2C3_CLK 0x38 - -/* LSP1 CRM */ -#define LSP1_UART4_CLK 0x08 -#define LSP1_UART5_CLK 0x0c -#define LSP1_PWM_CLK 0x10 -#define LSP1_I2C2_CLK 0x14 -#define LSP1_SSP2_CLK 0x1c -#define LSP1_SSP3_CLK 0x20 -#define LSP1_SSP4_CLK 0x24 -#define LSP1_USIM1_CLK 0x28 - -/* audio lsp */ -#define AUDIO_I2S0_DIV_CFG1 0x10 -#define AUDIO_I2S0_DIV_CFG2 0x14 -#define AUDIO_I2S0_CLK 0x18 -#define AUDIO_I2S1_DIV_CFG1 0x20 -#define AUDIO_I2S1_DIV_CFG2 0x24 -#define AUDIO_I2S1_CLK 0x28 -#define AUDIO_I2S2_DIV_CFG1 0x30 -#define AUDIO_I2S2_DIV_CFG2 0x34 -#define AUDIO_I2S2_CLK 0x38 -#define AUDIO_I2S3_DIV_CFG1 0x40 -#define AUDIO_I2S3_DIV_CFG2 0x44 -#define AUDIO_I2S3_CLK 0x48 -#define AUDIO_I2C0_CLK 0x50 -#define AUDIO_SPDIF0_DIV_CFG1 0x60 -#define AUDIO_SPDIF0_DIV_CFG2 0x64 -#define AUDIO_SPDIF0_CLK 0x68 -#define AUDIO_SPDIF1_DIV_CFG1 0x70 -#define AUDIO_SPDIF1_DIV_CFG2 0x74 -#define AUDIO_SPDIF1_CLK 0x78 -#define AUDIO_TIMER_CLK 0x80 -#define AUDIO_TDM_CLK 0x90 -#define AUDIO_TS_CLK 0xa0 - -static DEFINE_SPINLOCK(clk_lock); - -static const struct zx_pll_config pll_cpu_table[] = { - PLL_RATE(1312000000, 0x00103621, 0x04aaaaaa), - PLL_RATE(1407000000, 0x00103a21, 0x04aaaaaa), - PLL_RATE(1503000000, 0x00103e21, 0x04aaaaaa), - PLL_RATE(1600000000, 0x00104221, 0x04aaaaaa), -}; - -static const struct zx_pll_config pll_vga_table[] = { - PLL_RATE(36000000, 0x00102464, 0x04000000), /* 800x600@56 */ - PLL_RATE(40000000, 0x00102864, 0x04000000), /* 800x600@60 */ - PLL_RATE(49500000, 0x00103164, 0x04800000), /* 800x600@75 */ - PLL_RATE(50000000, 0x00103264, 0x04000000), /* 800x600@72 */ - PLL_RATE(56250000, 0x00103864, 0x04400000), /* 800x600@85 */ - PLL_RATE(65000000, 0x00104164, 0x04000000), /* 1024x768@60 */ - PLL_RATE(74375000, 0x00104a64, 0x04600000), /* 1280x720@60 */ - PLL_RATE(75000000, 0x00104b64, 0x04800000), /* 1024x768@70 */ - PLL_RATE(78750000, 0x00104e64, 0x04c00000), /* 1024x768@75 */ - PLL_RATE(85500000, 0x00105564, 0x04800000), /* 1360x768@60 */ - PLL_RATE(106500000, 0x00106a64, 0x04800000), /* 1440x900@60 */ - PLL_RATE(108000000, 0x00106c64, 0x04000000), /* 1280x1024@60 */ - PLL_RATE(110000000, 0x00106e64, 0x04000000), /* 1024x768@85 */ - PLL_RATE(135000000, 0x00105a44, 0x04000000), /* 1280x1024@75 */ - PLL_RATE(136750000, 0x00104462, 0x04600000), /* 1440x900@75 */ - PLL_RATE(148500000, 0x00104a62, 0x04400000), /* 1920x1080@60 */ - PLL_RATE(157000000, 0x00104e62, 0x04800000), /* 1440x900@85 */ - PLL_RATE(157500000, 0x00104e62, 0x04c00000), /* 1280x1024@85 */ - PLL_RATE(162000000, 0x00105162, 0x04000000), /* 1600x1200@60 */ - PLL_RATE(193250000, 0x00106062, 0x04a00000), /* 1920x1200@60 */ -}; - -PNAME(osc) = { - "osc24m", - "osc32k", -}; - -PNAME(dbg_wclk_p) = { - "clk334m", - "clk466m", - "clk396m", - "clk250m", -}; - -PNAME(a72_coreclk_p) = { - "osc24m", - "pll_mm0_1188m", - "pll_mm1_1296m", - "clk1000m", - "clk648m", - "clk1600m", - "pll_audio_1800m", - "pll_vga_1800m", -}; - -PNAME(cpu_periclk_p) = { - "osc24m", - "clk500m", - "clk594m", - "clk466m", - "clk294m", - "clk334m", - "clk250m", - "clk125m", -}; - -PNAME(a53_coreclk_p) = { - "osc24m", - "clk1000m", - "pll_mm0_1188m", - "clk648m", - "clk500m", - "clk800m", - "clk1600m", - "pll_audio_1800m", -}; - -PNAME(sec_wclk_p) = { - "osc24m", - "clk396m", - "clk334m", - "clk297m", - "clk250m", - "clk198m", - "clk148m5", - "clk99m", -}; - -PNAME(sd_nand_wclk_p) = { - "osc24m", - "clk49m5", - "clk99m", - "clk198m", - "clk167m", - "clk148m5", - "clk125m", - "clk216m", -}; - -PNAME(emmc_wclk_p) = { - "osc24m", - "clk198m", - "clk99m", - "clk396m", - "clk334m", - "clk297m", - "clk250m", - "clk148m5", -}; - -PNAME(clk32_p) = { - "osc32k", - "clk32k768", -}; - -PNAME(usb_ref24m_p) = { - "osc32k", - "clk32k768", -}; - -PNAME(sys_noc_alck_p) = { - "osc24m", - "clk250m", - "clk198m", - "clk148m5", - "clk108m", - "clk54m", - "clk216m", - "clk240m", -}; - -PNAME(vde_aclk_p) = { - "clk334m", - "clk594m", - "clk500m", - "clk432m", - "clk480m", - "clk297m", - "clk_vga", /*600MHz*/ - "clk294m", -}; - -PNAME(vce_aclk_p) = { - "clk334m", - "clk594m", - "clk500m", - "clk432m", - "clk396m", - "clk297m", - "clk_vga", /*600MHz*/ - "clk294m", -}; - -PNAME(hde_aclk_p) = { - "clk334m", - "clk594m", - "clk500m", - "clk432m", - "clk396m", - "clk297m", - "clk_vga", /*600MHz*/ - "clk294m", -}; - -PNAME(gpu_aclk_p) = { - "clk334m", - "clk648m", - "clk594m", - "clk500m", - "clk396m", - "clk297m", - "clk_vga", /*600MHz*/ - "clk294m", -}; - -PNAME(sappu_aclk_p) = { - "clk396m", - "clk500m", - "clk250m", - "clk148m5", -}; - -PNAME(sappu_wclk_p) = { - "clk198m", - "clk396m", - "clk334m", - "clk297m", - "clk250m", - "clk148m5", - "clk125m", - "clk99m", -}; - -PNAME(vou_aclk_p) = { - "clk334m", - "clk594m", - "clk500m", - "clk432m", - "clk396m", - "clk297m", - "clk_vga", /*600MHz*/ - "clk294m", -}; - -PNAME(vou_main_wclk_p) = { - "clk108m", - "clk594m", - "clk297m", - "clk148m5", - "clk74m25", - "clk54m", - "clk27m", - "clk_vga", -}; - -PNAME(vou_aux_wclk_p) = { - "clk108m", - "clk148m5", - "clk74m25", - "clk54m", - "clk27m", - "clk_vga", - "clk54m_mm0", - "clk" -}; - -PNAME(vou_ppu_wclk_p) = { - "clk334m", - "clk432m", - "clk396m", - "clk297m", - "clk250m", - "clk125m", - "clk198m", - "clk99m", -}; - -PNAME(vga_i2c_wclk_p) = { - "osc24m", - "clk99m", -}; - -PNAME(viu_m0_aclk_p) = { - "clk334m", - "clk432m", - "clk396m", - "clk297m", - "clk250m", - "clk125m", - "clk198m", - "osc24m", -}; - -PNAME(viu_m1_aclk_p) = { - "clk198m", - "clk250m", - "clk297m", - "clk125m", - "clk396m", - "clk334m", - "clk148m5", - "osc24m", -}; - -PNAME(viu_clk_p) = { - "clk198m", - "clk334m", - "clk297m", - "clk250m", - "clk396m", - "clk125m", - "clk99m", - "clk148m5", -}; - -PNAME(viu_jpeg_clk_p) = { - "clk334m", - "clk480m", - "clk432m", - "clk396m", - "clk297m", - "clk250m", - "clk125m", - "clk198m", -}; - -PNAME(ts_sys_clk_p) = { - "clk192m", - "clk167m", - "clk125m", - "clk99m", -}; - -PNAME(wdt_ares_p) = { - "osc24m", - "clk32k" -}; - -static struct clk_zx_pll zx296718_pll_clk[] = { - ZX296718_PLL("pll_cpu", "osc24m", PLL_CPU_REG, pll_cpu_table), - ZX296718_PLL("pll_vga", "osc24m", PLL_VGA_REG, pll_vga_table), -}; - -static struct zx_clk_fixed_factor top_ffactor_clk[] = { - FFACTOR(0, "clk4m", "osc24m", 1, 6, 0), - FFACTOR(0, "clk2m", "osc24m", 1, 12, 0), - /* pll cpu */ - FFACTOR(0, "clk1600m", "pll_cpu", 1, 1, CLK_SET_RATE_PARENT), - FFACTOR(0, "clk800m", "pll_cpu", 1, 2, CLK_SET_RATE_PARENT), - /* pll mac */ - FFACTOR(0, "clk25m", "pll_mac", 1, 40, 0), - FFACTOR(0, "clk125m", "pll_mac", 1, 8, 0), - FFACTOR(0, "clk250m", "pll_mac", 1, 4, 0), - FFACTOR(0, "clk50m", "pll_mac", 1, 20, 0), - FFACTOR(0, "clk500m", "pll_mac", 1, 2, 0), - FFACTOR(0, "clk1000m", "pll_mac", 1, 1, 0), - FFACTOR(0, "clk334m", "pll_mac", 1, 3, 0), - FFACTOR(0, "clk167m", "pll_mac", 1, 6, 0), - /* pll mm */ - FFACTOR(0, "clk54m_mm0", "pll_mm0", 1, 22, 0), - FFACTOR(0, "clk74m25", "pll_mm0", 1, 16, 0), - FFACTOR(0, "clk148m5", "pll_mm0", 1, 8, 0), - FFACTOR(0, "clk297m", "pll_mm0", 1, 4, 0), - FFACTOR(0, "clk594m", "pll_mm0", 1, 2, 0), - FFACTOR(0, "pll_mm0_1188m", "pll_mm0", 1, 1, 0), - FFACTOR(0, "clk396m", "pll_mm0", 1, 3, 0), - FFACTOR(0, "clk198m", "pll_mm0", 1, 6, 0), - FFACTOR(0, "clk99m", "pll_mm0", 1, 12, 0), - FFACTOR(0, "clk49m5", "pll_mm0", 1, 24, 0), - /* pll mm */ - FFACTOR(0, "clk324m", "pll_mm1", 1, 4, 0), - FFACTOR(0, "clk648m", "pll_mm1", 1, 2, 0), - FFACTOR(0, "pll_mm1_1296m", "pll_mm1", 1, 1, 0), - FFACTOR(0, "clk216m", "pll_mm1", 1, 6, 0), - FFACTOR(0, "clk432m", "pll_mm1", 1, 3, 0), - FFACTOR(0, "clk108m", "pll_mm1", 1, 12, 0), - FFACTOR(0, "clk72m", "pll_mm1", 1, 18, 0), - FFACTOR(0, "clk27m", "pll_mm1", 1, 48, 0), - FFACTOR(0, "clk54m", "pll_mm1", 1, 24, 0), - /* vga */ - FFACTOR(0, "pll_vga_1800m", "pll_vga", 1, 1, 0), - FFACTOR(0, "clk_vga", "pll_vga", 1, 1, CLK_SET_RATE_PARENT), - /* pll ddr */ - FFACTOR(0, "clk466m", "pll_ddr", 1, 2, 0), - - /* pll audio */ - FFACTOR(0, "pll_audio_1800m", "pll_audio", 1, 1, 0), - FFACTOR(0, "clk32k768", "pll_audio", 1, 27000, 0), - FFACTOR(0, "clk16m384", "pll_audio", 1, 54, 0), - FFACTOR(0, "clk294m", "pll_audio", 1, 3, 0), - - /* pll hsic*/ - FFACTOR(0, "clk240m", "pll_hsic", 1, 4, 0), - FFACTOR(0, "clk480m", "pll_hsic", 1, 2, 0), - FFACTOR(0, "clk192m", "pll_hsic", 1, 5, 0), - FFACTOR(0, "clk_pll_24m", "pll_hsic", 1, 40, 0), - FFACTOR(0, "emmc_mux_div2", "emmc_mux", 1, 2, CLK_SET_RATE_PARENT), -}; - -static const struct clk_div_table noc_div_table[] = { - { .val = 1, .div = 2, }, - { .val = 3, .div = 4, }, -}; -static struct zx_clk_div top_div_clk[] = { - DIV_T(0, "sys_noc_hclk", "sys_noc_aclk", TOP_CLK_DIV0, 0, 2, 0, noc_div_table), - DIV_T(0, "sys_noc_pclk", "sys_noc_aclk", TOP_CLK_DIV0, 4, 2, 0, noc_div_table), -}; - -static struct zx_clk_mux top_mux_clk[] = { - MUX(0, "dbg_mux", dbg_wclk_p, TOP_CLK_MUX0, 12, 2), - MUX(0, "a72_mux", a72_coreclk_p, TOP_CLK_MUX0, 8, 3), - MUX(0, "cpu_peri_mux", cpu_periclk_p, TOP_CLK_MUX0, 4, 3), - MUX_F(0, "a53_mux", a53_coreclk_p, TOP_CLK_MUX0, 0, 3, CLK_SET_RATE_PARENT, 0), - MUX(0, "sys_noc_aclk", sys_noc_alck_p, TOP_CLK_MUX1, 0, 3), - MUX(0, "sec_mux", sec_wclk_p, TOP_CLK_MUX2, 16, 3), - MUX(0, "sd1_mux", sd_nand_wclk_p, TOP_CLK_MUX2, 12, 3), - MUX(0, "sd0_mux", sd_nand_wclk_p, TOP_CLK_MUX2, 8, 3), - MUX(0, "emmc_mux", emmc_wclk_p, TOP_CLK_MUX2, 4, 3), - MUX(0, "nand_mux", sd_nand_wclk_p, TOP_CLK_MUX2, 0, 3), - MUX(0, "usb_ref24m_mux", usb_ref24m_p, TOP_CLK_MUX9, 16, 1), - MUX(0, "clk32k", clk32_p, TOP_CLK_MUX9, 12, 1), - MUX_F(0, "wdt_mux", wdt_ares_p, TOP_CLK_MUX9, 8, 1, CLK_SET_RATE_PARENT, 0), - MUX(0, "timer_mux", osc, TOP_CLK_MUX9, 4, 1), - MUX(0, "vde_mux", vde_aclk_p, TOP_CLK_MUX4, 0, 3), - MUX(0, "vce_mux", vce_aclk_p, TOP_CLK_MUX4, 4, 3), - MUX(0, "hde_mux", hde_aclk_p, TOP_CLK_MUX4, 8, 3), - MUX(0, "gpu_mux", gpu_aclk_p, TOP_CLK_MUX5, 0, 3), - MUX(0, "sappu_a_mux", sappu_aclk_p, TOP_CLK_MUX5, 4, 2), - MUX(0, "sappu_w_mux", sappu_wclk_p, TOP_CLK_MUX5, 8, 3), - MUX(0, "vou_a_mux", vou_aclk_p, TOP_CLK_MUX7, 0, 3), - MUX_F(0, "vou_main_w_mux", vou_main_wclk_p, TOP_CLK_MUX7, 4, 3, CLK_SET_RATE_PARENT, 0), - MUX_F(0, "vou_aux_w_mux", vou_aux_wclk_p, TOP_CLK_MUX7, 8, 3, CLK_SET_RATE_PARENT, 0), - MUX(0, "vou_ppu_w_mux", vou_ppu_wclk_p, TOP_CLK_MUX7, 12, 3), - MUX(0, "vga_i2c_mux", vga_i2c_wclk_p, TOP_CLK_MUX7, 16, 1), - MUX(0, "viu_m0_a_mux", viu_m0_aclk_p, TOP_CLK_MUX6, 0, 3), - MUX(0, "viu_m1_a_mux", viu_m1_aclk_p, TOP_CLK_MUX6, 4, 3), - MUX(0, "viu_w_mux", viu_clk_p, TOP_CLK_MUX6, 8, 3), - MUX(0, "viu_jpeg_w_mux", viu_jpeg_clk_p, TOP_CLK_MUX6, 12, 3), - MUX(0, "ts_sys_mux", ts_sys_clk_p, TOP_CLK_MUX6, 16, 2), -}; - -static struct zx_clk_gate top_gate_clk[] = { - GATE(CPU_DBG_GATE, "dbg_wclk", "dbg_mux", TOP_CLK_GATE0, 4, CLK_SET_RATE_PARENT, 0), - GATE(A72_GATE, "a72_coreclk", "a72_mux", TOP_CLK_GATE0, 3, CLK_SET_RATE_PARENT, 0), - GATE(CPU_PERI_GATE, "cpu_peri", "cpu_peri_mux", TOP_CLK_GATE0, 1, CLK_SET_RATE_PARENT, 0), - GATE(A53_GATE, "a53_coreclk", "a53_mux", TOP_CLK_GATE0, 0, CLK_SET_RATE_PARENT, 0), - GATE(SD1_WCLK, "sd1_wclk", "sd1_mux", TOP_CLK_GATE1, 13, CLK_SET_RATE_PARENT, 0), - GATE(SD0_WCLK, "sd0_wclk", "sd0_mux", TOP_CLK_GATE1, 9, CLK_SET_RATE_PARENT, 0), - GATE(EMMC_WCLK, "emmc_wclk", "emmc_mux_div2", TOP_CLK_GATE0, 5, CLK_SET_RATE_PARENT, 0), - GATE(EMMC_NAND_AXI, "emmc_nand_aclk", "sys_noc_aclk", TOP_CLK_GATE1, 4, CLK_SET_RATE_PARENT, 0), - GATE(NAND_WCLK, "nand_wclk", "nand_mux", TOP_CLK_GATE0, 1, CLK_SET_RATE_PARENT, 0), - GATE(EMMC_NAND_AHB, "emmc_nand_hclk", "sys_noc_hclk", TOP_CLK_GATE1, 0, CLK_SET_RATE_PARENT, 0), - GATE(0, "lsp1_pclk", "sys_noc_pclk", TOP_CLK_GATE2, 31, 0, 0), - GATE(LSP1_148M5, "lsp1_148m5", "clk148m5", TOP_CLK_GATE2, 30, 0, 0), - GATE(LSP1_99M, "lsp1_99m", "clk99m", TOP_CLK_GATE2, 29, 0, 0), - GATE(LSP1_24M, "lsp1_24m", "osc24m", TOP_CLK_GATE2, 28, 0, 0), - GATE(LSP0_74M25, "lsp0_74m25", "clk74m25", TOP_CLK_GATE2, 25, 0, 0), - GATE(0, "lsp0_pclk", "sys_noc_pclk", TOP_CLK_GATE2, 24, 0, 0), - GATE(LSP0_32K, "lsp0_32k", "osc32k", TOP_CLK_GATE2, 23, 0, 0), - GATE(LSP0_148M5, "lsp0_148m5", "clk148m5", TOP_CLK_GATE2, 22, 0, 0), - GATE(LSP0_99M, "lsp0_99m", "clk99m", TOP_CLK_GATE2, 21, 0, 0), - GATE(LSP0_24M, "lsp0_24m", "osc24m", TOP_CLK_GATE2, 20, 0, 0), - GATE(AUDIO_99M, "audio_99m", "clk99m", TOP_CLK_GATE5, 27, 0, 0), - GATE(AUDIO_24M, "audio_24m", "osc24m", TOP_CLK_GATE5, 28, 0, 0), - GATE(AUDIO_16M384, "audio_16m384", "clk16m384", TOP_CLK_GATE5, 29, 0, 0), - GATE(AUDIO_32K, "audio_32k", "clk32k", TOP_CLK_GATE5, 30, 0, 0), - GATE(WDT_WCLK, "wdt_wclk", "wdt_mux", TOP_CLK_GATE6, 9, CLK_SET_RATE_PARENT, 0), - GATE(TIMER_WCLK, "timer_wclk", "timer_mux", TOP_CLK_GATE6, 5, CLK_SET_RATE_PARENT, 0), - GATE(VDE_ACLK, "vde_aclk", "vde_mux", TOP_CLK_GATE3, 0, CLK_SET_RATE_PARENT, 0), - GATE(VCE_ACLK, "vce_aclk", "vce_mux", TOP_CLK_GATE3, 4, CLK_SET_RATE_PARENT, 0), - GATE(HDE_ACLK, "hde_aclk", "hde_mux", TOP_CLK_GATE3, 8, CLK_SET_RATE_PARENT, 0), - GATE(GPU_ACLK, "gpu_aclk", "gpu_mux", TOP_CLK_GATE3, 16, CLK_SET_RATE_PARENT, 0), - GATE(SAPPU_ACLK, "sappu_aclk", "sappu_a_mux", TOP_CLK_GATE3, 20, CLK_SET_RATE_PARENT, 0), - GATE(SAPPU_WCLK, "sappu_wclk", "sappu_w_mux", TOP_CLK_GATE3, 22, CLK_SET_RATE_PARENT, 0), - GATE(VOU_ACLK, "vou_aclk", "vou_a_mux", TOP_CLK_GATE4, 16, CLK_SET_RATE_PARENT, 0), - GATE(VOU_MAIN_WCLK, "vou_main_wclk", "vou_main_w_mux", TOP_CLK_GATE4, 18, CLK_SET_RATE_PARENT, 0), - GATE(VOU_AUX_WCLK, "vou_aux_wclk", "vou_aux_w_mux", TOP_CLK_GATE4, 19, CLK_SET_RATE_PARENT, 0), - GATE(VOU_PPU_WCLK, "vou_ppu_wclk", "vou_ppu_w_mux", TOP_CLK_GATE4, 20, CLK_SET_RATE_PARENT, 0), - GATE(MIPI_CFG_CLK, "mipi_cfg_clk", "osc24m", TOP_CLK_GATE4, 21, 0, 0), - GATE(VGA_I2C_WCLK, "vga_i2c_wclk", "vga_i2c_mux", TOP_CLK_GATE4, 23, CLK_SET_RATE_PARENT, 0), - GATE(MIPI_REF_CLK, "mipi_ref_clk", "clk27m", TOP_CLK_GATE4, 24, 0, 0), - GATE(HDMI_OSC_CEC, "hdmi_osc_cec", "clk2m", TOP_CLK_GATE4, 22, 0, 0), - GATE(HDMI_OSC_CLK, "hdmi_osc_clk", "clk240m", TOP_CLK_GATE4, 25, 0, 0), - GATE(HDMI_XCLK, "hdmi_xclk", "osc24m", TOP_CLK_GATE4, 26, 0, 0), - GATE(VIU_M0_ACLK, "viu_m0_aclk", "viu_m0_a_mux", TOP_CLK_GATE4, 0, CLK_SET_RATE_PARENT, 0), - GATE(VIU_M1_ACLK, "viu_m1_aclk", "viu_m1_a_mux", TOP_CLK_GATE4, 1, CLK_SET_RATE_PARENT, 0), - GATE(VIU_WCLK, "viu_wclk", "viu_w_mux", TOP_CLK_GATE4, 2, CLK_SET_RATE_PARENT, 0), - GATE(VIU_JPEG_WCLK, "viu_jpeg_wclk", "viu_jpeg_w_mux", TOP_CLK_GATE4, 3, CLK_SET_RATE_PARENT, 0), - GATE(VIU_CFG_CLK, "viu_cfg_clk", "osc24m", TOP_CLK_GATE4, 6, 0, 0), - GATE(TS_SYS_WCLK, "ts_sys_wclk", "ts_sys_mux", TOP_CLK_GATE5, 2, CLK_SET_RATE_PARENT, 0), - GATE(TS_SYS_108M, "ts_sys_108m", "clk108m", TOP_CLK_GATE5, 3, 0, 0), - GATE(USB20_HCLK, "usb20_hclk", "sys_noc_hclk", TOP_CLK_GATE2, 12, 0, 0), - GATE(USB20_PHY_CLK, "usb20_phy_clk", "usb_ref24m_mux", TOP_CLK_GATE2, 13, 0, 0), - GATE(USB21_HCLK, "usb21_hclk", "sys_noc_hclk", TOP_CLK_GATE2, 14, 0, 0), - GATE(USB21_PHY_CLK, "usb21_phy_clk", "usb_ref24m_mux", TOP_CLK_GATE2, 15, 0, 0), - GATE(GMAC_RMIICLK, "gmac_rmii_clk", "clk50m", TOP_CLK_GATE2, 3, 0, 0), - GATE(GMAC_PCLK, "gmac_pclk", "clk198m", TOP_CLK_GATE2, 1, 0, 0), - GATE(GMAC_ACLK, "gmac_aclk", "clk49m5", TOP_CLK_GATE2, 0, 0, 0), - GATE(GMAC_RFCLK, "gmac_refclk", "clk25m", TOP_CLK_GATE2, 4, 0, 0), - GATE(SD1_AHB, "sd1_hclk", "sys_noc_hclk", TOP_CLK_GATE1, 12, 0, 0), - GATE(SD0_AHB, "sd0_hclk", "sys_noc_hclk", TOP_CLK_GATE1, 8, 0, 0), - GATE(TEMPSENSOR_GATE, "tempsensor_gate", "clk4m", TOP_CLK_GATE5, 31, 0, 0), -}; - -static struct clk_hw_onecell_data top_hw_onecell_data = { - .num = TOP_NR_CLKS, - .hws = { - [TOP_NR_CLKS - 1] = NULL, - }, -}; - -static int __init top_clocks_init(struct device_node *np) -{ - void __iomem *reg_base; - int i, ret; - const char *name; - - reg_base = of_iomap(np, 0); - if (!reg_base) { - pr_err("%s: Unable to map clk base\n", __func__); - return -ENXIO; - } - - for (i = 0; i < ARRAY_SIZE(zx296718_pll_clk); i++) { - zx296718_pll_clk[i].reg_base += (uintptr_t)reg_base; - name = zx296718_pll_clk[i].hw.init->name; - ret = clk_hw_register(NULL, &zx296718_pll_clk[i].hw); - if (ret) - pr_warn("top clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(top_ffactor_clk); i++) { - if (top_ffactor_clk[i].id) - top_hw_onecell_data.hws[top_ffactor_clk[i].id] = - &top_ffactor_clk[i].factor.hw; - - name = top_ffactor_clk[i].factor.hw.init->name; - ret = clk_hw_register(NULL, &top_ffactor_clk[i].factor.hw); - if (ret) - pr_warn("top clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(top_mux_clk); i++) { - if (top_mux_clk[i].id) - top_hw_onecell_data.hws[top_mux_clk[i].id] = - &top_mux_clk[i].mux.hw; - - top_mux_clk[i].mux.reg += (uintptr_t)reg_base; - name = top_mux_clk[i].mux.hw.init->name; - ret = clk_hw_register(NULL, &top_mux_clk[i].mux.hw); - if (ret) - pr_warn("top clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(top_gate_clk); i++) { - if (top_gate_clk[i].id) - top_hw_onecell_data.hws[top_gate_clk[i].id] = - &top_gate_clk[i].gate.hw; - - top_gate_clk[i].gate.reg += (uintptr_t)reg_base; - name = top_gate_clk[i].gate.hw.init->name; - ret = clk_hw_register(NULL, &top_gate_clk[i].gate.hw); - if (ret) - pr_warn("top clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(top_div_clk); i++) { - if (top_div_clk[i].id) - top_hw_onecell_data.hws[top_div_clk[i].id] = - &top_div_clk[i].div.hw; - - top_div_clk[i].div.reg += (uintptr_t)reg_base; - name = top_div_clk[i].div.hw.init->name; - ret = clk_hw_register(NULL, &top_div_clk[i].div.hw); - if (ret) - pr_warn("top clk %s init error!\n", name); - } - - ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, - &top_hw_onecell_data); - if (ret) { - pr_err("failed to register top clk provider: %d\n", ret); - return ret; - } - - return 0; -} - -static const struct clk_div_table common_even_div_table[] = { - { .val = 0, .div = 1, }, - { .val = 1, .div = 2, }, - { .val = 3, .div = 4, }, - { .val = 5, .div = 6, }, - { .val = 7, .div = 8, }, - { .val = 9, .div = 10, }, - { .val = 11, .div = 12, }, - { .val = 13, .div = 14, }, - { .val = 15, .div = 16, }, -}; - -static const struct clk_div_table common_div_table[] = { - { .val = 0, .div = 1, }, - { .val = 1, .div = 2, }, - { .val = 2, .div = 3, }, - { .val = 3, .div = 4, }, - { .val = 4, .div = 5, }, - { .val = 5, .div = 6, }, - { .val = 6, .div = 7, }, - { .val = 7, .div = 8, }, - { .val = 8, .div = 9, }, - { .val = 9, .div = 10, }, - { .val = 10, .div = 11, }, - { .val = 11, .div = 12, }, - { .val = 12, .div = 13, }, - { .val = 13, .div = 14, }, - { .val = 14, .div = 15, }, - { .val = 15, .div = 16, }, -}; - -PNAME(lsp0_wclk_common_p) = { - "lsp0_24m", - "lsp0_99m", -}; - -PNAME(lsp0_wclk_timer3_p) = { - "timer3_div", - "lsp0_32k" -}; - -PNAME(lsp0_wclk_timer4_p) = { - "timer4_div", - "lsp0_32k" -}; - -PNAME(lsp0_wclk_timer5_p) = { - "timer5_div", - "lsp0_32k" -}; - -PNAME(lsp0_wclk_spifc0_p) = { - "lsp0_148m5", - "lsp0_24m", - "lsp0_99m", - "lsp0_74m25" -}; - -PNAME(lsp0_wclk_ssp_p) = { - "lsp0_148m5", - "lsp0_99m", - "lsp0_24m", -}; - -static struct zx_clk_mux lsp0_mux_clk[] = { - MUX(0, "timer3_wclk_mux", lsp0_wclk_timer3_p, LSP0_TIMER3_CLK, 4, 1), - MUX(0, "timer4_wclk_mux", lsp0_wclk_timer4_p, LSP0_TIMER4_CLK, 4, 1), - MUX(0, "timer5_wclk_mux", lsp0_wclk_timer5_p, LSP0_TIMER5_CLK, 4, 1), - MUX(0, "uart3_wclk_mux", lsp0_wclk_common_p, LSP0_UART3_CLK, 4, 1), - MUX(0, "uart1_wclk_mux", lsp0_wclk_common_p, LSP0_UART1_CLK, 4, 1), - MUX(0, "uart2_wclk_mux", lsp0_wclk_common_p, LSP0_UART2_CLK, 4, 1), - MUX(0, "spifc0_wclk_mux", lsp0_wclk_spifc0_p, LSP0_SPIFC0_CLK, 4, 2), - MUX(0, "i2c4_wclk_mux", lsp0_wclk_common_p, LSP0_I2C4_CLK, 4, 1), - MUX(0, "i2c5_wclk_mux", lsp0_wclk_common_p, LSP0_I2C5_CLK, 4, 1), - MUX(0, "ssp0_wclk_mux", lsp0_wclk_ssp_p, LSP0_SSP0_CLK, 4, 1), - MUX(0, "ssp1_wclk_mux", lsp0_wclk_ssp_p, LSP0_SSP1_CLK, 4, 1), - MUX(0, "i2c3_wclk_mux", lsp0_wclk_common_p, LSP0_I2C3_CLK, 4, 1), -}; - -static struct zx_clk_gate lsp0_gate_clk[] = { - GATE(LSP0_TIMER3_WCLK, "timer3_wclk", "timer3_wclk_mux", LSP0_TIMER3_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_TIMER4_WCLK, "timer4_wclk", "timer4_wclk_mux", LSP0_TIMER4_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_TIMER5_WCLK, "timer5_wclk", "timer5_wclk_mux", LSP0_TIMER5_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_UART3_WCLK, "uart3_wclk", "uart3_wclk_mux", LSP0_UART3_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_UART1_WCLK, "uart1_wclk", "uart1_wclk_mux", LSP0_UART1_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_UART2_WCLK, "uart2_wclk", "uart2_wclk_mux", LSP0_UART2_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_SPIFC0_WCLK, "spifc0_wclk", "spifc0_wclk_mux", LSP0_SPIFC0_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_I2C4_WCLK, "i2c4_wclk", "i2c4_wclk_mux", LSP0_I2C4_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_I2C5_WCLK, "i2c5_wclk", "i2c5_wclk_mux", LSP0_I2C5_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_SSP0_WCLK, "ssp0_wclk", "ssp0_div", LSP0_SSP0_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_SSP1_WCLK, "ssp1_wclk", "ssp1_div", LSP0_SSP1_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_I2C3_WCLK, "i2c3_wclk", "i2c3_wclk_mux", LSP0_I2C3_CLK, 1, CLK_SET_RATE_PARENT, 0), -}; - -static struct zx_clk_div lsp0_div_clk[] = { - DIV_T(0, "timer3_div", "lsp0_24m", LSP0_TIMER3_CLK, 12, 4, 0, common_even_div_table), - DIV_T(0, "timer4_div", "lsp0_24m", LSP0_TIMER4_CLK, 12, 4, 0, common_even_div_table), - DIV_T(0, "timer5_div", "lsp0_24m", LSP0_TIMER5_CLK, 12, 4, 0, common_even_div_table), - DIV_T(0, "ssp0_div", "ssp0_wclk_mux", LSP0_SSP0_CLK, 12, 4, 0, common_even_div_table), - DIV_T(0, "ssp1_div", "ssp1_wclk_mux", LSP0_SSP1_CLK, 12, 4, 0, common_even_div_table), -}; - -static struct clk_hw_onecell_data lsp0_hw_onecell_data = { - .num = LSP0_NR_CLKS, - .hws = { - [LSP0_NR_CLKS - 1] = NULL, - }, -}; - -static int __init lsp0_clocks_init(struct device_node *np) -{ - void __iomem *reg_base; - int i, ret; - const char *name; - - reg_base = of_iomap(np, 0); - if (!reg_base) { - pr_err("%s: Unable to map clk base\n", __func__); - return -ENXIO; - } - - for (i = 0; i < ARRAY_SIZE(lsp0_mux_clk); i++) { - if (lsp0_mux_clk[i].id) - lsp0_hw_onecell_data.hws[lsp0_mux_clk[i].id] = - &lsp0_mux_clk[i].mux.hw; - - lsp0_mux_clk[i].mux.reg += (uintptr_t)reg_base; - name = lsp0_mux_clk[i].mux.hw.init->name; - ret = clk_hw_register(NULL, &lsp0_mux_clk[i].mux.hw); - if (ret) - pr_warn("lsp0 clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(lsp0_gate_clk); i++) { - if (lsp0_gate_clk[i].id) - lsp0_hw_onecell_data.hws[lsp0_gate_clk[i].id] = - &lsp0_gate_clk[i].gate.hw; - - lsp0_gate_clk[i].gate.reg += (uintptr_t)reg_base; - name = lsp0_gate_clk[i].gate.hw.init->name; - ret = clk_hw_register(NULL, &lsp0_gate_clk[i].gate.hw); - if (ret) - pr_warn("lsp0 clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(lsp0_div_clk); i++) { - if (lsp0_div_clk[i].id) - lsp0_hw_onecell_data.hws[lsp0_div_clk[i].id] = - &lsp0_div_clk[i].div.hw; - - lsp0_div_clk[i].div.reg += (uintptr_t)reg_base; - name = lsp0_div_clk[i].div.hw.init->name; - ret = clk_hw_register(NULL, &lsp0_div_clk[i].div.hw); - if (ret) - pr_warn("lsp0 clk %s init error!\n", name); - } - - ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, - &lsp0_hw_onecell_data); - if (ret) { - pr_err("failed to register lsp0 clk provider: %d\n", ret); - return ret; - } - - return 0; -} - -PNAME(lsp1_wclk_common_p) = { - "lsp1_24m", - "lsp1_99m", -}; - -PNAME(lsp1_wclk_ssp_p) = { - "lsp1_148m5", - "lsp1_99m", - "lsp1_24m", -}; - -static struct zx_clk_mux lsp1_mux_clk[] = { - MUX(0, "uart4_wclk_mux", lsp1_wclk_common_p, LSP1_UART4_CLK, 4, 1), - MUX(0, "uart5_wclk_mux", lsp1_wclk_common_p, LSP1_UART5_CLK, 4, 1), - MUX(0, "pwm_wclk_mux", lsp1_wclk_common_p, LSP1_PWM_CLK, 4, 1), - MUX(0, "i2c2_wclk_mux", lsp1_wclk_common_p, LSP1_I2C2_CLK, 4, 1), - MUX(0, "ssp2_wclk_mux", lsp1_wclk_ssp_p, LSP1_SSP2_CLK, 4, 2), - MUX(0, "ssp3_wclk_mux", lsp1_wclk_ssp_p, LSP1_SSP3_CLK, 4, 2), - MUX(0, "ssp4_wclk_mux", lsp1_wclk_ssp_p, LSP1_SSP4_CLK, 4, 2), - MUX(0, "usim1_wclk_mux", lsp1_wclk_common_p, LSP1_USIM1_CLK, 4, 1), -}; - -static struct zx_clk_div lsp1_div_clk[] = { - DIV_T(0, "pwm_div", "pwm_wclk_mux", LSP1_PWM_CLK, 12, 4, CLK_SET_RATE_PARENT, common_div_table), - DIV_T(0, "ssp2_div", "ssp2_wclk_mux", LSP1_SSP2_CLK, 12, 4, CLK_SET_RATE_PARENT, common_even_div_table), - DIV_T(0, "ssp3_div", "ssp3_wclk_mux", LSP1_SSP3_CLK, 12, 4, CLK_SET_RATE_PARENT, common_even_div_table), - DIV_T(0, "ssp4_div", "ssp4_wclk_mux", LSP1_SSP4_CLK, 12, 4, CLK_SET_RATE_PARENT, common_even_div_table), -}; - -static struct zx_clk_gate lsp1_gate_clk[] = { - GATE(LSP1_UART4_WCLK, "lsp1_uart4_wclk", "uart4_wclk_mux", LSP1_UART4_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_UART5_WCLK, "lsp1_uart5_wclk", "uart5_wclk_mux", LSP1_UART5_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_PWM_WCLK, "lsp1_pwm_wclk", "pwm_div", LSP1_PWM_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_PWM_PCLK, "lsp1_pwm_pclk", "lsp1_pclk", LSP1_PWM_CLK, 0, 0, 0), - GATE(LSP1_I2C2_WCLK, "lsp1_i2c2_wclk", "i2c2_wclk_mux", LSP1_I2C2_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_SSP2_WCLK, "lsp1_ssp2_wclk", "ssp2_div", LSP1_SSP2_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_SSP3_WCLK, "lsp1_ssp3_wclk", "ssp3_div", LSP1_SSP3_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_SSP4_WCLK, "lsp1_ssp4_wclk", "ssp4_div", LSP1_SSP4_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_USIM1_WCLK, "lsp1_usim1_wclk", "usim1_wclk_mux", LSP1_USIM1_CLK, 1, CLK_SET_RATE_PARENT, 0), -}; - -static struct clk_hw_onecell_data lsp1_hw_onecell_data = { - .num = LSP1_NR_CLKS, - .hws = { - [LSP1_NR_CLKS - 1] = NULL, - }, -}; - -static int __init lsp1_clocks_init(struct device_node *np) -{ - void __iomem *reg_base; - int i, ret; - const char *name; - - reg_base = of_iomap(np, 0); - if (!reg_base) { - pr_err("%s: Unable to map clk base\n", __func__); - return -ENXIO; - } - - for (i = 0; i < ARRAY_SIZE(lsp1_mux_clk); i++) { - if (lsp1_mux_clk[i].id) - lsp1_hw_onecell_data.hws[lsp1_mux_clk[i].id] = - &lsp0_mux_clk[i].mux.hw; - - lsp1_mux_clk[i].mux.reg += (uintptr_t)reg_base; - name = lsp1_mux_clk[i].mux.hw.init->name; - ret = clk_hw_register(NULL, &lsp1_mux_clk[i].mux.hw); - if (ret) - pr_warn("lsp1 clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(lsp1_gate_clk); i++) { - if (lsp1_gate_clk[i].id) - lsp1_hw_onecell_data.hws[lsp1_gate_clk[i].id] = - &lsp1_gate_clk[i].gate.hw; - - lsp1_gate_clk[i].gate.reg += (uintptr_t)reg_base; - name = lsp1_gate_clk[i].gate.hw.init->name; - ret = clk_hw_register(NULL, &lsp1_gate_clk[i].gate.hw); - if (ret) - pr_warn("lsp1 clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(lsp1_div_clk); i++) { - if (lsp1_div_clk[i].id) - lsp1_hw_onecell_data.hws[lsp1_div_clk[i].id] = - &lsp1_div_clk[i].div.hw; - - lsp1_div_clk[i].div.reg += (uintptr_t)reg_base; - name = lsp1_div_clk[i].div.hw.init->name; - ret = clk_hw_register(NULL, &lsp1_div_clk[i].div.hw); - if (ret) - pr_warn("lsp1 clk %s init error!\n", name); - } - - ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, - &lsp1_hw_onecell_data); - if (ret) { - pr_err("failed to register lsp1 clk provider: %d\n", ret); - return ret; - } - - return 0; -} - -PNAME(audio_wclk_common_p) = { - "audio_99m", - "audio_24m", -}; - -PNAME(audio_timer_p) = { - "audio_24m", - "audio_32k", -}; - -static struct zx_clk_mux audio_mux_clk[] = { - MUX(I2S0_WCLK_MUX, "i2s0_wclk_mux", audio_wclk_common_p, AUDIO_I2S0_CLK, 0, 1), - MUX(I2S1_WCLK_MUX, "i2s1_wclk_mux", audio_wclk_common_p, AUDIO_I2S1_CLK, 0, 1), - MUX(I2S2_WCLK_MUX, "i2s2_wclk_mux", audio_wclk_common_p, AUDIO_I2S2_CLK, 0, 1), - MUX(I2S3_WCLK_MUX, "i2s3_wclk_mux", audio_wclk_common_p, AUDIO_I2S3_CLK, 0, 1), - MUX(0, "i2c0_wclk_mux", audio_wclk_common_p, AUDIO_I2C0_CLK, 0, 1), - MUX(0, "spdif0_wclk_mux", audio_wclk_common_p, AUDIO_SPDIF0_CLK, 0, 1), - MUX(0, "spdif1_wclk_mux", audio_wclk_common_p, AUDIO_SPDIF1_CLK, 0, 1), - MUX(0, "timer_wclk_mux", audio_timer_p, AUDIO_TIMER_CLK, 0, 1), -}; - -static struct clk_zx_audio_divider audio_adiv_clk[] = { - AUDIO_DIV(0, "i2s0_wclk_div", "i2s0_wclk_mux", AUDIO_I2S0_DIV_CFG1), - AUDIO_DIV(0, "i2s1_wclk_div", "i2s1_wclk_mux", AUDIO_I2S1_DIV_CFG1), - AUDIO_DIV(0, "i2s2_wclk_div", "i2s2_wclk_mux", AUDIO_I2S2_DIV_CFG1), - AUDIO_DIV(0, "i2s3_wclk_div", "i2s3_wclk_mux", AUDIO_I2S3_DIV_CFG1), - AUDIO_DIV(0, "spdif0_wclk_div", "spdif0_wclk_mux", AUDIO_SPDIF0_DIV_CFG1), - AUDIO_DIV(0, "spdif1_wclk_div", "spdif1_wclk_mux", AUDIO_SPDIF1_DIV_CFG1), -}; - -static struct zx_clk_div audio_div_clk[] = { - DIV_T(0, "tdm_wclk_div", "audio_16m384", AUDIO_TDM_CLK, 8, 4, 0, common_div_table), -}; - -static struct zx_clk_gate audio_gate_clk[] = { - GATE(AUDIO_I2S0_WCLK, "i2s0_wclk", "i2s0_wclk_div", AUDIO_I2S0_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_I2S1_WCLK, "i2s1_wclk", "i2s1_wclk_div", AUDIO_I2S1_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_I2S2_WCLK, "i2s2_wclk", "i2s2_wclk_div", AUDIO_I2S2_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_I2S3_WCLK, "i2s3_wclk", "i2s3_wclk_div", AUDIO_I2S3_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_I2S0_PCLK, "i2s0_pclk", "clk49m5", AUDIO_I2S0_CLK, 8, 0, 0), - GATE(AUDIO_I2S1_PCLK, "i2s1_pclk", "clk49m5", AUDIO_I2S1_CLK, 8, 0, 0), - GATE(AUDIO_I2S2_PCLK, "i2s2_pclk", "clk49m5", AUDIO_I2S2_CLK, 8, 0, 0), - GATE(AUDIO_I2S3_PCLK, "i2s3_pclk", "clk49m5", AUDIO_I2S3_CLK, 8, 0, 0), - GATE(AUDIO_I2C0_WCLK, "i2c0_wclk", "i2c0_wclk_mux", AUDIO_I2C0_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_SPDIF0_WCLK, "spdif0_wclk", "spdif0_wclk_div", AUDIO_SPDIF0_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_SPDIF1_WCLK, "spdif1_wclk", "spdif1_wclk_div", AUDIO_SPDIF1_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_TDM_WCLK, "tdm_wclk", "tdm_wclk_div", AUDIO_TDM_CLK, 17, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_TS_PCLK, "tempsensor_pclk", "clk49m5", AUDIO_TS_CLK, 1, 0, 0), -}; - -static struct clk_hw_onecell_data audio_hw_onecell_data = { - .num = AUDIO_NR_CLKS, - .hws = { - [AUDIO_NR_CLKS - 1] = NULL, - }, -}; - -static int __init audio_clocks_init(struct device_node *np) -{ - void __iomem *reg_base; - int i, ret; - const char *name; - - reg_base = of_iomap(np, 0); - if (!reg_base) { - pr_err("%s: Unable to map audio clk base\n", __func__); - return -ENXIO; - } - - for (i = 0; i < ARRAY_SIZE(audio_mux_clk); i++) { - if (audio_mux_clk[i].id) - audio_hw_onecell_data.hws[audio_mux_clk[i].id] = - &audio_mux_clk[i].mux.hw; - - audio_mux_clk[i].mux.reg += (uintptr_t)reg_base; - name = audio_mux_clk[i].mux.hw.init->name; - ret = clk_hw_register(NULL, &audio_mux_clk[i].mux.hw); - if (ret) - pr_warn("audio clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(audio_adiv_clk); i++) { - if (audio_adiv_clk[i].id) - audio_hw_onecell_data.hws[audio_adiv_clk[i].id] = - &audio_adiv_clk[i].hw; - - audio_adiv_clk[i].reg_base += (uintptr_t)reg_base; - name = audio_adiv_clk[i].hw.init->name; - ret = clk_hw_register(NULL, &audio_adiv_clk[i].hw); - if (ret) - pr_warn("audio clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(audio_div_clk); i++) { - if (audio_div_clk[i].id) - audio_hw_onecell_data.hws[audio_div_clk[i].id] = - &audio_div_clk[i].div.hw; - - audio_div_clk[i].div.reg += (uintptr_t)reg_base; - name = audio_div_clk[i].div.hw.init->name; - ret = clk_hw_register(NULL, &audio_div_clk[i].div.hw); - if (ret) - pr_warn("audio clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(audio_gate_clk); i++) { - if (audio_gate_clk[i].id) - audio_hw_onecell_data.hws[audio_gate_clk[i].id] = - &audio_gate_clk[i].gate.hw; - - audio_gate_clk[i].gate.reg += (uintptr_t)reg_base; - name = audio_gate_clk[i].gate.hw.init->name; - ret = clk_hw_register(NULL, &audio_gate_clk[i].gate.hw); - if (ret) - pr_warn("audio clk %s init error!\n", name); - } - - ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, - &audio_hw_onecell_data); - if (ret) { - pr_err("failed to register audio clk provider: %d\n", ret); - return ret; - } - - return 0; -} - -static const struct of_device_id zx_clkc_match_table[] = { - { .compatible = "zte,zx296718-topcrm", .data = &top_clocks_init }, - { .compatible = "zte,zx296718-lsp0crm", .data = &lsp0_clocks_init }, - { .compatible = "zte,zx296718-lsp1crm", .data = &lsp1_clocks_init }, - { .compatible = "zte,zx296718-audiocrm", .data = &audio_clocks_init }, - { } -}; - -static int zx_clkc_probe(struct platform_device *pdev) -{ - int (*init_fn)(struct device_node *np); - struct device_node *np = pdev->dev.of_node; - - init_fn = of_device_get_match_data(&pdev->dev); - if (!init_fn) { - dev_err(&pdev->dev, "Error: No device match found\n"); - return -ENODEV; - } - - return init_fn(np); -} - -static struct platform_driver zx_clk_driver = { - .probe = zx_clkc_probe, - .driver = { - .name = "zx296718-clkc", - .of_match_table = zx_clkc_match_table, - }, -}; - -static int __init zx_clk_init(void) -{ - return platform_driver_register(&zx_clk_driver); -} -core_initcall(zx_clk_init); diff --git a/drivers/clk/zte/clk.c b/drivers/clk/zte/clk.c deleted file mode 100644 index 8bda6d41ad3a..000000000000 --- a/drivers/clk/zte/clk.c +++ /dev/null @@ -1,446 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2014 Linaro Ltd. - * Copyright (C) 2014 ZTE Corporation. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "clk.h" - -#define to_clk_zx_pll(_hw) container_of(_hw, struct clk_zx_pll, hw) -#define to_clk_zx_audio(_hw) container_of(_hw, struct clk_zx_audio, hw) - -#define CFG0_CFG1_OFFSET 4 -#define LOCK_FLAG 30 -#define POWER_DOWN 31 - -static int rate_to_idx(struct clk_zx_pll *zx_pll, unsigned long rate) -{ - const struct zx_pll_config *config = zx_pll->lookup_table; - int i; - - for (i = 0; i < zx_pll->count; i++) { - if (config[i].rate > rate) - return i > 0 ? i - 1 : 0; - - if (config[i].rate == rate) - return i; - } - - return i - 1; -} - -static int hw_to_idx(struct clk_zx_pll *zx_pll) -{ - const struct zx_pll_config *config = zx_pll->lookup_table; - u32 hw_cfg0, hw_cfg1; - int i; - - hw_cfg0 = readl_relaxed(zx_pll->reg_base); - hw_cfg1 = readl_relaxed(zx_pll->reg_base + CFG0_CFG1_OFFSET); - - /* For matching the value in lookup table */ - hw_cfg0 &= ~BIT(zx_pll->lock_bit); - - /* Check availability of pd_bit */ - if (zx_pll->pd_bit < 32) - hw_cfg0 |= BIT(zx_pll->pd_bit); - - for (i = 0; i < zx_pll->count; i++) { - if (hw_cfg0 == config[i].cfg0 && hw_cfg1 == config[i].cfg1) - return i; - } - - return -EINVAL; -} - -static unsigned long zx_pll_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - int idx; - - idx = hw_to_idx(zx_pll); - if (unlikely(idx == -EINVAL)) - return 0; - - return zx_pll->lookup_table[idx].rate; -} - -static long zx_pll_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - int idx; - - idx = rate_to_idx(zx_pll, rate); - - return zx_pll->lookup_table[idx].rate; -} - -static int zx_pll_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - /* Assume current cpu is not running on current PLL */ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - const struct zx_pll_config *config; - int idx; - - idx = rate_to_idx(zx_pll, rate); - config = &zx_pll->lookup_table[idx]; - - writel_relaxed(config->cfg0, zx_pll->reg_base); - writel_relaxed(config->cfg1, zx_pll->reg_base + CFG0_CFG1_OFFSET); - - return 0; -} - -static int zx_pll_enable(struct clk_hw *hw) -{ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - u32 reg; - - /* If pd_bit is not available, simply return success. */ - if (zx_pll->pd_bit > 31) - return 0; - - reg = readl_relaxed(zx_pll->reg_base); - writel_relaxed(reg & ~BIT(zx_pll->pd_bit), zx_pll->reg_base); - - return readl_relaxed_poll_timeout(zx_pll->reg_base, reg, - reg & BIT(zx_pll->lock_bit), 0, 100); -} - -static void zx_pll_disable(struct clk_hw *hw) -{ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - u32 reg; - - if (zx_pll->pd_bit > 31) - return; - - reg = readl_relaxed(zx_pll->reg_base); - writel_relaxed(reg | BIT(zx_pll->pd_bit), zx_pll->reg_base); -} - -static int zx_pll_is_enabled(struct clk_hw *hw) -{ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - u32 reg; - - reg = readl_relaxed(zx_pll->reg_base); - - return !(reg & BIT(zx_pll->pd_bit)); -} - -const struct clk_ops zx_pll_ops = { - .recalc_rate = zx_pll_recalc_rate, - .round_rate = zx_pll_round_rate, - .set_rate = zx_pll_set_rate, - .enable = zx_pll_enable, - .disable = zx_pll_disable, - .is_enabled = zx_pll_is_enabled, -}; -EXPORT_SYMBOL(zx_pll_ops); - -struct clk *clk_register_zx_pll(const char *name, const char *parent_name, - unsigned long flags, void __iomem *reg_base, - const struct zx_pll_config *lookup_table, - int count, spinlock_t *lock) -{ - struct clk_zx_pll *zx_pll; - struct clk *clk; - struct clk_init_data init; - - zx_pll = kzalloc(sizeof(*zx_pll), GFP_KERNEL); - if (!zx_pll) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = &zx_pll_ops; - init.flags = flags; - init.parent_names = parent_name ? &parent_name : NULL; - init.num_parents = parent_name ? 1 : 0; - - zx_pll->reg_base = reg_base; - zx_pll->lookup_table = lookup_table; - zx_pll->count = count; - zx_pll->lock_bit = LOCK_FLAG; - zx_pll->pd_bit = POWER_DOWN; - zx_pll->lock = lock; - zx_pll->hw.init = &init; - - clk = clk_register(NULL, &zx_pll->hw); - if (IS_ERR(clk)) - kfree(zx_pll); - - return clk; -} - -#define BPAR 1000000 -static u32 calc_reg(u32 parent_rate, u32 rate) -{ - u32 sel, integ, fra_div, tmp; - u64 tmp64 = (u64)parent_rate * BPAR; - - do_div(tmp64, rate); - integ = (u32)tmp64 / BPAR; - integ = integ >> 1; - - tmp = (u32)tmp64 % BPAR; - sel = tmp / BPAR; - - tmp = tmp % BPAR; - fra_div = tmp * 0xff / BPAR; - tmp = (sel << 24) | (integ << 16) | (0xff << 8) | fra_div; - - /* Set I2S integer divider as 1. This bit is reserved for SPDIF - * and do no harm. - */ - tmp |= BIT(28); - return tmp; -} - -static u32 calc_rate(u32 reg, u32 parent_rate) -{ - u32 sel, integ, fra_div, tmp; - u64 tmp64 = (u64)parent_rate * BPAR; - - tmp = reg; - sel = (tmp >> 24) & BIT(0); - integ = (tmp >> 16) & 0xff; - fra_div = tmp & 0xff; - - tmp = fra_div * BPAR; - tmp = tmp / 0xff; - tmp += sel * BPAR; - tmp += 2 * integ * BPAR; - do_div(tmp64, tmp); - - return (u32)tmp64; -} - -static unsigned long zx_audio_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct clk_zx_audio *zx_audio = to_clk_zx_audio(hw); - u32 reg; - - reg = readl_relaxed(zx_audio->reg_base); - return calc_rate(reg, parent_rate); -} - -static long zx_audio_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - u32 reg; - - if (rate * 2 > *prate) - return -EINVAL; - - reg = calc_reg(*prate, rate); - return calc_rate(reg, *prate); -} - -static int zx_audio_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct clk_zx_audio *zx_audio = to_clk_zx_audio(hw); - u32 reg; - - reg = calc_reg(parent_rate, rate); - writel_relaxed(reg, zx_audio->reg_base); - - return 0; -} - -#define ZX_AUDIO_EN BIT(25) -static int zx_audio_enable(struct clk_hw *hw) -{ - struct clk_zx_audio *zx_audio = to_clk_zx_audio(hw); - u32 reg; - - reg = readl_relaxed(zx_audio->reg_base); - writel_relaxed(reg & ~ZX_AUDIO_EN, zx_audio->reg_base); - return 0; -} - -static void zx_audio_disable(struct clk_hw *hw) -{ - struct clk_zx_audio *zx_audio = to_clk_zx_audio(hw); - u32 reg; - - reg = readl_relaxed(zx_audio->reg_base); - writel_relaxed(reg | ZX_AUDIO_EN, zx_audio->reg_base); -} - -static const struct clk_ops zx_audio_ops = { - .recalc_rate = zx_audio_recalc_rate, - .round_rate = zx_audio_round_rate, - .set_rate = zx_audio_set_rate, - .enable = zx_audio_enable, - .disable = zx_audio_disable, -}; - -struct clk *clk_register_zx_audio(const char *name, - const char * const parent_name, - unsigned long flags, - void __iomem *reg_base) -{ - struct clk_zx_audio *zx_audio; - struct clk *clk; - struct clk_init_data init; - - zx_audio = kzalloc(sizeof(*zx_audio), GFP_KERNEL); - if (!zx_audio) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = &zx_audio_ops; - init.flags = flags; - init.parent_names = parent_name ? &parent_name : NULL; - init.num_parents = parent_name ? 1 : 0; - - zx_audio->reg_base = reg_base; - zx_audio->hw.init = &init; - - clk = clk_register(NULL, &zx_audio->hw); - if (IS_ERR(clk)) - kfree(zx_audio); - - return clk; -} - -#define CLK_AUDIO_DIV_FRAC BIT(0) -#define CLK_AUDIO_DIV_INT BIT(1) -#define CLK_AUDIO_DIV_UNCOMMON BIT(1) - -#define CLK_AUDIO_DIV_FRAC_NSHIFT 16 -#define CLK_AUDIO_DIV_INT_FRAC_RE BIT(16) -#define CLK_AUDIO_DIV_INT_FRAC_MAX (0xffff) -#define CLK_AUDIO_DIV_INT_FRAC_MIN (0x2) -#define CLK_AUDIO_DIV_INT_INT_SHIFT 24 -#define CLK_AUDIO_DIV_INT_INT_WIDTH 4 - -struct zx_clk_audio_div_table { - unsigned long rate; - unsigned int int_reg; - unsigned int frac_reg; -}; - -#define to_clk_zx_audio_div(_hw) container_of(_hw, struct clk_zx_audio_divider, hw) - -static unsigned long audio_calc_rate(struct clk_zx_audio_divider *audio_div, - u32 reg_frac, u32 reg_int, - unsigned long parent_rate) -{ - unsigned long rate, m, n; - - m = reg_frac & 0xffff; - n = (reg_frac >> 16) & 0xffff; - - m = (reg_int & 0xffff) * n + m; - rate = (parent_rate * n) / m; - - return rate; -} - -static void audio_calc_reg(struct clk_zx_audio_divider *audio_div, - struct zx_clk_audio_div_table *div_table, - unsigned long rate, unsigned long parent_rate) -{ - unsigned int reg_int, reg_frac; - unsigned long m, n, div; - - reg_int = parent_rate / rate; - - if (reg_int > CLK_AUDIO_DIV_INT_FRAC_MAX) - reg_int = CLK_AUDIO_DIV_INT_FRAC_MAX; - else if (reg_int < CLK_AUDIO_DIV_INT_FRAC_MIN) - reg_int = 0; - m = parent_rate - rate * reg_int; - n = rate; - - div = gcd(m, n); - m = m / div; - n = n / div; - - if ((m >> 16) || (n >> 16)) { - if (m > n) { - n = n * 0xffff / m; - m = 0xffff; - } else { - m = m * 0xffff / n; - n = 0xffff; - } - } - reg_frac = m | (n << 16); - - div_table->rate = parent_rate * n / (reg_int * n + m); - div_table->int_reg = reg_int; - div_table->frac_reg = reg_frac; -} - -static unsigned long zx_audio_div_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct clk_zx_audio_divider *zx_audio_div = to_clk_zx_audio_div(hw); - u32 reg_frac, reg_int; - - reg_frac = readl_relaxed(zx_audio_div->reg_base); - reg_int = readl_relaxed(zx_audio_div->reg_base + 0x4); - - return audio_calc_rate(zx_audio_div, reg_frac, reg_int, parent_rate); -} - -static long zx_audio_div_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - struct clk_zx_audio_divider *zx_audio_div = to_clk_zx_audio_div(hw); - struct zx_clk_audio_div_table divt; - - audio_calc_reg(zx_audio_div, &divt, rate, *prate); - - return audio_calc_rate(zx_audio_div, divt.frac_reg, divt.int_reg, *prate); -} - -static int zx_audio_div_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct clk_zx_audio_divider *zx_audio_div = to_clk_zx_audio_div(hw); - struct zx_clk_audio_div_table divt; - unsigned int val; - - audio_calc_reg(zx_audio_div, &divt, rate, parent_rate); - if (divt.rate != rate) - pr_debug("the real rate is:%ld", divt.rate); - - writel_relaxed(divt.frac_reg, zx_audio_div->reg_base); - - val = readl_relaxed(zx_audio_div->reg_base + 0x4); - val &= ~0xffff; - val |= divt.int_reg | CLK_AUDIO_DIV_INT_FRAC_RE; - writel_relaxed(val, zx_audio_div->reg_base + 0x4); - - mdelay(1); - - val = readl_relaxed(zx_audio_div->reg_base + 0x4); - val &= ~CLK_AUDIO_DIV_INT_FRAC_RE; - writel_relaxed(val, zx_audio_div->reg_base + 0x4); - - return 0; -} - -const struct clk_ops zx_audio_div_ops = { - .recalc_rate = zx_audio_div_recalc_rate, - .round_rate = zx_audio_div_round_rate, - .set_rate = zx_audio_div_set_rate, -}; diff --git a/drivers/clk/zte/clk.h b/drivers/clk/zte/clk.h deleted file mode 100644 index aeaf2a380ba6..000000000000 --- a/drivers/clk/zte/clk.h +++ /dev/null @@ -1,174 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright 2015 Linaro Ltd. - * Copyright (C) 2014 ZTE Corporation. - */ - -#ifndef __ZTE_CLK_H -#define __ZTE_CLK_H -#include -#include - -#define PNAME(x) static const char *x[] - -struct zx_pll_config { - unsigned long rate; - u32 cfg0; - u32 cfg1; -}; - -struct clk_zx_pll { - struct clk_hw hw; - void __iomem *reg_base; - const struct zx_pll_config *lookup_table; /* order by rate asc */ - int count; - spinlock_t *lock; - u8 pd_bit; /* power down bit */ - u8 lock_bit; /* pll lock flag bit */ -}; - -#define PLL_RATE(_rate, _cfg0, _cfg1) \ -{ \ - .rate = _rate, \ - .cfg0 = _cfg0, \ - .cfg1 = _cfg1, \ -} - -#define ZX_PLL(_name, _parent, _reg, _table, _pd, _lock) \ -{ \ - .reg_base = (void __iomem *) _reg, \ - .lookup_table = _table, \ - .count = ARRAY_SIZE(_table), \ - .pd_bit = _pd, \ - .lock_bit = _lock, \ - .hw.init = CLK_HW_INIT(_name, _parent, &zx_pll_ops, \ - CLK_GET_RATE_NOCACHE), \ -} - -/* - * The pd_bit is not available on ZX296718, so let's pass something - * bigger than 31, e.g. 0xff, to indicate that. - */ -#define ZX296718_PLL(_name, _parent, _reg, _table) \ -ZX_PLL(_name, _parent, _reg, _table, 0xff, 30) - -struct zx_clk_gate { - struct clk_gate gate; - u16 id; -}; - -#define GATE(_id, _name, _parent, _reg, _bit, _flag, _gflags) \ -{ \ - .gate = { \ - .reg = (void __iomem *) _reg, \ - .bit_idx = (_bit), \ - .flags = _gflags, \ - .lock = &clk_lock, \ - .hw.init = CLK_HW_INIT(_name, \ - _parent, \ - &clk_gate_ops, \ - _flag | CLK_IGNORE_UNUSED), \ - }, \ - .id = _id, \ -} - -struct zx_clk_fixed_factor { - struct clk_fixed_factor factor; - u16 id; -}; - -#define FFACTOR(_id, _name, _parent, _mult, _div, _flag) \ -{ \ - .factor = { \ - .div = _div, \ - .mult = _mult, \ - .hw.init = CLK_HW_INIT(_name, \ - _parent, \ - &clk_fixed_factor_ops, \ - _flag), \ - }, \ - .id = _id, \ -} - -struct zx_clk_mux { - struct clk_mux mux; - u16 id; -}; - -#define MUX_F(_id, _name, _parent, _reg, _shift, _width, _flag, _mflag) \ -{ \ - .mux = { \ - .reg = (void __iomem *) _reg, \ - .mask = BIT(_width) - 1, \ - .shift = _shift, \ - .flags = _mflag, \ - .lock = &clk_lock, \ - .hw.init = CLK_HW_INIT_PARENTS(_name, \ - _parent, \ - &clk_mux_ops, \ - _flag), \ - }, \ - .id = _id, \ -} - -#define MUX(_id, _name, _parent, _reg, _shift, _width) \ -MUX_F(_id, _name, _parent, _reg, _shift, _width, 0, 0) - -struct zx_clk_div { - struct clk_divider div; - u16 id; -}; - -#define DIV_T(_id, _name, _parent, _reg, _shift, _width, _flag, _table) \ -{ \ - .div = { \ - .reg = (void __iomem *) _reg, \ - .shift = _shift, \ - .width = _width, \ - .flags = 0, \ - .table = _table, \ - .lock = &clk_lock, \ - .hw.init = CLK_HW_INIT(_name, \ - _parent, \ - &clk_divider_ops, \ - _flag), \ - }, \ - .id = _id, \ -} - -struct clk_zx_audio_divider { - struct clk_hw hw; - void __iomem *reg_base; - unsigned int rate_count; - spinlock_t *lock; - u16 id; -}; - -#define AUDIO_DIV(_id, _name, _parent, _reg) \ -{ \ - .reg_base = (void __iomem *) _reg, \ - .lock = &clk_lock, \ - .hw.init = CLK_HW_INIT(_name, \ - _parent, \ - &zx_audio_div_ops, \ - 0), \ - .id = _id, \ -} - -struct clk *clk_register_zx_pll(const char *name, const char *parent_name, - unsigned long flags, void __iomem *reg_base, - const struct zx_pll_config *lookup_table, int count, spinlock_t *lock); - -struct clk_zx_audio { - struct clk_hw hw; - void __iomem *reg_base; -}; - -struct clk *clk_register_zx_audio(const char *name, - const char * const parent_name, - unsigned long flags, void __iomem *reg_base); - -extern const struct clk_ops zx_pll_ops; -extern const struct clk_ops zx_audio_div_ops; - -#endif diff --git a/drivers/clk/zynq/clkc.c b/drivers/clk/zynq/clkc.c index ffbb9008c1c9..204b83d911b9 100644 --- a/drivers/clk/zynq/clkc.c +++ b/drivers/clk/zynq/clkc.c @@ -103,7 +103,6 @@ static void __init zynq_clk_register_fclk(enum zynq_clk fclk, const char *clk_name, void __iomem *fclk_ctrl_reg, const char **parents, int enable) { - struct clk *clk; u32 enable_reg; char *mux_name; char *div0_name; @@ -131,15 +130,15 @@ static void __init zynq_clk_register_fclk(enum zynq_clk fclk, if (!div1_name) goto err_div1_name; - clk = clk_register_mux(NULL, mux_name, parents, 4, + clk_register_mux(NULL, mux_name, parents, 4, CLK_SET_RATE_NO_REPARENT, fclk_ctrl_reg, 4, 2, 0, fclk_lock); - clk = clk_register_divider(NULL, div0_name, mux_name, + clk_register_divider(NULL, div0_name, mux_name, 0, fclk_ctrl_reg, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, fclk_lock); - clk = clk_register_divider(NULL, div1_name, div0_name, + clk_register_divider(NULL, div1_name, div0_name, CLK_SET_RATE_PARENT, fclk_ctrl_reg, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, fclk_lock); @@ -176,7 +175,6 @@ static void __init zynq_clk_register_periph_clk(enum zynq_clk clk0, const char *clk_name1, void __iomem *clk_ctrl, const char **parents, unsigned int two_gates) { - struct clk *clk; char *mux_name; char *div_name; spinlock_t *lock; @@ -189,10 +187,10 @@ static void __init zynq_clk_register_periph_clk(enum zynq_clk clk0, mux_name = kasprintf(GFP_KERNEL, "%s_mux", clk_name0); div_name = kasprintf(GFP_KERNEL, "%s_div", clk_name0); - clk = clk_register_mux(NULL, mux_name, parents, 4, + clk_register_mux(NULL, mux_name, parents, 4, CLK_SET_RATE_NO_REPARENT, clk_ctrl, 4, 2, 0, lock); - clk = clk_register_divider(NULL, div_name, mux_name, 0, clk_ctrl, 8, 6, + clk_register_divider(NULL, div_name, mux_name, 0, clk_ctrl, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, lock); clks[clk0] = clk_register_gate(NULL, clk_name0, div_name, @@ -217,7 +215,6 @@ static void __init zynq_clk_setup(struct device_node *np) int i; u32 tmp; int ret; - struct clk *clk; char *clk_name; unsigned int fclk_enable = 0; const char *clk_output_name[clk_max]; @@ -257,19 +254,19 @@ static void __init zynq_clk_setup(struct device_node *np) ps_clk = clk_register_fixed_rate(NULL, "ps_clk", NULL, 0, tmp); /* PLLs */ - clk = clk_register_zynq_pll("armpll_int", "ps_clk", SLCR_ARMPLL_CTRL, + clk_register_zynq_pll("armpll_int", "ps_clk", SLCR_ARMPLL_CTRL, SLCR_PLL_STATUS, 0, &armpll_lock); clks[armpll] = clk_register_mux(NULL, clk_output_name[armpll], armpll_parents, 2, CLK_SET_RATE_NO_REPARENT, SLCR_ARMPLL_CTRL, 4, 1, 0, &armpll_lock); - clk = clk_register_zynq_pll("ddrpll_int", "ps_clk", SLCR_DDRPLL_CTRL, + clk_register_zynq_pll("ddrpll_int", "ps_clk", SLCR_DDRPLL_CTRL, SLCR_PLL_STATUS, 1, &ddrpll_lock); clks[ddrpll] = clk_register_mux(NULL, clk_output_name[ddrpll], ddrpll_parents, 2, CLK_SET_RATE_NO_REPARENT, SLCR_DDRPLL_CTRL, 4, 1, 0, &ddrpll_lock); - clk = clk_register_zynq_pll("iopll_int", "ps_clk", SLCR_IOPLL_CTRL, + clk_register_zynq_pll("iopll_int", "ps_clk", SLCR_IOPLL_CTRL, SLCR_PLL_STATUS, 2, &iopll_lock); clks[iopll] = clk_register_mux(NULL, clk_output_name[iopll], iopll_parents, 2, CLK_SET_RATE_NO_REPARENT, @@ -277,10 +274,10 @@ static void __init zynq_clk_setup(struct device_node *np) /* CPU clocks */ tmp = readl(SLCR_621_TRUE) & 1; - clk = clk_register_mux(NULL, "cpu_mux", cpu_parents, 4, + clk_register_mux(NULL, "cpu_mux", cpu_parents, 4, CLK_SET_RATE_NO_REPARENT, SLCR_ARM_CLK_CTRL, 4, 2, 0, &armclk_lock); - clk = clk_register_divider(NULL, "cpu_div", "cpu_mux", 0, + clk_register_divider(NULL, "cpu_div", "cpu_mux", 0, SLCR_ARM_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &armclk_lock); @@ -288,20 +285,20 @@ static void __init zynq_clk_setup(struct device_node *np) "cpu_div", CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, SLCR_ARM_CLK_CTRL, 24, 0, &armclk_lock); - clk = clk_register_fixed_factor(NULL, "cpu_3or2x_div", "cpu_div", 0, + clk_register_fixed_factor(NULL, "cpu_3or2x_div", "cpu_div", 0, 1, 2); clks[cpu_3or2x] = clk_register_gate(NULL, clk_output_name[cpu_3or2x], "cpu_3or2x_div", CLK_IGNORE_UNUSED, SLCR_ARM_CLK_CTRL, 25, 0, &armclk_lock); - clk = clk_register_fixed_factor(NULL, "cpu_2x_div", "cpu_div", 0, 1, + clk_register_fixed_factor(NULL, "cpu_2x_div", "cpu_div", 0, 1, 2 + tmp); clks[cpu_2x] = clk_register_gate(NULL, clk_output_name[cpu_2x], "cpu_2x_div", CLK_IGNORE_UNUSED, SLCR_ARM_CLK_CTRL, 26, 0, &armclk_lock); clk_prepare_enable(clks[cpu_2x]); - clk = clk_register_fixed_factor(NULL, "cpu_1x_div", "cpu_div", 0, 1, + clk_register_fixed_factor(NULL, "cpu_1x_div", "cpu_div", 0, 1, 4 + 2 * tmp); clks[cpu_1x] = clk_register_gate(NULL, clk_output_name[cpu_1x], "cpu_1x_div", CLK_IGNORE_UNUSED, SLCR_ARM_CLK_CTRL, 27, @@ -324,23 +321,23 @@ static void __init zynq_clk_setup(struct device_node *np) &swdtclk_lock); /* DDR clocks */ - clk = clk_register_divider(NULL, "ddr2x_div", "ddrpll", 0, + clk_register_divider(NULL, "ddr2x_div", "ddrpll", 0, SLCR_DDR_CLK_CTRL, 26, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &ddrclk_lock); clks[ddr2x] = clk_register_gate(NULL, clk_output_name[ddr2x], "ddr2x_div", 0, SLCR_DDR_CLK_CTRL, 1, 0, &ddrclk_lock); clk_prepare_enable(clks[ddr2x]); - clk = clk_register_divider(NULL, "ddr3x_div", "ddrpll", 0, + clk_register_divider(NULL, "ddr3x_div", "ddrpll", 0, SLCR_DDR_CLK_CTRL, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &ddrclk_lock); clks[ddr3x] = clk_register_gate(NULL, clk_output_name[ddr3x], "ddr3x_div", 0, SLCR_DDR_CLK_CTRL, 0, 0, &ddrclk_lock); clk_prepare_enable(clks[ddr3x]); - clk = clk_register_divider(NULL, "dci_div0", "ddrpll", 0, + clk_register_divider(NULL, "dci_div0", "ddrpll", 0, SLCR_DCI_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &dciclk_lock); - clk = clk_register_divider(NULL, "dci_div1", "dci_div0", + clk_register_divider(NULL, "dci_div1", "dci_div0", CLK_SET_RATE_PARENT, SLCR_DCI_CLK_CTRL, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &dciclk_lock); @@ -385,17 +382,17 @@ static void __init zynq_clk_setup(struct device_node *np) gem0_mux_parents[i + 1] = of_clk_get_parent_name(np, idx); } - clk = clk_register_mux(NULL, "gem0_mux", periph_parents, 4, + clk_register_mux(NULL, "gem0_mux", periph_parents, 4, CLK_SET_RATE_NO_REPARENT, SLCR_GEM0_CLK_CTRL, 4, 2, 0, &gem0clk_lock); - clk = clk_register_divider(NULL, "gem0_div0", "gem0_mux", 0, + clk_register_divider(NULL, "gem0_div0", "gem0_mux", 0, SLCR_GEM0_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &gem0clk_lock); - clk = clk_register_divider(NULL, "gem0_div1", "gem0_div0", + clk_register_divider(NULL, "gem0_div1", "gem0_div0", CLK_SET_RATE_PARENT, SLCR_GEM0_CLK_CTRL, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &gem0clk_lock); - clk = clk_register_mux(NULL, "gem0_emio_mux", gem0_mux_parents, 2, + clk_register_mux(NULL, "gem0_emio_mux", gem0_mux_parents, 2, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, SLCR_GEM0_CLK_CTRL, 6, 1, 0, &gem0clk_lock); @@ -410,17 +407,17 @@ static void __init zynq_clk_setup(struct device_node *np) gem1_mux_parents[i + 1] = of_clk_get_parent_name(np, idx); } - clk = clk_register_mux(NULL, "gem1_mux", periph_parents, 4, + clk_register_mux(NULL, "gem1_mux", periph_parents, 4, CLK_SET_RATE_NO_REPARENT, SLCR_GEM1_CLK_CTRL, 4, 2, 0, &gem1clk_lock); - clk = clk_register_divider(NULL, "gem1_div0", "gem1_mux", 0, + clk_register_divider(NULL, "gem1_div0", "gem1_mux", 0, SLCR_GEM1_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &gem1clk_lock); - clk = clk_register_divider(NULL, "gem1_div1", "gem1_div0", + clk_register_divider(NULL, "gem1_div1", "gem1_div0", CLK_SET_RATE_PARENT, SLCR_GEM1_CLK_CTRL, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &gem1clk_lock); - clk = clk_register_mux(NULL, "gem1_emio_mux", gem1_mux_parents, 2, + clk_register_mux(NULL, "gem1_emio_mux", gem1_mux_parents, 2, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, SLCR_GEM1_CLK_CTRL, 6, 1, 0, &gem1clk_lock); @@ -442,27 +439,27 @@ static void __init zynq_clk_setup(struct device_node *np) can_mio_mux_parents[i] = dummy_nm; } kfree(clk_name); - clk = clk_register_mux(NULL, "can_mux", periph_parents, 4, + clk_register_mux(NULL, "can_mux", periph_parents, 4, CLK_SET_RATE_NO_REPARENT, SLCR_CAN_CLK_CTRL, 4, 2, 0, &canclk_lock); - clk = clk_register_divider(NULL, "can_div0", "can_mux", 0, + clk_register_divider(NULL, "can_div0", "can_mux", 0, SLCR_CAN_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &canclk_lock); - clk = clk_register_divider(NULL, "can_div1", "can_div0", + clk_register_divider(NULL, "can_div1", "can_div0", CLK_SET_RATE_PARENT, SLCR_CAN_CLK_CTRL, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &canclk_lock); - clk = clk_register_gate(NULL, "can0_gate", "can_div1", + clk_register_gate(NULL, "can0_gate", "can_div1", CLK_SET_RATE_PARENT, SLCR_CAN_CLK_CTRL, 0, 0, &canclk_lock); - clk = clk_register_gate(NULL, "can1_gate", "can_div1", + clk_register_gate(NULL, "can1_gate", "can_div1", CLK_SET_RATE_PARENT, SLCR_CAN_CLK_CTRL, 1, 0, &canclk_lock); - clk = clk_register_mux(NULL, "can0_mio_mux", + clk_register_mux(NULL, "can0_mio_mux", can_mio_mux_parents, 54, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, SLCR_CAN_MIOCLK_CTRL, 0, 6, 0, &canmioclk_lock); - clk = clk_register_mux(NULL, "can1_mio_mux", + clk_register_mux(NULL, "can1_mio_mux", can_mio_mux_parents, 54, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, SLCR_CAN_MIOCLK_CTRL, 16, 6, 0, &canmioclk_lock); @@ -482,13 +479,13 @@ static void __init zynq_clk_setup(struct device_node *np) dbg_emio_mux_parents[i + 1] = of_clk_get_parent_name(np, idx); } - clk = clk_register_mux(NULL, "dbg_mux", periph_parents, 4, + clk_register_mux(NULL, "dbg_mux", periph_parents, 4, CLK_SET_RATE_NO_REPARENT, SLCR_DBG_CLK_CTRL, 4, 2, 0, &dbgclk_lock); - clk = clk_register_divider(NULL, "dbg_div", "dbg_mux", 0, + clk_register_divider(NULL, "dbg_div", "dbg_mux", 0, SLCR_DBG_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &dbgclk_lock); - clk = clk_register_mux(NULL, "dbg_emio_mux", dbg_emio_mux_parents, 2, + clk_register_mux(NULL, "dbg_emio_mux", dbg_emio_mux_parents, 2, CLK_SET_RATE_NO_REPARENT, SLCR_DBG_CLK_CTRL, 6, 1, 0, &dbgclk_lock); clks[dbg_trc] = clk_register_gate(NULL, clk_output_name[dbg_trc], diff --git a/drivers/clk/zynq/pll.c b/drivers/clk/zynq/pll.c index dcb2037a9596..54f4184de89a 100644 --- a/drivers/clk/zynq/pll.c +++ b/drivers/clk/zynq/pll.c @@ -173,12 +173,12 @@ static const struct clk_ops zynq_pll_ops = { /** * clk_register_zynq_pll() - Register PLL with the clock framework - * @name PLL name - * @parent Parent clock name - * @pll_ctrl Pointer to PLL control register - * @pll_status Pointer to PLL status register - * @lock_index Bit index to this PLL's lock status bit in @pll_status - * @lock Register lock + * @name: PLL name + * @parent: Parent clock name + * @pll_ctrl: Pointer to PLL control register + * @pll_status: Pointer to PLL status register + * @lock_index: Bit index to this PLL's lock status bit in @pll_status + * @lock: Register lock * Returns handle to the registered clock. */ struct clk *clk_register_zynq_pll(const char *name, const char *parent, diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c index 66da02b83d39..e9bf7958b821 100644 --- a/drivers/clk/zynqmp/divider.c +++ b/drivers/clk/zynqmp/divider.c @@ -35,6 +35,7 @@ * @is_frac: The divider is a fractional divider * @clk_id: Id of clock * @div_type: divisor type (TYPE_DIV1 or TYPE_DIV2) + * @max_div: maximum supported divisor (fetched from firmware) */ struct zynqmp_clk_divider { struct clk_hw hw; diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index d184000eb09e..39aa21d01e05 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -79,6 +79,7 @@ config IXP4XX_TIMER bool "Intel XScale IXP4xx timer driver" if COMPILE_TEST depends on HAS_IOMEM select CLKSRC_MMIO + select TIMER_OF if OF help Enables support for the Intel XScale IXP4xx SoC timer. diff --git a/drivers/clocksource/mxs_timer.c b/drivers/clocksource/mxs_timer.c index bc96a4cbf26c..e52e12d27d2a 100644 --- a/drivers/clocksource/mxs_timer.c +++ b/drivers/clocksource/mxs_timer.c @@ -131,10 +131,7 @@ static void mxs_irq_clear(char *state) /* Clear pending interrupt */ timrot_irq_acknowledge(); - -#ifdef DEBUG - pr_info("%s: changing mode to %s\n", __func__, state) -#endif /* DEBUG */ + pr_debug("%s: changing mode to %s\n", __func__, state); } static int mxs_shutdown(struct clock_event_device *evt) diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c index e258230d432c..c98f8851fd68 100644 --- a/drivers/clocksource/sh_cmt.c +++ b/drivers/clocksource/sh_cmt.c @@ -235,6 +235,8 @@ static const struct sh_cmt_info sh_cmt_info[] = { #define CMCNT 1 /* channel register */ #define CMCOR 2 /* channel register */ +#define CMCLKE 0x1000 /* CLK Enable Register (R-Car Gen2) */ + static inline u32 sh_cmt_read_cmstr(struct sh_cmt_channel *ch) { if (ch->iostart) @@ -853,6 +855,7 @@ static int sh_cmt_setup_channel(struct sh_cmt_channel *ch, unsigned int index, unsigned int hwidx, bool clockevent, bool clocksource, struct sh_cmt_device *cmt) { + u32 value; int ret; /* Skip unused channels. */ @@ -882,6 +885,11 @@ static int sh_cmt_setup_channel(struct sh_cmt_channel *ch, unsigned int index, ch->iostart = cmt->mapbase + ch->hwidx * 0x100; ch->ioctrl = ch->iostart + 0x10; ch->timer_bit = 0; + + /* Enable the clock supply to the channel */ + value = ioread32(cmt->mapbase + CMCLKE); + value |= BIT(hwidx); + iowrite32(value, cmt->mapbase + CMCLKE); break; } @@ -1014,12 +1022,10 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) else cmt->rate = clk_get_rate(cmt->clk) / 8; - clk_disable(cmt->clk); - /* Map the memory resource(s). */ ret = sh_cmt_map_memory(cmt); if (ret < 0) - goto err_clk_unprepare; + goto err_clk_disable; /* Allocate and setup the channels. */ cmt->num_channels = hweight8(cmt->hw_channels); @@ -1047,6 +1053,8 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) mask &= ~(1 << hwidx); } + clk_disable(cmt->clk); + platform_set_drvdata(pdev, cmt); return 0; @@ -1054,6 +1062,8 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) err_unmap: kfree(cmt->channels); iounmap(cmt->mapbase); +err_clk_disable: + clk_disable(cmt->clk); err_clk_unprepare: clk_unprepare(cmt->clk); err_clk_put: diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 399526289320..92701a18bdd9 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -62,16 +62,6 @@ config X86_ACPI_CPUFREQ_CPB By enabling this option the acpi_cpufreq driver provides the old entry in addition to the new boost ones, for compatibility reasons. -config X86_SFI_CPUFREQ - tristate "SFI Performance-States driver" - depends on X86_INTEL_MID && SFI - help - This adds a CPUFreq driver for some Silvermont based Intel Atom - architectures like Z34xx and Z35xx which enumerate processor - performance states through SFI. - - If in doubt, say N. - config ELAN_CPUFREQ tristate "AMD Elan SC400 and SC410" depends on MELAN diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 1ab9b1536304..27d3bd7ea9d4 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -43,7 +43,6 @@ obj-$(CONFIG_X86_P4_CLOCKMOD) += p4-clockmod.o obj-$(CONFIG_X86_CPUFREQ_NFORCE2) += cpufreq-nforce2.o obj-$(CONFIG_X86_INTEL_PSTATE) += intel_pstate.o obj-$(CONFIG_X86_AMD_FREQ_SENSITIVITY) += amd_freq_sensitivity.o -obj-$(CONFIG_X86_SFI_CPUFREQ) += sfi-cpufreq.o ################################################################################## # ARM SoC drivers diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index d3e5a6fceb61..d1bbc16fba4b 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -54,7 +54,6 @@ struct acpi_cpufreq_data { unsigned int resume; unsigned int cpu_feature; unsigned int acpi_perf_cpu; - unsigned int first_perf_state; cpumask_var_t freqdomain_cpus; void (*cpu_freq_write)(struct acpi_pct_register *reg, u32 val); u32 (*cpu_freq_read)(struct acpi_pct_register *reg); @@ -223,10 +222,10 @@ static unsigned extract_msr(struct cpufreq_policy *policy, u32 msr) perf = to_perf_data(data); - cpufreq_for_each_entry(pos, policy->freq_table + data->first_perf_state) + cpufreq_for_each_entry(pos, policy->freq_table) if (msr == perf->states[pos->driver_data].status) return pos->frequency; - return policy->freq_table[data->first_perf_state].frequency; + return policy->freq_table[0].frequency; } static unsigned extract_freq(struct cpufreq_policy *policy, u32 val) @@ -365,7 +364,6 @@ static unsigned int get_cur_freq_on_cpu(unsigned int cpu) struct cpufreq_policy *policy; unsigned int freq; unsigned int cached_freq; - unsigned int state; pr_debug("%s (%d)\n", __func__, cpu); @@ -377,11 +375,7 @@ static unsigned int get_cur_freq_on_cpu(unsigned int cpu) if (unlikely(!data || !policy->freq_table)) return 0; - state = to_perf_data(data)->state; - if (state < data->first_perf_state) - state = data->first_perf_state; - - cached_freq = policy->freq_table[state].frequency; + cached_freq = policy->freq_table[to_perf_data(data)->state].frequency; freq = extract_freq(policy, get_cur_val(cpumask_of(cpu), data)); if (freq != cached_freq) { /* @@ -680,7 +674,6 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) struct cpuinfo_x86 *c = &cpu_data(cpu); unsigned int valid_states = 0; unsigned int result = 0; - unsigned int state_count; u64 max_boost_ratio; unsigned int i; #ifdef CONFIG_SMP @@ -795,28 +788,8 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) goto err_unreg; } - state_count = perf->state_count + 1; - - max_boost_ratio = get_max_boost_ratio(cpu); - if (max_boost_ratio) { - /* - * Make a room for one more entry to represent the highest - * available "boost" frequency. - */ - state_count++; - valid_states++; - data->first_perf_state = valid_states; - } else { - /* - * If the maximum "boost" frequency is unknown, ask the arch - * scale-invariance code to use the "nominal" performance for - * CPU utilization scaling so as to prevent the schedutil - * governor from selecting inadequate CPU frequencies. - */ - arch_set_max_freq_ratio(true); - } - - freq_table = kcalloc(state_count, sizeof(*freq_table), GFP_KERNEL); + freq_table = kcalloc(perf->state_count + 1, sizeof(*freq_table), + GFP_KERNEL); if (!freq_table) { result = -ENOMEM; goto err_unreg; @@ -851,27 +824,25 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) } freq_table[valid_states].frequency = CPUFREQ_TABLE_END; + max_boost_ratio = get_max_boost_ratio(cpu); if (max_boost_ratio) { - unsigned int state = data->first_perf_state; - unsigned int freq = freq_table[state].frequency; + unsigned int freq = freq_table[0].frequency; /* * Because the loop above sorts the freq_table entries in the * descending order, freq is the maximum frequency in the table. * Assume that it corresponds to the CPPC nominal frequency and - * use it to populate the frequency field of the extra "boost" - * frequency entry. + * use it to set cpuinfo.max_freq. */ - freq_table[0].frequency = freq * max_boost_ratio >> SCHED_CAPACITY_SHIFT; + policy->cpuinfo.max_freq = freq * max_boost_ratio >> SCHED_CAPACITY_SHIFT; + } else { /* - * The purpose of the extra "boost" frequency entry is to make - * the rest of cpufreq aware of the real maximum frequency, but - * the way to request it is the same as for the first_perf_state - * entry that is expected to cover the entire range of "boost" - * frequencies of the CPU, so copy the driver_data value from - * that entry. + * If the maximum "boost" frequency is unknown, ask the arch + * scale-invariance code to use the "nominal" performance for + * CPU utilization scaling so as to prevent the schedutil + * governor from selecting inadequate CPU frequencies. */ - freq_table[0].driver_data = freq_table[state].driver_data; + arch_set_max_freq_ratio(true); } policy->freq_table = freq_table; @@ -947,8 +918,7 @@ static void acpi_cpufreq_cpu_ready(struct cpufreq_policy *policy) { struct acpi_processor_performance *perf = per_cpu_ptr(acpi_perf_data, policy->cpu); - struct acpi_cpufreq_data *data = policy->driver_data; - unsigned int freq = policy->freq_table[data->first_perf_state].frequency; + unsigned int freq = policy->freq_table[0].frequency; if (perf->states[0].core_frequency * 1000 != freq) pr_warn(FW_WARN "P-state 0 is not max freq\n"); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 7d0ae968def7..1d1b563cea4b 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2101,7 +2101,7 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go. * @cpu: Target CPU. * @min_perf: Minimum (required) performance level (units of @capacity). - * @target_perf: Terget (desired) performance level (units of @capacity). + * @target_perf: Target (desired) performance level (units of @capacity). * @capacity: Capacity of the target CPU. * * Carry out a fast performance level switch of @cpu without sleeping. diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index f839dc9852c0..d3f756f7b5a0 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -52,7 +52,13 @@ int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, } policy->min = policy->cpuinfo.min_freq = min_freq; - policy->max = policy->cpuinfo.max_freq = max_freq; + policy->max = max_freq; + /* + * If the driver has set its own cpuinfo.max_freq above max_freq, leave + * it as is. + */ + if (policy->cpuinfo.max_freq < max_freq) + policy->max = policy->cpuinfo.max_freq = max_freq; if (policy->min == ~0) return -EINVAL; diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 356244510b18..d3c23447b892 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -32,6 +32,7 @@ struct qcom_cpufreq_soc_data { struct qcom_cpufreq_data { void __iomem *base; + struct resource *res; const struct qcom_cpufreq_soc_data *soc_data; }; @@ -280,6 +281,7 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) struct of_phandle_args args; struct device_node *cpu_np; struct device *cpu_dev; + struct resource *res; void __iomem *base; struct qcom_cpufreq_data *data; int ret, index; @@ -303,18 +305,33 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) index = args.args[0]; - base = devm_platform_ioremap_resource(pdev, index); - if (IS_ERR(base)) - return PTR_ERR(base); + res = platform_get_resource(pdev, IORESOURCE_MEM, index); + if (!res) { + dev_err(dev, "failed to get mem resource %d\n", index); + return -ENODEV; + } - data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!request_mem_region(res->start, resource_size(res), res->name)) { + dev_err(dev, "failed to request resource %pR\n", res); + return -EBUSY; + } + + base = ioremap(res->start, resource_size(res)); + if (IS_ERR(base)) { + dev_err(dev, "failed to map resource %pR\n", res); + ret = PTR_ERR(base); + goto release_region; + } + + data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) { ret = -ENOMEM; - goto error; + goto unmap_base; } data->soc_data = of_device_get_match_data(&pdev->dev); data->base = base; + data->res = res; /* HW should be in enabled state to proceed */ if (!(readl_relaxed(base + data->soc_data->reg_enable) & 0x1)) { @@ -355,7 +372,11 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) return 0; error: - devm_iounmap(dev, base); + kfree(data); +unmap_base: + iounmap(data->base); +release_region: + release_mem_region(res->start, resource_size(res)); return ret; } @@ -363,12 +384,15 @@ static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) { struct device *cpu_dev = get_cpu_device(policy->cpu); struct qcom_cpufreq_data *data = policy->driver_data; - struct platform_device *pdev = cpufreq_get_driver_data(); + struct resource *res = data->res; + void __iomem *base = data->base; dev_pm_opp_remove_all_dynamic(cpu_dev); dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); kfree(policy->freq_table); - devm_iounmap(&pdev->dev, data->base); + kfree(data); + iounmap(base); + release_mem_region(res->start, resource_size(res)); return 0; } diff --git a/drivers/cpufreq/sfi-cpufreq.c b/drivers/cpufreq/sfi-cpufreq.c deleted file mode 100644 index 45cfdf67cf03..000000000000 --- a/drivers/cpufreq/sfi-cpufreq.c +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * SFI Performance States Driver - * - * Author: Vishwesh M Rudramuni - * Author: Srinidhi Kasagar - */ - -#include -#include -#include -#include -#include -#include -#include - -#include - -static struct cpufreq_frequency_table *freq_table; -static struct sfi_freq_table_entry *sfi_cpufreq_array; -static int num_freq_table_entries; - -static int sfi_parse_freq(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_freq_table_entry *pentry; - int totallen; - - sb = (struct sfi_table_simple *)table; - num_freq_table_entries = SFI_GET_NUM_ENTRIES(sb, - struct sfi_freq_table_entry); - if (num_freq_table_entries <= 1) { - pr_err("No p-states discovered\n"); - return -ENODEV; - } - - pentry = (struct sfi_freq_table_entry *)sb->pentry; - totallen = num_freq_table_entries * sizeof(*pentry); - - sfi_cpufreq_array = kmemdup(pentry, totallen, GFP_KERNEL); - if (!sfi_cpufreq_array) - return -ENOMEM; - - return 0; -} - -static int sfi_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) -{ - unsigned int next_perf_state = 0; /* Index into perf table */ - u32 lo, hi; - - next_perf_state = policy->freq_table[index].driver_data; - - rdmsr_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, &lo, &hi); - lo = (lo & ~INTEL_PERF_CTL_MASK) | - ((u32) sfi_cpufreq_array[next_perf_state].ctrl_val & - INTEL_PERF_CTL_MASK); - wrmsr_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, lo, hi); - - return 0; -} - -static int sfi_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - policy->shared_type = CPUFREQ_SHARED_TYPE_HW; - policy->cpuinfo.transition_latency = 100000; /* 100us */ - policy->freq_table = freq_table; - - return 0; -} - -static struct cpufreq_driver sfi_cpufreq_driver = { - .flags = CPUFREQ_CONST_LOOPS, - .verify = cpufreq_generic_frequency_table_verify, - .target_index = sfi_cpufreq_target, - .init = sfi_cpufreq_cpu_init, - .name = "sfi-cpufreq", - .attr = cpufreq_generic_attr, -}; - -static int __init sfi_cpufreq_init(void) -{ - int ret, i; - - /* parse the freq table from SFI */ - ret = sfi_table_parse(SFI_SIG_FREQ, NULL, NULL, sfi_parse_freq); - if (ret) - return ret; - - freq_table = kcalloc(num_freq_table_entries + 1, sizeof(*freq_table), - GFP_KERNEL); - if (!freq_table) { - ret = -ENOMEM; - goto err_free_array; - } - - for (i = 0; i < num_freq_table_entries; i++) { - freq_table[i].driver_data = i; - freq_table[i].frequency = sfi_cpufreq_array[i].freq_mhz * 1000; - } - freq_table[i].frequency = CPUFREQ_TABLE_END; - - ret = cpufreq_register_driver(&sfi_cpufreq_driver); - if (ret) - goto err_free_tbl; - - return ret; - -err_free_tbl: - kfree(freq_table); -err_free_array: - kfree(sfi_cpufreq_array); - return ret; -} -late_initcall(sfi_cpufreq_init); - -static void __exit sfi_cpufreq_exit(void) -{ - cpufreq_unregister_driver(&sfi_cpufreq_driver); - kfree(freq_table); - kfree(sfi_cpufreq_array); -} -module_exit(sfi_cpufreq_exit); - -MODULE_AUTHOR("Vishwesh M Rudramuni "); -MODULE_DESCRIPTION("SFI Performance-States Driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index e535f28a8028..9a4c275a1335 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -348,7 +348,7 @@ config CRYPTO_DEV_PPC4XX config HW_RANDOM_PPC4XX bool "PowerPC 4xx generic true random number generator support" - depends on CRYPTO_DEV_PPC4XX && HW_RANDOM + depends on CRYPTO_DEV_PPC4XX && HW_RANDOM=y default y help This option provides the kernel-side support for the TRNG hardware @@ -404,24 +404,6 @@ config CRYPTO_DEV_OMAP_DES endif # CRYPTO_DEV_OMAP -config CRYPTO_DEV_PICOXCELL - tristate "Support for picoXcell IPSEC and Layer2 crypto engines" - depends on (ARCH_PICOXCELL || COMPILE_TEST) && HAVE_CLK - select CRYPTO_AEAD - select CRYPTO_AES - select CRYPTO_AUTHENC - select CRYPTO_SKCIPHER - select CRYPTO_LIB_DES - select CRYPTO_CBC - select CRYPTO_ECB - select CRYPTO_SEQIV - help - This option enables support for the hardware offload engines in the - Picochip picoXcell SoC devices. Select this for IPSEC ESP offload - and for 3gpp Layer 2 ciphering support. - - Saying m here will build a module named picoxcell_crypto. - config CRYPTO_DEV_SAHARA tristate "Support for SAHARA crypto accelerator" depends on ARCH_MXC && OF @@ -773,21 +755,6 @@ config CRYPTO_DEV_ZYNQMP_AES accelerator. Select this if you want to use the ZynqMP module for AES algorithms. -config CRYPTO_DEV_MEDIATEK - tristate "MediaTek's EIP97 Cryptographic Engine driver" - depends on (ARM && ARCH_MEDIATEK) || COMPILE_TEST - select CRYPTO_LIB_AES - select CRYPTO_AEAD - select CRYPTO_SKCIPHER - select CRYPTO_SHA1 - select CRYPTO_SHA256 - select CRYPTO_SHA512 - select CRYPTO_HMAC - help - This driver allows you to utilize the hardware crypto accelerator - EIP97 which can be found on the MT7623 MT2701, MT8521p, etc .... - Select this if you want to use it for AES/SHA1/SHA2 algorithms. - source "drivers/crypto/chelsio/Kconfig" source "drivers/crypto/virtio/Kconfig" diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index fff9a70348e1..fa22cb19e242 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -19,7 +19,6 @@ obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o obj-$(CONFIG_CRYPTO_DEV_MARVELL) += marvell/ -obj-$(CONFIG_CRYPTO_DEV_MEDIATEK) += mediatek/ obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o n2_crypto-y := n2_core.o n2_asm.o @@ -31,7 +30,6 @@ obj-$(CONFIG_CRYPTO_DEV_OMAP_DES) += omap-des.o obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o -obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 180c8a9db819..856fb2045656 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -32,6 +32,15 @@ config CRYPTO_DEV_SUN4I_SS_PRNG Select this option if you want to provide kernel-side support for the Pseudo-Random Number Generator found in the Security System. +config CRYPTO_DEV_SUN4I_SS_DEBUG + bool "Enable sun4i-ss stats" + depends on CRYPTO_DEV_SUN4I_SS + depends on DEBUG_FS + help + Say y to enable sun4i-ss debug stats. + This will create /sys/kernel/debug/sun4i-ss/stats for displaying + the number of requests per algorithm. + config CRYPTO_DEV_SUN8I_CE tristate "Support for Allwinner Crypto Engine cryptographic offloader" select CRYPTO_SKCIPHER diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c index b72de8939497..c2e6f5ed1d79 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c @@ -20,6 +20,7 @@ static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) unsigned int ivsize = crypto_skcipher_ivsize(tfm); struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); u32 mode = ctx->mode; + void *backup_iv = NULL; /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ u32 rx_cnt = SS_RX_DEFAULT; u32 tx_cnt = 0; @@ -30,9 +31,13 @@ static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) unsigned int ileft = areq->cryptlen; unsigned int oleft = areq->cryptlen; unsigned int todo; + unsigned long pi = 0, po = 0; /* progress for in and out */ + bool miter_err; struct sg_mapping_iter mi, mo; unsigned int oi, oo; /* offset for in and out */ unsigned long flags; + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun4i_ss_alg_template *algt; if (!areq->cryptlen) return 0; @@ -42,52 +47,77 @@ static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) return -EINVAL; } + if (areq->iv && ivsize > 0 && mode & SS_DECRYPTION) { + backup_iv = kzalloc(ivsize, GFP_KERNEL); + if (!backup_iv) + return -ENOMEM; + scatterwalk_map_and_copy(backup_iv, areq->src, areq->cryptlen - ivsize, ivsize, 0); + } + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + algt->stat_opti++; + algt->stat_bytes += areq->cryptlen; + } + spin_lock_irqsave(&ss->slock, flags); - for (i = 0; i < op->keylen; i += 4) - writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); + for (i = 0; i < op->keylen / 4; i++) + writesl(ss->base + SS_KEY0 + i * 4, &op->key[i], 1); if (areq->iv) { for (i = 0; i < 4 && i < ivsize / 4; i++) { v = *(u32 *)(areq->iv + i * 4); - writel(v, ss->base + SS_IV0 + i * 4); + writesl(ss->base + SS_IV0 + i * 4, &v, 1); } } writel(mode, ss->base + SS_CTL); - sg_miter_start(&mi, areq->src, sg_nents(areq->src), - SG_MITER_FROM_SG | SG_MITER_ATOMIC); - sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), - SG_MITER_TO_SG | SG_MITER_ATOMIC); - sg_miter_next(&mi); - sg_miter_next(&mo); - if (!mi.addr || !mo.addr) { - dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); - err = -EINVAL; - goto release_ss; - } ileft = areq->cryptlen / 4; oleft = areq->cryptlen / 4; oi = 0; oo = 0; do { - todo = min(rx_cnt, ileft); - todo = min_t(size_t, todo, (mi.length - oi) / 4); - if (todo) { - ileft -= todo; - writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); - oi += todo * 4; - } - if (oi == mi.length) { - sg_miter_next(&mi); - oi = 0; + if (ileft) { + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + if (pi) + sg_miter_skip(&mi, pi); + miter_err = sg_miter_next(&mi); + if (!miter_err || !mi.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } + todo = min(rx_cnt, ileft); + todo = min_t(size_t, todo, (mi.length - oi) / 4); + if (todo) { + ileft -= todo; + writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); + oi += todo * 4; + } + if (oi == mi.length) { + pi += mi.length; + oi = 0; + } + sg_miter_stop(&mi); } spaces = readl(ss->base + SS_FCSR); rx_cnt = SS_RXFIFO_SPACES(spaces); tx_cnt = SS_TXFIFO_SPACES(spaces); + sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), + SG_MITER_TO_SG | SG_MITER_ATOMIC); + if (po) + sg_miter_skip(&mo, po); + miter_err = sg_miter_next(&mo); + if (!miter_err || !mo.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } todo = min(tx_cnt, oleft); todo = min_t(size_t, todo, (mo.length - oo) / 4); if (todo) { @@ -96,33 +126,41 @@ static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) oo += todo * 4; } if (oo == mo.length) { - sg_miter_next(&mo); oo = 0; + po += mo.length; } + sg_miter_stop(&mo); } while (oleft); if (areq->iv) { - for (i = 0; i < 4 && i < ivsize / 4; i++) { - v = readl(ss->base + SS_IV0 + i * 4); - *(u32 *)(areq->iv + i * 4) = v; + if (mode & SS_DECRYPTION) { + memcpy(areq->iv, backup_iv, ivsize); + kfree_sensitive(backup_iv); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, areq->cryptlen - ivsize, + ivsize, 0); } } release_ss: - sg_miter_stop(&mi); - sg_miter_stop(&mo); writel(0, ss->base + SS_CTL); spin_unlock_irqrestore(&ss->slock, flags); return err; } - static int noinline_for_stack sun4i_ss_cipher_poll_fallback(struct skcipher_request *areq) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); int err; + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun4i_ss_alg_template *algt; + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + algt->stat_fb++; + } skcipher_request_set_tfm(&ctx->fallback_req, op->fallback_tfm); skcipher_request_set_callback(&ctx->fallback_req, areq->base.flags, @@ -161,13 +199,16 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) unsigned int ileft = areq->cryptlen; unsigned int oleft = areq->cryptlen; unsigned int todo; + void *backup_iv = NULL; struct sg_mapping_iter mi, mo; + unsigned long pi = 0, po = 0; /* progress for in and out */ + bool miter_err; unsigned int oi, oo; /* offset for in and out */ unsigned int ob = 0; /* offset in buf */ unsigned int obo = 0; /* offset in bufo*/ unsigned int obl = 0; /* length of data in bufo */ unsigned long flags; - bool need_fallback; + bool need_fallback = false; if (!areq->cryptlen) return 0; @@ -186,12 +227,12 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) * we can use the SS optimized function */ while (in_sg && no_chunk == 1) { - if (in_sg->length % 4) + if ((in_sg->length | in_sg->offset) & 3u) no_chunk = 0; in_sg = sg_next(in_sg); } while (out_sg && no_chunk == 1) { - if (out_sg->length % 4) + if ((out_sg->length | out_sg->offset) & 3u) no_chunk = 0; out_sg = sg_next(out_sg); } @@ -202,30 +243,31 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) if (need_fallback) return sun4i_ss_cipher_poll_fallback(areq); + if (areq->iv && ivsize > 0 && mode & SS_DECRYPTION) { + backup_iv = kzalloc(ivsize, GFP_KERNEL); + if (!backup_iv) + return -ENOMEM; + scatterwalk_map_and_copy(backup_iv, areq->src, areq->cryptlen - ivsize, ivsize, 0); + } + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt->stat_req++; + algt->stat_bytes += areq->cryptlen; + } + spin_lock_irqsave(&ss->slock, flags); - for (i = 0; i < op->keylen; i += 4) - writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); + for (i = 0; i < op->keylen / 4; i++) + writesl(ss->base + SS_KEY0 + i * 4, &op->key[i], 1); if (areq->iv) { for (i = 0; i < 4 && i < ivsize / 4; i++) { v = *(u32 *)(areq->iv + i * 4); - writel(v, ss->base + SS_IV0 + i * 4); + writesl(ss->base + SS_IV0 + i * 4, &v, 1); } } writel(mode, ss->base + SS_CTL); - sg_miter_start(&mi, areq->src, sg_nents(areq->src), - SG_MITER_FROM_SG | SG_MITER_ATOMIC); - sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), - SG_MITER_TO_SG | SG_MITER_ATOMIC); - sg_miter_next(&mi); - sg_miter_next(&mo); - if (!mi.addr || !mo.addr) { - dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); - err = -EINVAL; - goto release_ss; - } ileft = areq->cryptlen; oleft = areq->cryptlen; oi = 0; @@ -233,8 +275,16 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) while (oleft) { if (ileft) { - char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ - + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + if (pi) + sg_miter_skip(&mi, pi); + miter_err = sg_miter_next(&mi); + if (!miter_err || !mi.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } /* * todo is the number of consecutive 4byte word that we * can read from current SG @@ -256,52 +306,57 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) */ todo = min(rx_cnt * 4 - ob, ileft); todo = min_t(size_t, todo, mi.length - oi); - memcpy(buf + ob, mi.addr + oi, todo); + memcpy(ss->buf + ob, mi.addr + oi, todo); ileft -= todo; oi += todo; ob += todo; if (!(ob % 4)) { - writesl(ss->base + SS_RXFIFO, buf, + writesl(ss->base + SS_RXFIFO, ss->buf, ob / 4); ob = 0; } } if (oi == mi.length) { - sg_miter_next(&mi); + pi += mi.length; oi = 0; } + sg_miter_stop(&mi); } spaces = readl(ss->base + SS_FCSR); rx_cnt = SS_RXFIFO_SPACES(spaces); tx_cnt = SS_TXFIFO_SPACES(spaces); - dev_dbg(ss->dev, - "%x %u/%zu %u/%u cnt=%u %u/%zu %u/%u cnt=%u %u\n", - mode, - oi, mi.length, ileft, areq->cryptlen, rx_cnt, - oo, mo.length, oleft, areq->cryptlen, tx_cnt, ob); if (!tx_cnt) continue; + sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), + SG_MITER_TO_SG | SG_MITER_ATOMIC); + if (po) + sg_miter_skip(&mo, po); + miter_err = sg_miter_next(&mo); + if (!miter_err || !mo.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } /* todo in 4bytes word */ todo = min(tx_cnt, oleft / 4); todo = min_t(size_t, todo, (mo.length - oo) / 4); + if (todo) { readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); oleft -= todo * 4; oo += todo * 4; if (oo == mo.length) { - sg_miter_next(&mo); + po += mo.length; oo = 0; } } else { - char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ - /* * read obl bytes in bufo, we read at maximum for * emptying the device */ - readsl(ss->base + SS_TXFIFO, bufo, tx_cnt); + readsl(ss->base + SS_TXFIFO, ss->bufo, tx_cnt); obl = tx_cnt * 4; obo = 0; do { @@ -313,28 +368,31 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) */ todo = min_t(size_t, mo.length - oo, obl - obo); - memcpy(mo.addr + oo, bufo + obo, todo); + memcpy(mo.addr + oo, ss->bufo + obo, todo); oleft -= todo; obo += todo; oo += todo; if (oo == mo.length) { + po += mo.length; sg_miter_next(&mo); oo = 0; } } while (obo < obl); /* bufo must be fully used here */ } + sg_miter_stop(&mo); } if (areq->iv) { - for (i = 0; i < 4 && i < ivsize / 4; i++) { - v = readl(ss->base + SS_IV0 + i * 4); - *(u32 *)(areq->iv + i * 4) = v; + if (mode & SS_DECRYPTION) { + memcpy(areq->iv, backup_iv, ivsize); + kfree_sensitive(backup_iv); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, areq->cryptlen - ivsize, + ivsize, 0); } } release_ss: - sg_miter_stop(&mi); - sg_miter_stop(&mo); writel(0, ss->base + SS_CTL); spin_unlock_irqrestore(&ss->slock, flags); @@ -503,7 +561,6 @@ int sun4i_ss_cipher_init(struct crypto_tfm *tfm) sizeof(struct sun4i_cipher_req_ctx) + crypto_skcipher_reqsize(op->fallback_tfm)); - err = pm_runtime_get_sync(op->ss->dev); if (err < 0) goto error_pm; @@ -590,5 +647,4 @@ int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); - } diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c index a2b67f7f8a81..709905ec4680 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c @@ -10,6 +10,7 @@ */ #include #include +#include #include #include #include @@ -234,6 +235,51 @@ static struct sun4i_ss_alg_template ss_algs[] = { #endif }; +static int sun4i_ss_dbgfs_read(struct seq_file *seq, void *v) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + if (!ss_algs[i].ss) + continue; + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + seq_printf(seq, "%s %s reqs=%lu opti=%lu fallback=%lu tsize=%lu\n", + ss_algs[i].alg.crypto.base.cra_driver_name, + ss_algs[i].alg.crypto.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_opti, ss_algs[i].stat_fb, + ss_algs[i].stat_bytes); + break; + case CRYPTO_ALG_TYPE_RNG: + seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n", + ss_algs[i].alg.rng.base.cra_driver_name, + ss_algs[i].alg.rng.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_bytes); + break; + case CRYPTO_ALG_TYPE_AHASH: + seq_printf(seq, "%s %s reqs=%lu\n", + ss_algs[i].alg.hash.halg.base.cra_driver_name, + ss_algs[i].alg.hash.halg.base.cra_name, + ss_algs[i].stat_req); + break; + } + } + return 0; +} + +static int sun4i_ss_dbgfs_open(struct inode *inode, struct file *file) +{ + return single_open(file, sun4i_ss_dbgfs_read, inode->i_private); +} + +static const struct file_operations sun4i_ss_debugfs_fops = { + .owner = THIS_MODULE, + .open = sun4i_ss_dbgfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + /* * Power management strategy: The device is suspended unless a TFM exists for * one of the algorithms proposed by this driver. @@ -454,6 +500,12 @@ static int sun4i_ss_probe(struct platform_device *pdev) break; } } + + /* Ignore error of debugfs */ + ss->dbgfs_dir = debugfs_create_dir("sun4i-ss", NULL); + ss->dbgfs_stats = debugfs_create_file("stats", 0444, ss->dbgfs_dir, ss, + &sun4i_ss_debugfs_fops); + return 0; error_alg: i--; diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c index 1dff48558f53..c1b4585e9bbc 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c @@ -191,8 +191,10 @@ static int sun4i_hash(struct ahash_request *areq) u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32] = {0}, v, ivmode = 0; struct sun4i_req_ctx *op = ahash_request_ctx(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); struct sun4i_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); struct sun4i_ss_ctx *ss = tfmctx->ss; + struct sun4i_ss_alg_template *algt; struct scatterlist *in_sg = areq->src; struct sg_mapping_iter mi; int in_r, err = 0; @@ -398,6 +400,10 @@ static int sun4i_hash(struct ahash_request *areq) */ hash_final: + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash); + algt->stat_req++; + } /* write the remaining words of the wait buffer */ if (op->len) { diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c index 729aafdbea84..443160a114bb 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later #include "sun4i-ss.h" int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, @@ -32,6 +33,11 @@ int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, if (err < 0) return err; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt->stat_req++; + algt->stat_bytes += todo; + } + spin_lock_bh(&ss->slock); writel(mode, ss->base + SS_CTL); diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h index 5c291e4a6857..0fee6f4e2d90 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h @@ -148,10 +148,14 @@ struct sun4i_ss_ctx { struct reset_control *reset; struct device *dev; struct resource *res; + char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ + char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ spinlock_t slock; /* control the use of the device */ #ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG u32 seed[SS_SEED_LEN / BITS_PER_LONG]; #endif + struct dentry *dbgfs_dir; + struct dentry *dbgfs_stats; }; struct sun4i_ss_alg_template { @@ -163,6 +167,10 @@ struct sun4i_ss_alg_template { struct rng_alg rng; } alg; struct sun4i_ss_ctx *ss; + unsigned long stat_req; + unsigned long stat_fb; + unsigned long stat_bytes; + unsigned long stat_opti; }; struct sun4i_tfm_ctx { diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 30390a7324b2..851b149f7170 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -42,7 +42,7 @@ /* ================= Device Structure ================== */ -struct device_private iproc_priv; +struct bcm_device_private iproc_priv; /* ==================== Parameters ===================== */ @@ -471,10 +471,8 @@ static int handle_skcipher_req(struct iproc_reqctx_s *rctx) static void handle_skcipher_resp(struct iproc_reqctx_s *rctx) { struct spu_hw *spu = &iproc_priv.spu; -#ifdef DEBUG struct crypto_async_request *areq = rctx->parent; struct skcipher_request *req = skcipher_request_cast(areq); -#endif struct iproc_ctx_s *ctx = rctx->ctx; u32 payload_len; @@ -996,13 +994,11 @@ static int ahash_req_done(struct iproc_reqctx_s *rctx) static void handle_ahash_resp(struct iproc_reqctx_s *rctx) { struct iproc_ctx_s *ctx = rctx->ctx; -#ifdef DEBUG struct crypto_async_request *areq = rctx->parent; struct ahash_request *req = ahash_request_cast(areq); struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); -#endif /* * Save hash to use as input to next op if incremental. Might be copying * too much, but that's easier than figuring out actual digest size here diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index 0ad5892b445d..71281a3bdbdc 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -420,7 +420,7 @@ struct spu_hw { u32 num_chan; }; -struct device_private { +struct bcm_device_private { struct platform_device *pdev; struct spu_hw spu; @@ -467,6 +467,6 @@ struct device_private { struct mbox_chan **mbox; }; -extern struct device_private iproc_priv; +extern struct bcm_device_private iproc_priv; #endif diff --git a/drivers/crypto/bcm/spu.c b/drivers/crypto/bcm/spu.c index fe126f95c702..007abf92cc05 100644 --- a/drivers/crypto/bcm/spu.c +++ b/drivers/crypto/bcm/spu.c @@ -41,7 +41,7 @@ void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len) packet_log("SPU Message header %p len: %u\n", buf, buf_len); /* ========== Decode MH ========== */ - packet_log(" MH 0x%08x\n", be32_to_cpu(*((u32 *)ptr))); + packet_log(" MH 0x%08x\n", be32_to_cpup((__be32 *)ptr)); if (spuh->mh.flags & MH_SCTX_PRES) packet_log(" SCTX present\n"); if (spuh->mh.flags & MH_BDESC_PRES) @@ -273,22 +273,21 @@ void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len) /* ========== Decode BDESC ========== */ if (spuh->mh.flags & MH_BDESC_PRES) { -#ifdef DEBUG struct BDESC_HEADER *bdesc = (struct BDESC_HEADER *)ptr; -#endif - packet_log(" BDESC[0] 0x%08x\n", be32_to_cpu(*((u32 *)ptr))); + + packet_log(" BDESC[0] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); packet_log(" OffsetMAC:%u LengthMAC:%u\n", be16_to_cpu(bdesc->offset_mac), be16_to_cpu(bdesc->length_mac)); ptr += sizeof(u32); - packet_log(" BDESC[1] 0x%08x\n", be32_to_cpu(*((u32 *)ptr))); + packet_log(" BDESC[1] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); packet_log(" OffsetCrypto:%u LengthCrypto:%u\n", be16_to_cpu(bdesc->offset_crypto), be16_to_cpu(bdesc->length_crypto)); ptr += sizeof(u32); - packet_log(" BDESC[2] 0x%08x\n", be32_to_cpu(*((u32 *)ptr))); + packet_log(" BDESC[2] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); packet_log(" OffsetICV:%u OffsetIV:%u\n", be16_to_cpu(bdesc->offset_icv), be16_to_cpu(bdesc->offset_iv)); @@ -297,10 +296,9 @@ void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len) /* ========== Decode BD ========== */ if (spuh->mh.flags & MH_BD_PRES) { -#ifdef DEBUG struct BD_HEADER *bd = (struct BD_HEADER *)ptr; -#endif - packet_log(" BD[0] 0x%08x\n", be32_to_cpu(*((u32 *)ptr))); + + packet_log(" BD[0] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); packet_log(" Size:%ubytes PrevLength:%u\n", be16_to_cpu(bd->size), be16_to_cpu(bd->prev_length)); ptr += 4; @@ -1056,9 +1054,9 @@ void spum_request_pad(u8 *pad_start, /* add the size at the end as required per alg */ if (auth_alg == HASH_ALG_MD5) - *(u64 *)ptr = cpu_to_le64((u64)total_sent * 8); + *(__le64 *)ptr = cpu_to_le64(total_sent * 8ull); else /* SHA1, SHA2-224, SHA2-256 */ - *(u64 *)ptr = cpu_to_be64((u64)total_sent * 8); + *(__be64 *)ptr = cpu_to_be64(total_sent * 8ull); ptr += sizeof(u64); } } diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c index c860ffb0b4c3..2db35b5ccaa2 100644 --- a/drivers/crypto/bcm/spu2.c +++ b/drivers/crypto/bcm/spu2.c @@ -964,7 +964,6 @@ u32 spu2_create_request(u8 *spu_hdr, unsigned int cipher_offset = aead_parms->assoc_size + aead_parms->aad_pad_len + aead_parms->iv_len; -#ifdef DEBUG /* total size of the data following OMD (without STAT word padding) */ unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size, aead_parms->iv_len, @@ -973,7 +972,6 @@ u32 spu2_create_request(u8 *spu_hdr, aead_parms->aad_pad_len, aead_parms->data_pad_len, hash_parms->pad_len); -#endif unsigned int assoc_size = aead_parms->assoc_size; if (req_opts->is_aead && @@ -1263,9 +1261,9 @@ void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len, /* add the size at the end as required per alg */ if (auth_alg == HASH_ALG_MD5) - *(u64 *)ptr = cpu_to_le64((u64)total_sent * 8); + *(__le64 *)ptr = cpu_to_le64(total_sent * 8ull); else /* SHA1, SHA2-224, SHA2-256 */ - *(u64 *)ptr = cpu_to_be64((u64)total_sent * 8); + *(__be64 *)ptr = cpu_to_be64(total_sent * 8ull); ptr += sizeof(u64); } diff --git a/drivers/crypto/bcm/spu2.h b/drivers/crypto/bcm/spu2.h index 6e666bfb3cfc..a76d4e054466 100644 --- a/drivers/crypto/bcm/spu2.h +++ b/drivers/crypto/bcm/spu2.h @@ -73,10 +73,10 @@ enum spu2_ret_md_opts { /* Fixed Metadata format */ struct SPU2_FMD { - u64 ctrl0; - u64 ctrl1; - u64 ctrl2; - u64 ctrl3; + __le64 ctrl0; + __le64 ctrl1; + __le64 ctrl2; + __le64 ctrl3; }; #define FMD_SIZE sizeof(struct SPU2_FMD) diff --git a/drivers/crypto/bcm/spum.h b/drivers/crypto/bcm/spum.h index 6116ad1dd26e..f062f75808de 100644 --- a/drivers/crypto/bcm/spum.h +++ b/drivers/crypto/bcm/spum.h @@ -69,18 +69,18 @@ /* Buffer Descriptor Header [BDESC]. SPU in big-endian mode. */ struct BDESC_HEADER { - u16 offset_mac; /* word 0 [31-16] */ - u16 length_mac; /* word 0 [15-0] */ - u16 offset_crypto; /* word 1 [31-16] */ - u16 length_crypto; /* word 1 [15-0] */ - u16 offset_icv; /* word 2 [31-16] */ - u16 offset_iv; /* word 2 [15-0] */ + __be16 offset_mac; /* word 0 [31-16] */ + __be16 length_mac; /* word 0 [15-0] */ + __be16 offset_crypto; /* word 1 [31-16] */ + __be16 length_crypto; /* word 1 [15-0] */ + __be16 offset_icv; /* word 2 [31-16] */ + __be16 offset_iv; /* word 2 [15-0] */ }; /* Buffer Data Header [BD]. SPU in big-endian mode. */ struct BD_HEADER { - u16 size; - u16 prev_length; + __be16 size; + __be16 prev_length; }; /* Command Context Header. SPU-M in big endian mode. */ @@ -144,13 +144,13 @@ struct MHEADER { /* Generic Mode Security Context Structure [SCTX] */ struct SCTX { /* word 0: protocol flags */ - u32 proto_flags; + __be32 proto_flags; /* word 1: cipher flags */ - u32 cipher_flags; + __be32 cipher_flags; /* word 2: Extended cipher flags */ - u32 ecf; + __be32 ecf; }; diff --git a/drivers/crypto/bcm/util.c b/drivers/crypto/bcm/util.c index 2b304fc78059..c4669a96eaec 100644 --- a/drivers/crypto/bcm/util.c +++ b/drivers/crypto/bcm/util.c @@ -268,6 +268,7 @@ do_shash_err: return rc; } +#ifdef DEBUG /* Dump len bytes of a scatterlist starting at skip bytes into the sg */ void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len) { @@ -289,6 +290,7 @@ void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len) if (debug_logging_sleep) msleep(debug_logging_sleep); } +#endif /* Returns the name for a given cipher alg/mode */ char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode) @@ -348,7 +350,7 @@ char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode) static ssize_t spu_debugfs_read(struct file *filp, char __user *ubuf, size_t count, loff_t *offp) { - struct device_private *ipriv; + struct bcm_device_private *ipriv; char *buf; ssize_t ret, out_offset, out_count; int i; diff --git a/drivers/crypto/bcm/util.h b/drivers/crypto/bcm/util.h index a89b2b9c1f52..61c256384816 100644 --- a/drivers/crypto/bcm/util.h +++ b/drivers/crypto/bcm/util.h @@ -58,12 +58,26 @@ void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len); #else /* !DEBUG_ON */ -#define flow_log(...) do {} while (0) -#define flow_dump(msg, var, var_len) do {} while (0) -#define packet_log(...) do {} while (0) -#define packet_dump(msg, var, var_len) do {} while (0) +static inline void flow_log(const char *format, ...) +{ +} -#define dump_sg(sg, skip, len) do {} while (0) +static inline void flow_dump(const char *msg, const void *var, size_t var_len) +{ +} + +static inline void packet_log(const char *format, ...) +{ +} + +static inline void packet_dump(const char *msg, const void *var, size_t var_len) +{ +} + +static inline void dump_sg(struct scatterlist *sg, unsigned int skip, + unsigned int len) +{ +} #endif /* DEBUG_ON */ diff --git a/drivers/crypto/caam/debugfs.c b/drivers/crypto/caam/debugfs.c index 8ebf18398166..806bb20d2aa1 100644 --- a/drivers/crypto/caam/debugfs.c +++ b/drivers/crypto/caam/debugfs.c @@ -19,8 +19,8 @@ static int caam_debugfs_u32_get(void *data, u64 *val) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); #ifdef CONFIG_CAAM_QI /* diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index f016448e43bb..112b12a32542 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -233,10 +233,10 @@ static int alloc_command_queues(struct cpt_vf *cptvf, c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes : rem_q_size; - curr->head = (u8 *)dma_alloc_coherent(&pdev->dev, - c_size + CPT_NEXT_CHUNK_PTR_SIZE, - &curr->dma_addr, - GFP_KERNEL); + curr->head = dma_alloc_coherent(&pdev->dev, + c_size + CPT_NEXT_CHUNK_PTR_SIZE, + &curr->dma_addr, + GFP_KERNEL); if (!curr->head) { dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n", i, queue->nchunks); diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index cdfee501fbd9..78833491f534 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -921,7 +921,7 @@ static int cc_cipher_process(struct skcipher_request *req, return crypto_skcipher_decrypt(subreq); } - /* The IV we are handed may be allocted from the stack so + /* The IV we are handed may be allocated from the stack so * we must copy it to a DMAable buffer before use. */ req_ctx->iv = kmemdup(iv, ivsize, flags); diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index 5f1d4602eb8f..f49579aa1452 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -23,7 +23,6 @@ #include #include #include -#include #include #include diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c index f4f18bfc2247..4ee010f39912 100644 --- a/drivers/crypto/geode-aes.c +++ b/drivers/crypto/geode-aes.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -434,3 +435,4 @@ module_pci_driver(geode_aes_driver); MODULE_AUTHOR("Advanced Micro Devices, Inc."); MODULE_DESCRIPTION("Geode LX Hardware AES driver"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index f69252b24671..181c109b19f7 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -14,8 +14,7 @@ enum { HPRE_CLUSTER0, HPRE_CLUSTER1, HPRE_CLUSTER2, - HPRE_CLUSTER3, - HPRE_CLUSTERS_NUM, + HPRE_CLUSTER3 }; enum hpre_ctrl_dbgfs_file { @@ -36,7 +35,10 @@ enum hpre_dfx_dbgfs_file { HPRE_DFX_FILE_NUM }; -#define HPRE_DEBUGFS_FILE_NUM (HPRE_DEBUG_FILE_NUM + HPRE_CLUSTERS_NUM - 1) +#define HPRE_CLUSTERS_NUM_V2 (HPRE_CLUSTER3 + 1) +#define HPRE_CLUSTERS_NUM_V3 1 +#define HPRE_CLUSTERS_NUM_MAX HPRE_CLUSTERS_NUM_V2 +#define HPRE_DEBUGFS_FILE_NUM (HPRE_DEBUG_FILE_NUM + HPRE_CLUSTERS_NUM_MAX - 1) struct hpre_debugfs_file { int index; diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index e5c991913f09..e7a2c70eb9cf 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "hpre.h" #define HPRE_QUEUE_NUM_V2 1024 @@ -29,13 +30,14 @@ #define HPRE_BD_ARUSR_CFG 0x301030 #define HPRE_BD_AWUSR_CFG 0x301034 #define HPRE_TYPES_ENB 0x301038 +#define HPRE_RSA_ENB BIT(0) +#define HPRE_ECC_ENB BIT(1) #define HPRE_DATA_RUSER_CFG 0x30103c #define HPRE_DATA_WUSER_CFG 0x301040 #define HPRE_INT_MASK 0x301400 #define HPRE_INT_STATUS 0x301800 #define HPRE_CORE_INT_ENABLE 0 #define HPRE_CORE_INT_DISABLE 0x003fffff -#define HPRE_RAS_ECC_1BIT_TH 0x30140c #define HPRE_RDCHN_INI_ST 0x301a00 #define HPRE_CLSTR_BASE 0x302000 #define HPRE_CORE_EN_OFFSET 0x04 @@ -45,7 +47,7 @@ #define HPRE_CORE_IS_SCHD_OFFSET 0x90 #define HPRE_RAS_CE_ENB 0x301410 -#define HPRE_HAC_RAS_CE_ENABLE 0x1 +#define HPRE_HAC_RAS_CE_ENABLE (BIT(0) | BIT(22) | BIT(23)) #define HPRE_RAS_NFE_ENB 0x301414 #define HPRE_HAC_RAS_NFE_ENABLE 0x3ffffe #define HPRE_RAS_FE_ENB 0x301418 @@ -73,7 +75,8 @@ #define HPRE_QM_AXI_CFG_MASK 0xffff #define HPRE_QM_VFG_AX_MASK 0xff #define HPRE_BD_USR_MASK 0x3 -#define HPRE_CLUSTER_CORE_MASK 0xf +#define HPRE_CLUSTER_CORE_MASK_V2 0xf +#define HPRE_CLUSTER_CORE_MASK_V3 0xff #define HPRE_AM_OOO_SHUTDOWN_ENB 0x301044 #define HPRE_AM_OOO_SHUTDOWN_ENABLE BIT(0) @@ -86,6 +89,11 @@ #define HPRE_QM_PM_FLR BIT(11) #define HPRE_QM_SRIOV_FLR BIT(12) +#define HPRE_CLUSTERS_NUM(qm) \ + (((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 : HPRE_CLUSTERS_NUM_V2) +#define HPRE_CLUSTER_CORE_MASK(qm) \ + (((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTER_CORE_MASK_V3 :\ + HPRE_CLUSTER_CORE_MASK_V2) #define HPRE_VIA_MSI_DSM 1 #define HPRE_SQE_MASK_OFFSET 8 #define HPRE_SQE_MASK_LEN 24 @@ -129,7 +137,11 @@ static const struct hpre_hw_error hpre_hw_errors[] = { { .int_msk = BIT(9), .msg = "cluster4_shb_timeout_int_set" }, { .int_msk = GENMASK(15, 10), .msg = "ooo_rdrsp_err_int_set" }, { .int_msk = GENMASK(21, 16), .msg = "ooo_wrrsp_err_int_set" }, - { /* sentinel */ } + { .int_msk = BIT(22), .msg = "pt_rng_timeout_int_set"}, + { .int_msk = BIT(23), .msg = "sva_fsm_timeout_int_set"}, + { + /* sentinel */ + } }; static const u64 hpre_cluster_offsets[] = { @@ -178,6 +190,19 @@ static const char *hpre_dfx_files[HPRE_DFX_FILE_NUM] = { "invalid_req_cnt" }; +static const struct kernel_param_ops hpre_uacce_mode_ops = { + .set = uacce_mode_set, + .get = param_get_int, +}; + +/* + * uacce_mode = 0 means hpre only register to crypto, + * uacce_mode = 1 means hpre both register to crypto and uacce. + */ +static u32 uacce_mode = UACCE_MODE_NOUACCE; +module_param_cb(uacce_mode, &hpre_uacce_mode_ops, &uacce_mode, 0444); +MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); + static int pf_q_num_set(const char *val, const struct kernel_param *kp) { return q_num_set(val, kp, HPRE_PCI_DEVICE_ID); @@ -214,6 +239,30 @@ struct hisi_qp *hpre_create_qp(void) return NULL; } +static void hpre_pasid_enable(struct hisi_qm *qm) +{ + u32 val; + + val = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG); + val |= BIT(HPRE_PASID_EN_BIT); + writel_relaxed(val, qm->io_base + HPRE_DATA_RUSER_CFG); + val = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG); + val |= BIT(HPRE_PASID_EN_BIT); + writel_relaxed(val, qm->io_base + HPRE_DATA_WUSER_CFG); +} + +static void hpre_pasid_disable(struct hisi_qm *qm) +{ + u32 val; + + val = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG); + val &= ~BIT(HPRE_PASID_EN_BIT); + writel_relaxed(val, qm->io_base + HPRE_DATA_RUSER_CFG); + val = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG); + val &= ~BIT(HPRE_PASID_EN_BIT); + writel_relaxed(val, qm->io_base + HPRE_DATA_WUSER_CFG); +} + static int hpre_cfg_by_dsm(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; @@ -238,8 +287,40 @@ static int hpre_cfg_by_dsm(struct hisi_qm *qm) return 0; } +static int hpre_set_cluster(struct hisi_qm *qm) +{ + u32 cluster_core_mask = HPRE_CLUSTER_CORE_MASK(qm); + u8 clusters_num = HPRE_CLUSTERS_NUM(qm); + struct device *dev = &qm->pdev->dev; + unsigned long offset; + u32 val = 0; + int ret, i; + + for (i = 0; i < clusters_num; i++) { + offset = i * HPRE_CLSTR_ADDR_INTRVL; + + /* clusters initiating */ + writel(cluster_core_mask, + HPRE_ADDR(qm, offset + HPRE_CORE_ENB)); + writel(0x1, HPRE_ADDR(qm, offset + HPRE_CORE_INI_CFG)); + ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, offset + + HPRE_CORE_INI_STATUS), val, + ((val & cluster_core_mask) == + cluster_core_mask), + HPRE_REG_RD_INTVRL_US, + HPRE_REG_RD_TMOUT_US); + if (ret) { + dev_err(dev, + "cluster %d int st status timeout!\n", i); + return -ETIMEDOUT; + } + } + + return 0; +} + /* - * For Hi1620, we shoul disable FLR triggered by hardware (BME/PM/SRIOV). + * For Kunpeng 920, we shoul disable FLR triggered by hardware (BME/PM/SRIOV). * Or it may stay in D3 state when we bind and unbind hpre quickly, * as it does FLR triggered by hardware. */ @@ -257,9 +338,8 @@ static void disable_flr_of_bme(struct hisi_qm *qm) static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - unsigned long offset; - int ret, i; u32 val; + int ret; writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_ARUSER_M_CFG_ENABLE)); writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_AWUSER_M_CFG_ENABLE)); @@ -270,11 +350,15 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) val |= BIT(HPRE_TIMEOUT_ABNML_BIT); writel_relaxed(val, HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK)); - writel(0x1, HPRE_ADDR(qm, HPRE_TYPES_ENB)); + if (qm->ver >= QM_HW_V3) + writel(HPRE_RSA_ENB | HPRE_ECC_ENB, + HPRE_ADDR(qm, HPRE_TYPES_ENB)); + else + writel(HPRE_RSA_ENB, HPRE_ADDR(qm, HPRE_TYPES_ENB)); + writel(HPRE_QM_VFG_AX_MASK, HPRE_ADDR(qm, HPRE_VFG_AXCACHE)); writel(0x0, HPRE_ADDR(qm, HPRE_BD_ENDIAN)); writel(0x0, HPRE_ADDR(qm, HPRE_INT_MASK)); - writel(0x0, HPRE_ADDR(qm, HPRE_RAS_ECC_1BIT_TH)); writel(0x0, HPRE_ADDR(qm, HPRE_POISON_BYPASS)); writel(0x0, HPRE_ADDR(qm, HPRE_COMM_CNT_CLR_CE)); writel(0x0, HPRE_ADDR(qm, HPRE_ECC_BYPASS)); @@ -291,37 +375,29 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) return -ETIMEDOUT; } - for (i = 0; i < HPRE_CLUSTERS_NUM; i++) { - offset = i * HPRE_CLSTR_ADDR_INTRVL; - - /* clusters initiating */ - writel(HPRE_CLUSTER_CORE_MASK, - HPRE_ADDR(qm, offset + HPRE_CORE_ENB)); - writel(0x1, HPRE_ADDR(qm, offset + HPRE_CORE_INI_CFG)); - ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, offset + - HPRE_CORE_INI_STATUS), val, - ((val & HPRE_CLUSTER_CORE_MASK) == - HPRE_CLUSTER_CORE_MASK), - HPRE_REG_RD_INTVRL_US, - HPRE_REG_RD_TMOUT_US); - if (ret) { - dev_err(dev, - "cluster %d int st status timeout!\n", i); - return -ETIMEDOUT; - } - } - - ret = hpre_cfg_by_dsm(qm); + ret = hpre_set_cluster(qm); if (ret) - dev_err(dev, "acpi_evaluate_dsm err.\n"); + return -ETIMEDOUT; - disable_flr_of_bme(qm); + /* This setting is only needed by Kunpeng 920. */ + if (qm->ver == QM_HW_V2) { + ret = hpre_cfg_by_dsm(qm); + if (ret) + dev_err(dev, "acpi_evaluate_dsm err.\n"); + + disable_flr_of_bme(qm); + + /* Enable data buffer pasid */ + if (qm->use_sva) + hpre_pasid_enable(qm); + } return ret; } static void hpre_cnt_regs_clear(struct hisi_qm *qm) { + u8 clusters_num = HPRE_CLUSTERS_NUM(qm); unsigned long offset; int i; @@ -330,7 +406,7 @@ static void hpre_cnt_regs_clear(struct hisi_qm *qm) writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); /* clear clusterX/cluster_ctrl */ - for (i = 0; i < HPRE_CLUSTERS_NUM; i++) { + for (i = 0; i < clusters_num; i++) { offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL; writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY); } @@ -629,13 +705,14 @@ static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm) static int hpre_cluster_debugfs_init(struct hisi_qm *qm) { + u8 clusters_num = HPRE_CLUSTERS_NUM(qm); struct device *dev = &qm->pdev->dev; char buf[HPRE_DBGFS_VAL_MAX_LEN]; struct debugfs_regset32 *regset; struct dentry *tmp_d; int i, ret; - for (i = 0; i < HPRE_CLUSTERS_NUM; i++) { + for (i = 0; i < clusters_num; i++) { ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i); if (ret < 0) return -EINVAL; @@ -734,6 +811,11 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return -EINVAL; } + if (pdev->revision >= QM_HW_V3) + qm->algs = "rsa\ndh\necdh\nx25519\nx448\necdsa\nsm2\n"; + else + qm->algs = "rsa\ndh\n"; + qm->mode = uacce_mode; qm->pdev = pdev; qm->ver = pdev->revision; qm->sqe_size = HPRE_SQE_SIZE; @@ -799,6 +881,7 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .fe = 0, .ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | HPRE_OOO_ECC_2BIT_ERR, + .dev_ce_mask = HPRE_HAC_RAS_CE_ENABLE, .msi_wr_port = HPRE_WR_MSI_PORT, .acpi_rst = "HRST", } @@ -872,6 +955,14 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_with_qm_start; } + if (qm->uacce) { + ret = uacce_register(qm->uacce); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_with_alg_register; + } + } + if (qm->fun_type == QM_HW_PF && vfs_num) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) @@ -904,20 +995,24 @@ static void hpre_remove(struct pci_dev *pdev) hisi_qm_wait_task_finish(qm, &hpre_devices); hisi_qm_alg_unregister(qm, &hpre_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) { - ret = hisi_qm_sriov_disable(pdev, qm->is_frozen); + ret = hisi_qm_sriov_disable(pdev, true); if (ret) { pci_err(pdev, "Disable SRIOV fail!\n"); return; } } - if (qm->fun_type == QM_HW_PF) { - hpre_cnt_regs_clear(qm); - qm->debug.curr_qm_qp_num = 0; - } hpre_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); - hisi_qm_dev_err_uninit(qm); + + if (qm->fun_type == QM_HW_PF) { + if (qm->use_sva && qm->ver == QM_HW_V2) + hpre_pasid_disable(qm); + hpre_cnt_regs_clear(qm); + qm->debug.curr_qm_qp_num = 0; + hisi_qm_dev_err_uninit(qm); + } + hisi_qm_uninit(qm); } diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index f21ccae0e8ea..13cb4216561a 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -54,6 +54,8 @@ #define QM_SQ_PRIORITY_SHIFT 0 #define QM_SQ_ORDERS_SHIFT 4 #define QM_SQ_TYPE_SHIFT 8 +#define QM_QC_PASID_ENABLE 0x1 +#define QM_QC_PASID_ENABLE_SHIFT 7 #define QM_SQ_TYPE_MASK GENMASK(3, 0) #define QM_SQ_TAIL_IDX(sqc) ((le16_to_cpu((sqc)->w11) >> 6) & 0x1) @@ -120,7 +122,7 @@ #define QM_CQC_VFT_VALID (1ULL << 28) #define QM_SQC_VFT_BASE_SHIFT_V2 28 -#define QM_SQC_VFT_BASE_MASK_V2 GENMASK(5, 0) +#define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) #define QM_SQC_VFT_NUM_SHIFT_V2 45 #define QM_SQC_VFT_NUM_MASK_v2 GENMASK(9, 0) @@ -147,7 +149,6 @@ #define QM_RAS_CE_TIMES_PER_IRQ 1 #define QM_RAS_MSI_INT_SEL 0x1040f4 -#define QM_DEV_RESET_FLAG 0 #define QM_RESET_WAIT_TIMEOUT 400 #define QM_PEH_VENDOR_ID 0x1000d8 #define ACC_VENDOR_ID_VALUE 0x5a5a @@ -185,6 +186,10 @@ #define QM_SQE_ADDR_MASK GENMASK(7, 0) #define QM_EQ_DEPTH (1024 * 2) +#define QM_DRIVER_REMOVING 0 +#define QM_RST_SCHED 1 +#define QM_RESETTING 2 + #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \ @@ -619,6 +624,9 @@ static void qm_cq_head_update(struct hisi_qp *qp) static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm) { + if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP)) + return; + if (qp->event_cb) { qp->event_cb(qp); return; @@ -717,7 +725,7 @@ static irqreturn_t qm_aeq_irq(int irq, void *data) dev_err(&qm->pdev->dev, "%s overflow\n", qm_fifo_overflow[type]); else - dev_err(&qm->pdev->dev, "unknown error type %d\n", + dev_err(&qm->pdev->dev, "unknown error type %u\n", type); if (qm->status.aeq_head == QM_Q_DEPTH - 1) { @@ -1121,7 +1129,7 @@ static int dump_show(struct hisi_qm *qm, void *info, dev_info(dev, "%s DUMP\n", info_name); for (i = 0; i < info_size; i += BYTE_PER_DW) { - pr_info("DW%d: %02X%02X %02X%02X\n", i / BYTE_PER_DW, + pr_info("DW%u: %02X%02X %02X%02X\n", i / BYTE_PER_DW, info_buf[i], info_buf[i + 1UL], info_buf[i + 2UL], info_buf[i + 3UL]); } @@ -1154,7 +1162,7 @@ static int qm_sqc_dump(struct hisi_qm *qm, const char *s) ret = kstrtou32(s, 0, &qp_id); if (ret || qp_id >= qm->qp_num) { - dev_err(dev, "Please input qp num (0-%d)", qm->qp_num - 1); + dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1); return -EINVAL; } @@ -1200,7 +1208,7 @@ static int qm_cqc_dump(struct hisi_qm *qm, const char *s) ret = kstrtou32(s, 0, &qp_id); if (ret || qp_id >= qm->qp_num) { - dev_err(dev, "Please input qp num (0-%d)", qm->qp_num - 1); + dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1); return -EINVAL; } @@ -1279,7 +1287,7 @@ static int q_dump_param_parse(struct hisi_qm *qm, char *s, ret = kstrtou32(presult, 0, q_id); if (ret || *q_id >= qp_num) { - dev_err(dev, "Please input qp num (0-%d)", qp_num - 1); + dev_err(dev, "Please input qp num (0-%u)", qp_num - 1); return -EINVAL; } @@ -1604,7 +1612,7 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) { - u32 error_status, tmp; + u32 error_status, tmp, val; /* read err sts */ tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS); @@ -1615,9 +1623,13 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) qm->err_status.is_qm_ecc_mbit = true; qm_log_hw_error(qm, error_status); - if (error_status == QM_DB_RANDOM_INVALID) { + val = error_status | QM_DB_RANDOM_INVALID | QM_BASE_CE; + /* ce error does not need to be reset */ + if (val == (QM_DB_RANDOM_INVALID | QM_BASE_CE)) { writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm->err_ini->err_info.nfe, + qm->io_base + QM_RAS_NFE_ENABLE); return ACC_ERR_RECOVERED; } @@ -1685,6 +1697,7 @@ static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) qp->req_cb = NULL; qp->qp_id = qp_id; qp->alg_type = alg_type; + qp->is_in_kernel = true; qm->qp_in_used++; atomic_set(&qp->qp_status.flags, QP_INIT); @@ -1747,12 +1760,6 @@ static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL); if (!sqc) return -ENOMEM; - sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc), - DMA_TO_DEVICE); - if (dma_mapping_error(dev, sqc_dma)) { - kfree(sqc); - return -ENOMEM; - } INIT_QC_COMMON(sqc, qp->sqe_dma, pasid); if (ver == QM_HW_V1) { @@ -1765,6 +1772,17 @@ static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) sqc->cq_num = cpu_to_le16(qp_id); sqc->w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type)); + if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) + sqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE << + QM_QC_PASID_ENABLE_SHIFT); + + sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc), + DMA_TO_DEVICE); + if (dma_mapping_error(dev, sqc_dma)) { + kfree(sqc); + return -ENOMEM; + } + ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0); dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE); kfree(sqc); @@ -1784,12 +1802,6 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL); if (!cqc) return -ENOMEM; - cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc), - DMA_TO_DEVICE); - if (dma_mapping_error(dev, cqc_dma)) { - kfree(cqc); - return -ENOMEM; - } INIT_QC_COMMON(cqc, qp->cqe_dma, pasid); if (ver == QM_HW_V1) { @@ -1802,6 +1814,16 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) } cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); + if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) + cqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE); + + cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc), + DMA_TO_DEVICE); + if (dma_mapping_error(dev, cqc_dma)) { + kfree(cqc); + return -ENOMEM; + } + ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0); dma_unmap_single(dev, cqc_dma, sizeof(struct qm_cqc), DMA_TO_DEVICE); kfree(cqc); @@ -1864,6 +1886,28 @@ int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) } EXPORT_SYMBOL_GPL(hisi_qm_start_qp); +/** + * qp_stop_fail_cb() - call request cb. + * @qp: stopped failed qp. + * + * Callback function should be called whether task completed or not. + */ +static void qp_stop_fail_cb(struct hisi_qp *qp) +{ + int qp_used = atomic_read(&qp->qp_status.used); + u16 cur_tail = qp->qp_status.sq_tail; + u16 cur_head = (cur_tail + QM_Q_DEPTH - qp_used) % QM_Q_DEPTH; + struct hisi_qm *qm = qp->qm; + u16 pos; + int i; + + for (i = 0; i < qp_used; i++) { + pos = (i + cur_head) % QM_Q_DEPTH; + qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); + atomic_dec(&qp->qp_status.used); + } +} + /** * qm_drain_qp() - Drain a qp. * @qp: The qp we want to drain. @@ -1959,6 +2003,9 @@ static int qm_stop_qp_nolock(struct hisi_qp *qp) else flush_work(&qp->qm->work); + if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used))) + qp_stop_fail_cb(qp); + dev_dbg(dev, "stop queue %u!", qp->qp_id); return 0; @@ -2065,6 +2112,7 @@ static int hisi_qm_uacce_get_queue(struct uacce_device *uacce, qp->uacce_q = q; qp->event_cb = qm_qp_event_notifier; qp->pasid = arg; + qp->is_in_kernel = false; return 0; } @@ -2206,7 +2254,7 @@ static int qm_alloc_uacce(struct hisi_qm *qm) if (IS_ERR(uacce)) return PTR_ERR(uacce); - if (uacce->flags & UACCE_DEV_SVA) { + if (uacce->flags & UACCE_DEV_SVA && qm->mode == UACCE_MODE_SVA) { qm->use_sva = true; } else { /* only consider sva case */ @@ -2248,17 +2296,15 @@ static int qm_alloc_uacce(struct hisi_qm *qm) */ static int qm_frozen(struct hisi_qm *qm) { - down_write(&qm->qps_lock); - - if (qm->is_frozen) { - up_write(&qm->qps_lock); + if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) return 0; - } + + down_write(&qm->qps_lock); if (!qm->qp_in_used) { qm->qp_in_used = qm->qp_num; - qm->is_frozen = true; up_write(&qm->qps_lock); + set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl); return 0; } @@ -2311,6 +2357,10 @@ void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) msleep(WAIT_PERIOD); } + while (test_bit(QM_RST_SCHED, &qm->misc_ctl) || + test_bit(QM_RESETTING, &qm->misc_ctl)) + msleep(WAIT_PERIOD); + udelay(REMOVE_WAIT_DELAY); } EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); @@ -2439,7 +2489,7 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) mutex_init(&qm->mailbox_lock); init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; - qm->is_frozen = false; + qm->misc_ctl = false; } static void hisi_qm_pci_uninit(struct hisi_qm *qm) @@ -2558,15 +2608,9 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) dma_addr_t eqc_dma; int ret; - eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); //todo + eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); if (!eqc) return -ENOMEM; - eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc), - DMA_TO_DEVICE); - if (dma_mapping_error(dev, eqc_dma)) { - kfree(eqc); - return -ENOMEM; - } eqc->base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma)); eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); @@ -2574,6 +2618,13 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc), + DMA_TO_DEVICE); + if (dma_mapping_error(dev, eqc_dma)) { + kfree(eqc); + return -ENOMEM; + } + ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE); kfree(eqc); @@ -2591,6 +2642,11 @@ static int qm_aeq_ctx_cfg(struct hisi_qm *qm) aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL); if (!aeqc) return -ENOMEM; + + aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma)); + aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma)); + aeqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc), DMA_TO_DEVICE); if (dma_mapping_error(dev, aeqc_dma)) { @@ -2598,10 +2654,6 @@ static int qm_aeq_ctx_cfg(struct hisi_qm *qm) return -ENOMEM; } - aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma)); - aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma)); - aeqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); - ret = qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0); dma_unmap_single(dev, aeqc_dma, sizeof(struct qm_aeqc), DMA_TO_DEVICE); kfree(aeqc); @@ -2677,7 +2729,7 @@ int hisi_qm_start(struct hisi_qm *qm) return -EPERM; } - dev_dbg(dev, "qm start with %d queue pairs\n", qm->qp_num); + dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num); if (!qm->qp_num) { dev_err(dev, "qp_num should not be 0\n"); @@ -3112,7 +3164,7 @@ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, mutex_unlock(&qm_list->lock); if (ret) - pr_info("Failed to create qps, node[%d], alg[%d], qp[%d]!\n", + pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n", node, alg_type, qp_num); err: @@ -3248,7 +3300,7 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs) { if (num_vfs == 0) - return hisi_qm_sriov_disable(pdev, 0); + return hisi_qm_sriov_disable(pdev, false); else return hisi_qm_sriov_enable(pdev, num_vfs); } @@ -3269,12 +3321,19 @@ static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) if (err_sts & qm->err_ini->err_info.ecc_2bits_mask) qm->err_status.is_dev_ecc_mbit = true; - if (!qm->err_ini->log_dev_hw_err) { - dev_err(&qm->pdev->dev, "Device doesn't support log hw error!\n"); - return ACC_ERR_NEED_RESET; + if (qm->err_ini->log_dev_hw_err) + qm->err_ini->log_dev_hw_err(qm, err_sts); + + /* ce error does not need to be reset */ + if ((err_sts | qm->err_ini->err_info.dev_ce_mask) == + qm->err_ini->err_info.dev_ce_mask) { + if (qm->err_ini->clear_dev_hw_err_status) + qm->err_ini->clear_dev_hw_err_status(qm, + err_sts); + + return ACC_ERR_RECOVERED; } - qm->err_ini->log_dev_hw_err(qm, err_sts); return ACC_ERR_NEED_RESET; } @@ -3313,7 +3372,7 @@ pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, if (pdev->is_virtfn) return PCI_ERS_RESULT_NONE; - pci_info(pdev, "PCI error detected, state(=%d)!!\n", state); + pci_info(pdev, "PCI error detected, state(=%u)!!\n", state); if (state == pci_channel_io_perm_failure) return PCI_ERS_RESULT_DISCONNECT; @@ -3465,7 +3524,7 @@ static int qm_reset_prepare_ready(struct hisi_qm *qm) int delay = 0; /* All reset requests need to be queued for processing */ - while (test_and_set_bit(QM_DEV_RESET_FLAG, &pf_qm->reset_flag)) { + while (test_and_set_bit(QM_RESETTING, &pf_qm->misc_ctl)) { msleep(++delay); if (delay > QM_RESET_WAIT_TIMEOUT) return -EBUSY; @@ -3489,6 +3548,7 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop VFs!\n"); + clear_bit(QM_RESETTING, &qm->misc_ctl); return ret; } } @@ -3496,9 +3556,12 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) ret = hisi_qm_stop(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop QM!\n"); + clear_bit(QM_RESETTING, &qm->misc_ctl); return ret; } + clear_bit(QM_RST_SCHED, &qm->misc_ctl); + return 0; } @@ -3736,7 +3799,7 @@ static int qm_controller_reset_done(struct hisi_qm *qm) hisi_qm_dev_err_init(qm); qm_restart_done(qm); - clear_bit(QM_DEV_RESET_FLAG, &qm->reset_flag); + clear_bit(QM_RESETTING, &qm->misc_ctl); return 0; } @@ -3749,18 +3812,23 @@ static int qm_controller_reset(struct hisi_qm *qm) pci_info(pdev, "Controller resetting...\n"); ret = qm_controller_reset_prepare(qm); - if (ret) + if (ret) { + clear_bit(QM_RST_SCHED, &qm->misc_ctl); return ret; + } ret = qm_soft_reset(qm); if (ret) { pci_err(pdev, "Controller reset failed (%d)\n", ret); + clear_bit(QM_RESETTING, &qm->misc_ctl); return ret; } ret = qm_controller_reset_done(qm); - if (ret) + if (ret) { + clear_bit(QM_RESETTING, &qm->misc_ctl); return ret; + } pci_info(pdev, "Controller reset complete\n"); @@ -3867,8 +3935,6 @@ static bool qm_flr_reset_complete(struct pci_dev *pdev) return false; } - clear_bit(QM_DEV_RESET_FLAG, &qm->reset_flag); - return true; } @@ -3912,6 +3978,8 @@ void hisi_qm_reset_done(struct pci_dev *pdev) flr_done: if (qm_flr_reset_complete(pdev)) pci_info(pdev, "FLR reset complete\n"); + + clear_bit(QM_RESETTING, &qm->misc_ctl); } EXPORT_SYMBOL_GPL(hisi_qm_reset_done); @@ -3922,7 +3990,9 @@ static irqreturn_t qm_abnormal_irq(int irq, void *data) atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt); ret = qm_process_dev_error(qm); - if (ret == ACC_ERR_NEED_RESET) + if (ret == ACC_ERR_NEED_RESET && + !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) && + !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl)) schedule_work(&qm->rst_work); return IRQ_HANDLED; @@ -3934,21 +4004,20 @@ static int qm_irq_register(struct hisi_qm *qm) int ret; ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), - qm_irq, IRQF_SHARED, qm->dev_name, qm); + qm_irq, 0, qm->dev_name, qm); if (ret) return ret; if (qm->ver != QM_HW_V1) { ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), - qm_aeq_irq, IRQF_SHARED, qm->dev_name, qm); + qm_aeq_irq, 0, qm->dev_name, qm); if (ret) goto err_aeq_irq; if (qm->fun_type == QM_HW_PF) { ret = request_irq(pci_irq_vector(pdev, QM_ABNORMAL_EVENT_IRQ_VECTOR), - qm_abnormal_irq, IRQF_SHARED, - qm->dev_name, qm); + qm_abnormal_irq, 0, qm->dev_name, qm); if (ret) goto err_abonormal_irq; } @@ -4004,6 +4073,9 @@ int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) { int flag = 0; int ret = 0; + /* HW V2 not support both use uacce sva mode and hardware crypto algs */ + if (qm->ver <= QM_HW_V2 && qm->use_sva) + return 0; mutex_lock(&qm_list->lock); if (list_empty(&qm_list->list)) @@ -4035,6 +4107,9 @@ EXPORT_SYMBOL_GPL(hisi_qm_alg_register); */ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) { + if (qm->ver <= QM_HW_V2 && qm->use_sva) + return; + mutex_lock(&qm_list->lock); list_del(&qm->list); mutex_unlock(&qm_list->lock); diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 8624d1288afe..54967c6b9c78 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -85,6 +85,11 @@ /* page number for queue file region */ #define QM_DOORBELL_PAGE_NR 1 +/* uacce mode of the driver */ +#define UACCE_MODE_NOUACCE 0 /* don't use uacce */ +#define UACCE_MODE_SVA 1 /* use uacce sva mode */ +#define UACCE_MODE_DESC "0(default) means only register to crypto, 1 means both register to crypto and uacce" + enum qm_stop_reason { QM_NORMAL, QM_SOFT_RESET, @@ -168,6 +173,7 @@ struct hisi_qm_err_info { char *acpi_rst; u32 msi_wr_port; u32 ecc_2bits_mask; + u32 dev_ce_mask; u32 ce; u32 nfe; u32 fe; @@ -225,7 +231,7 @@ struct hisi_qm { struct hisi_qm_status status; const struct hisi_qm_err_ini *err_ini; struct hisi_qm_err_status err_status; - unsigned long reset_flag; + unsigned long misc_ctl; /* driver removing and reset sched */ struct rw_semaphore qps_lock; struct idr qp_idr; @@ -249,6 +255,7 @@ struct hisi_qm { resource_size_t phys_base; resource_size_t phys_size; struct uacce_device *uacce; + int mode; }; struct hisi_qp_status { @@ -282,6 +289,7 @@ struct hisi_qp { struct hisi_qm *qm; bool is_resetting; + bool is_in_kernel; u16 pasid; struct uacce_queue *uacce_q; }; @@ -299,7 +307,7 @@ static inline int q_num_set(const char *val, const struct kernel_param *kp, if (!pdev) { q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2); - pr_info("No device found currently, suppose queue number is %d\n", + pr_info("No device found currently, suppose queue number is %u\n", q_num); } else { if (pdev->revision == QM_HW_V1) @@ -333,6 +341,27 @@ static inline int vfs_num_set(const char *val, const struct kernel_param *kp) return param_set_int(val, kp); } +static inline int mode_set(const char *val, const struct kernel_param *kp) +{ + u32 n; + int ret; + + if (!val) + return -EINVAL; + + ret = kstrtou32(val, 10, &n); + if (ret != 0 || (n != UACCE_MODE_SVA && + n != UACCE_MODE_NOUACCE)) + return -EINVAL; + + return param_set_int(val, kp); +} + +static inline int uacce_mode_set(const char *val, const struct kernel_param *kp) +{ + return mode_set(val, kp); +} + static inline void hisi_qm_init_list(struct hisi_qm_list *qm_list) { INIT_LIST_HEAD(&qm_list->list); diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index b35c1c2271a3..dc68ba76f65e 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "sec.h" @@ -74,6 +75,16 @@ #define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15)) #define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7)) +#define SEC_USER1_ENABLE_CONTEXT_SSV BIT(24) +#define SEC_USER1_ENABLE_DATA_SSV BIT(16) +#define SEC_USER1_WB_CONTEXT_SSV BIT(8) +#define SEC_USER1_WB_DATA_SSV BIT(0) +#define SEC_USER1_SVA_SET (SEC_USER1_ENABLE_CONTEXT_SSV | \ + SEC_USER1_ENABLE_DATA_SSV | \ + SEC_USER1_WB_CONTEXT_SSV | \ + SEC_USER1_WB_DATA_SSV) +#define SEC_USER1_SMMU_SVA (SEC_USER1_SMMU_NORMAL | SEC_USER1_SVA_SET) +#define SEC_USER1_SMMU_MASK (~SEC_USER1_SVA_SET) #define SEC_CORE_INT_STATUS_M_ECC BIT(2) #define SEC_DELAY_10_US 10 @@ -233,6 +244,18 @@ struct hisi_qp **sec_create_qps(void) return NULL; } +static const struct kernel_param_ops sec_uacce_mode_ops = { + .set = uacce_mode_set, + .get = param_get_int, +}; + +/* + * uacce_mode = 0 means sec only register to crypto, + * uacce_mode = 1 means sec both register to crypto and uacce. + */ +static u32 uacce_mode = UACCE_MODE_NOUACCE; +module_param_cb(uacce_mode, &sec_uacce_mode_ops, &uacce_mode, 0444); +MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); static const struct pci_device_id sec_dev_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_PF_PCI_DEVICE_ID) }, @@ -299,7 +322,11 @@ static int sec_engine_init(struct hisi_qm *qm) writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG)); reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG)); - reg |= SEC_USER1_SMMU_NORMAL; + reg &= SEC_USER1_SMMU_MASK; + if (qm->use_sva && qm->ver == QM_HW_V2) + reg |= SEC_USER1_SMMU_SVA; + else + reg |= SEC_USER1_SMMU_NORMAL; writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG)); writel(SEC_SINGLE_PORT_MAX_TRANS, @@ -725,6 +752,7 @@ static const struct hisi_qm_err_ini sec_err_ini = { QM_ACC_WB_NOT_READY_TIMEOUT, .fe = 0, .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, + .dev_ce_mask = SEC_RAS_CE_ENB_MSK, .msi_wr_port = BIT(0), .acpi_rst = "SRST", } @@ -758,6 +786,8 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->pdev = pdev; qm->ver = pdev->revision; + qm->algs = "cipher\ndigest\naead\n"; + qm->mode = uacce_mode; qm->sqe_size = SEC_SQE_SIZE; qm->dev_name = sec_name; @@ -885,6 +915,14 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_qm_stop; } + if (qm->uacce) { + ret = uacce_register(qm->uacce); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_alg_unregister; + } + } + if (qm->fun_type == QM_HW_PF && vfs_num) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) @@ -912,7 +950,7 @@ static void sec_remove(struct pci_dev *pdev) hisi_qm_wait_task_finish(qm, &sec_devices); hisi_qm_alg_unregister(qm, &sec_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) - hisi_qm_sriov_disable(pdev, qm->is_frozen); + hisi_qm_sriov_disable(pdev, true); sec_debugfs_exit(qm); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 4fb5a32bf830..02c445722445 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -66,6 +66,7 @@ #define HZIP_CORE_INT_STATUS_M_ECC BIT(1) #define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148 #define HZIP_CORE_INT_RAS_CE_ENB 0x301160 +#define HZIP_CORE_INT_RAS_CE_ENABLE 0x1 #define HZIP_CORE_INT_RAS_NFE_ENB 0x301164 #define HZIP_CORE_INT_RAS_FE_ENB 0x301168 #define HZIP_CORE_INT_RAS_NFE_ENABLE 0x7FE @@ -211,6 +212,19 @@ static const struct debugfs_reg32 hzip_dfx_regs[] = { {"HZIP_DECOMP_LZ77_CURR_ST ", 0x9cull}, }; +static const struct kernel_param_ops zip_uacce_mode_ops = { + .set = uacce_mode_set, + .get = param_get_int, +}; + +/* + * uacce_mode = 0 means zip only register to crypto, + * uacce_mode = 1 means zip both register to crypto and uacce. + */ +static u32 uacce_mode = UACCE_MODE_NOUACCE; +module_param_cb(uacce_mode, &zip_uacce_mode_ops, &uacce_mode, 0444); +MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); + static int pf_q_num_set(const char *val, const struct kernel_param *kp) { return q_num_set(val, kp, PCI_DEVICE_ID_ZIP_PF); @@ -279,7 +293,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63); - if (qm->use_sva) { + if (qm->use_sva && qm->ver == QM_HW_V2) { writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63); writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63); } else { @@ -314,7 +328,8 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_SOURCE); /* configure error type */ - writel(0x1, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); + writel(HZIP_CORE_INT_RAS_CE_ENABLE, + qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); writel(0x0, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); writel(HZIP_CORE_INT_RAS_NFE_ENABLE, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); @@ -714,6 +729,7 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = { QM_ACC_WB_NOT_READY_TIMEOUT, .fe = 0, .ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC, + .dev_ce_mask = HZIP_CORE_INT_RAS_CE_ENABLE, .msi_wr_port = HZIP_WR_PORT, .acpi_rst = "ZRST", } @@ -752,6 +768,7 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->pdev = pdev; qm->ver = pdev->revision; qm->algs = "zlib\ngzip"; + qm->mode = uacce_mode; qm->sqe_size = HZIP_SQE_SIZE; qm->dev_name = hisi_zip_name; @@ -887,7 +904,7 @@ static void hisi_zip_remove(struct pci_dev *pdev) hisi_qm_alg_unregister(qm, &zip_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) - hisi_qm_sriov_disable(pdev, qm->is_frozen); + hisi_qm_sriov_disable(pdev, true); hisi_zip_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 2e1562108a85..6364583b88b2 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1166,11 +1166,8 @@ static int safexcel_request_ring_irq(void *pdev, int irqid, dev = &plf_pdev->dev; irq = platform_get_irq_byname(plf_pdev, irq_name); - if (irq < 0) { - dev_err(dev, "unable to get IRQ '%s' (err %d)\n", - irq_name, irq); + if (irq < 0) return irq; - } } else { return -ENXIO; } @@ -1999,3 +1996,4 @@ MODULE_AUTHOR("Ofer Heifetz "); MODULE_AUTHOR("Igal Liberman "); MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 50fb6d90a2e0..bc60b5802256 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig index f2e17b0c4fa0..00cf8f028cb9 100644 --- a/drivers/crypto/keembay/Kconfig +++ b/drivers/crypto/keembay/Kconfig @@ -38,3 +38,34 @@ config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS Provides OCS version of cts(cbc(aes)) and cts(cbc(sm4)). Intel does not recommend use of CTS mode with AES/SM4. + +config CRYPTO_DEV_KEEMBAY_OCS_HCU + tristate "Support for Intel Keem Bay OCS HCU HW acceleration" + select CRYPTO_HASH + select CRYPTO_ENGINE + depends on HAS_IOMEM + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on OF || COMPILE_TEST + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash + Control Unit (HCU) hardware acceleration for use with Crypto API. + + Provides OCS HCU hardware acceleration of sha256, sha384, sha512, and + sm3, as well as the HMAC variant of these algorithms. + + Say Y or M if you're building for the Intel Keem Bay SoC. If compiled + as a module, the module will be called keembay-ocs-hcu. + + If unsure, say N. + +config CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + bool "Enable sha224 and hmac(sha224) support in Intel Keem Bay OCS HCU" + depends on CRYPTO_DEV_KEEMBAY_OCS_HCU + help + Enables support for sha224 and hmac(sha224) algorithms in the Intel + Keem Bay OCS HCU driver. Intel recommends not to use these + algorithms. + + Provides OCS HCU hardware acceleration of sha224 and hmac(224). + + If unsure, say N. diff --git a/drivers/crypto/keembay/Makefile b/drivers/crypto/keembay/Makefile index f21e2c4ab3b3..aea03d4432c4 100644 --- a/drivers/crypto/keembay/Makefile +++ b/drivers/crypto/keembay/Makefile @@ -3,3 +3,6 @@ # obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o + +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU) += keembay-ocs-hcu.o +keembay-ocs-hcu-objs := keembay-ocs-hcu-core.o ocs-hcu.o diff --git a/drivers/crypto/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/keembay/keembay-ocs-hcu-core.c new file mode 100644 index 000000000000..c4b97b4160e9 --- /dev/null +++ b/drivers/crypto/keembay/keembay-ocs-hcu-core.c @@ -0,0 +1,1264 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "ocs-hcu.h" + +#define DRV_NAME "keembay-ocs-hcu" + +/* Flag marking a final request. */ +#define REQ_FINAL BIT(0) +/* Flag marking a HMAC request. */ +#define REQ_FLAGS_HMAC BIT(1) +/* Flag set when HW HMAC is being used. */ +#define REQ_FLAGS_HMAC_HW BIT(2) +/* Flag set when SW HMAC is being used. */ +#define REQ_FLAGS_HMAC_SW BIT(3) + +/** + * struct ocs_hcu_ctx: OCS HCU Transform context. + * @engine_ctx: Crypto Engine context. + * @hcu_dev: The OCS HCU device used by the transformation. + * @key: The key (used only for HMAC transformations). + * @key_len: The length of the key. + * @is_sm3_tfm: Whether or not this is an SM3 transformation. + * @is_hmac_tfm: Whether or not this is a HMAC transformation. + */ +struct ocs_hcu_ctx { + struct crypto_engine_ctx engine_ctx; + struct ocs_hcu_dev *hcu_dev; + u8 key[SHA512_BLOCK_SIZE]; + size_t key_len; + bool is_sm3_tfm; + bool is_hmac_tfm; +}; + +/** + * struct ocs_hcu_rctx - Context for the request. + * @hcu_dev: OCS HCU device to be used to service the request. + * @flags: Flags tracking request status. + * @algo: Algorithm to use for the request. + * @blk_sz: Block size of the transformation / request. + * @dig_sz: Digest size of the transformation / request. + * @dma_list: OCS DMA linked list. + * @hash_ctx: OCS HCU hashing context. + * @buffer: Buffer to store: partial block of data and SW HMAC + * artifacts (ipad, opad, etc.). + * @buf_cnt: Number of bytes currently stored in the buffer. + * @buf_dma_addr: The DMA address of @buffer (when mapped). + * @buf_dma_count: The number of bytes in @buffer currently DMA-mapped. + * @sg: Head of the scatterlist entries containing data. + * @sg_data_total: Total data in the SG list at any time. + * @sg_data_offset: Offset into the data of the current individual SG node. + * @sg_dma_nents: Number of sg entries mapped in dma_list. + */ +struct ocs_hcu_rctx { + struct ocs_hcu_dev *hcu_dev; + u32 flags; + enum ocs_hcu_algo algo; + size_t blk_sz; + size_t dig_sz; + struct ocs_hcu_dma_list *dma_list; + struct ocs_hcu_hash_ctx hash_ctx; + /* + * Buffer is double the block size because we need space for SW HMAC + * artifacts, i.e: + * - ipad (1 block) + a possible partial block of data. + * - opad (1 block) + digest of H(k ^ ipad || m) + */ + u8 buffer[2 * SHA512_BLOCK_SIZE]; + size_t buf_cnt; + dma_addr_t buf_dma_addr; + size_t buf_dma_count; + struct scatterlist *sg; + unsigned int sg_data_total; + unsigned int sg_data_offset; + unsigned int sg_dma_nents; +}; + +/** + * struct ocs_hcu_drv - Driver data + * @dev_list: The list of HCU devices. + * @lock: The lock protecting dev_list. + */ +struct ocs_hcu_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +static struct ocs_hcu_drv ocs_hcu = { + .dev_list = LIST_HEAD_INIT(ocs_hcu.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_hcu.lock), +}; + +/* + * Return the total amount of data in the request; that is: the data in the + * request buffer + the data in the sg list. + */ +static inline unsigned int kmb_get_total_data(struct ocs_hcu_rctx *rctx) +{ + return rctx->sg_data_total + rctx->buf_cnt; +} + +/* Move remaining content of scatter-gather list to context buffer. */ +static int flush_sg_to_ocs_buffer(struct ocs_hcu_rctx *rctx) +{ + size_t count; + + if (rctx->sg_data_total > (sizeof(rctx->buffer) - rctx->buf_cnt)) { + WARN(1, "%s: sg data does not fit in buffer\n", __func__); + return -EINVAL; + } + + while (rctx->sg_data_total) { + if (!rctx->sg) { + WARN(1, "%s: unexpected NULL sg\n", __func__); + return -EINVAL; + } + /* + * If current sg has been fully processed, skip to the next + * one. + */ + if (rctx->sg_data_offset == rctx->sg->length) { + rctx->sg = sg_next(rctx->sg); + rctx->sg_data_offset = 0; + continue; + } + /* + * Determine the maximum data available to copy from the node. + * Minimum of the length left in the sg node, or the total data + * in the request. + */ + count = min(rctx->sg->length - rctx->sg_data_offset, + rctx->sg_data_total); + /* Copy from scatter-list entry to context buffer. */ + scatterwalk_map_and_copy(&rctx->buffer[rctx->buf_cnt], + rctx->sg, rctx->sg_data_offset, + count, 0); + + rctx->sg_data_offset += count; + rctx->sg_data_total -= count; + rctx->buf_cnt += count; + } + + return 0; +} + +static struct ocs_hcu_dev *kmb_ocs_hcu_find_dev(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); + + /* If the HCU device for the request was previously set, return it. */ + if (tctx->hcu_dev) + return tctx->hcu_dev; + + /* + * Otherwise, get the first HCU device available (there should be one + * and only one device). + */ + spin_lock_bh(&ocs_hcu.lock); + tctx->hcu_dev = list_first_entry_or_null(&ocs_hcu.dev_list, + struct ocs_hcu_dev, + list); + spin_unlock_bh(&ocs_hcu.lock); + + return tctx->hcu_dev; +} + +/* Free OCS DMA linked list and DMA-able context buffer. */ +static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req, + struct ocs_hcu_rctx *rctx) +{ + struct ocs_hcu_dev *hcu_dev = rctx->hcu_dev; + struct device *dev = hcu_dev->dev; + + /* Unmap rctx->buffer (if mapped). */ + if (rctx->buf_dma_count) { + dma_unmap_single(dev, rctx->buf_dma_addr, rctx->buf_dma_count, + DMA_TO_DEVICE); + rctx->buf_dma_count = 0; + } + + /* Unmap req->src (if mapped). */ + if (rctx->sg_dma_nents) { + dma_unmap_sg(dev, req->src, rctx->sg_dma_nents, DMA_TO_DEVICE); + rctx->sg_dma_nents = 0; + } + + /* Free dma_list (if allocated). */ + if (rctx->dma_list) { + ocs_hcu_dma_list_free(hcu_dev, rctx->dma_list); + rctx->dma_list = NULL; + } +} + +/* + * Prepare for DMA operation: + * - DMA-map request context buffer (if needed) + * - DMA-map SG list (only the entries to be processed, see note below) + * - Allocate OCS HCU DMA linked list (number of elements = SG entries to + * process + context buffer (if not empty)). + * - Add DMA-mapped request context buffer to OCS HCU DMA list. + * - Add SG entries to DMA list. + * + * Note: if this is a final request, we process all the data in the SG list, + * otherwise we can only process up to the maximum amount of block-aligned data + * (the remainder will be put into the context buffer and processed in the next + * request). + */ +static int kmb_ocs_dma_prepare(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + struct device *dev = rctx->hcu_dev->dev; + unsigned int remainder = 0; + unsigned int total; + size_t nents; + size_t count; + int rc; + int i; + + /* This function should be called only when there is data to process. */ + total = kmb_get_total_data(rctx); + if (!total) + return -EINVAL; + + /* + * If this is not a final DMA (terminated DMA), the data passed to the + * HCU must be aligned to the block size; compute the remainder data to + * be processed in the next request. + */ + if (!(rctx->flags & REQ_FINAL)) + remainder = total % rctx->blk_sz; + + /* Determine the number of scatter gather list entries to process. */ + nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder); + + /* If there are entries to process, map them. */ + if (nents) { + rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents, + DMA_TO_DEVICE); + if (!rctx->sg_dma_nents) { + dev_err(dev, "Failed to MAP SG\n"); + rc = -ENOMEM; + goto cleanup; + } + /* + * The value returned by dma_map_sg() can be < nents; so update + * nents accordingly. + */ + nents = rctx->sg_dma_nents; + } + + /* + * If context buffer is not empty, map it and add extra DMA entry for + * it. + */ + if (rctx->buf_cnt) { + rctx->buf_dma_addr = dma_map_single(dev, rctx->buffer, + rctx->buf_cnt, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, rctx->buf_dma_addr)) { + dev_err(dev, "Failed to map request context buffer\n"); + rc = -ENOMEM; + goto cleanup; + } + rctx->buf_dma_count = rctx->buf_cnt; + /* Increase number of dma entries. */ + nents++; + } + + /* Allocate OCS HCU DMA list. */ + rctx->dma_list = ocs_hcu_dma_list_alloc(rctx->hcu_dev, nents); + if (!rctx->dma_list) { + rc = -ENOMEM; + goto cleanup; + } + + /* Add request context buffer (if previously DMA-mapped) */ + if (rctx->buf_dma_count) { + rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, rctx->dma_list, + rctx->buf_dma_addr, + rctx->buf_dma_count); + if (rc) + goto cleanup; + } + + /* Add the SG nodes to be processed to the DMA linked list. */ + for_each_sg(req->src, rctx->sg, rctx->sg_dma_nents, i) { + /* + * The number of bytes to add to the list entry is the minimum + * between: + * - The DMA length of the SG entry. + * - The data left to be processed. + */ + count = min(rctx->sg_data_total - remainder, + sg_dma_len(rctx->sg) - rctx->sg_data_offset); + /* + * Do not create a zero length DMA descriptor. Check in case of + * zero length SG node. + */ + if (count == 0) + continue; + /* Add sg to HCU DMA list. */ + rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, + rctx->dma_list, + rctx->sg->dma_address, + count); + if (rc) + goto cleanup; + + /* Update amount of data remaining in SG list. */ + rctx->sg_data_total -= count; + + /* + * If remaining data is equal to remainder (note: 'less than' + * case should never happen in practice), we are done: update + * offset and exit the loop. + */ + if (rctx->sg_data_total <= remainder) { + WARN_ON(rctx->sg_data_total < remainder); + rctx->sg_data_offset += count; + break; + } + + /* + * If we get here is because we need to process the next sg in + * the list; set offset within the sg to 0. + */ + rctx->sg_data_offset = 0; + } + + return 0; +cleanup: + dev_err(dev, "Failed to prepare DMA.\n"); + kmb_ocs_hcu_dma_cleanup(req, rctx); + + return rc; +} + +static void kmb_ocs_hcu_secure_cleanup(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + + /* Clear buffer of any data. */ + memzero_explicit(rctx->buffer, sizeof(rctx->buffer)); +} + +static int kmb_ocs_hcu_handle_queue(struct ahash_request *req) +{ + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + + if (!hcu_dev) + return -ENOENT; + + return crypto_transfer_hash_request_to_engine(hcu_dev->engine, req); +} + +static int prepare_ipad(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + int i; + + WARN(rctx->buf_cnt, "%s: Context buffer is not empty\n", __func__); + WARN(!(rctx->flags & REQ_FLAGS_HMAC_SW), + "%s: HMAC_SW flag is not set\n", __func__); + /* + * Key length must be equal to block size. If key is shorter, + * we pad it with zero (note: key cannot be longer, since + * longer keys are hashed by kmb_ocs_hcu_setkey()). + */ + if (ctx->key_len > rctx->blk_sz) { + WARN(1, "%s: Invalid key length in tfm context\n", __func__); + return -EINVAL; + } + memzero_explicit(&ctx->key[ctx->key_len], + rctx->blk_sz - ctx->key_len); + ctx->key_len = rctx->blk_sz; + /* + * Prepare IPAD for HMAC. Only done for first block. + * HMAC(k,m) = H(k ^ opad || H(k ^ ipad || m)) + * k ^ ipad will be first hashed block. + * k ^ opad will be calculated in the final request. + * Only needed if not using HW HMAC. + */ + for (i = 0; i < rctx->blk_sz; i++) + rctx->buffer[i] = ctx->key[i] ^ HMAC_IPAD_VALUE; + rctx->buf_cnt = rctx->blk_sz; + + return 0; +} + +static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); + int rc; + int i; + + if (!hcu_dev) { + rc = -ENOENT; + goto error; + } + + /* + * If hardware HMAC flag is set, perform HMAC in hardware. + * + * NOTE: this flag implies REQ_FINAL && kmb_get_total_data(rctx) + */ + if (rctx->flags & REQ_FLAGS_HMAC_HW) { + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + rc = ocs_hcu_hmac(hcu_dev, rctx->algo, tctx->key, tctx->key_len, + rctx->dma_list, req->result, rctx->dig_sz); + + /* Unmap data and free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + goto done; + } + + /* Handle update request case. */ + if (!(rctx->flags & REQ_FINAL)) { + /* Update should always have input data. */ + if (!kmb_get_total_data(rctx)) + return -EINVAL; + + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + /* Do hashing step. */ + rc = ocs_hcu_hash_update(hcu_dev, &rctx->hash_ctx, + rctx->dma_list); + + /* Unmap data and free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + /* + * Reset request buffer count (data in the buffer was just + * processed). + */ + rctx->buf_cnt = 0; + /* + * Move remaining sg data into the request buffer, so that it + * will be processed during the next request. + * + * NOTE: we have remaining data if kmb_get_total_data() was not + * a multiple of block size. + */ + rc = flush_sg_to_ocs_buffer(rctx); + if (rc) + goto error; + + goto done; + } + + /* If we get here, this is a final request. */ + + /* If there is data to process, use finup. */ + if (kmb_get_total_data(rctx)) { + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + /* Do hashing step. */ + rc = ocs_hcu_hash_finup(hcu_dev, &rctx->hash_ctx, + rctx->dma_list, + req->result, rctx->dig_sz); + /* Free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + } else { /* Otherwise (if we have no data), use final. */ + rc = ocs_hcu_hash_final(hcu_dev, &rctx->hash_ctx, req->result, + rctx->dig_sz); + if (rc) + goto error; + } + + /* + * If we are finalizing a SW HMAC request, we just computed the result + * of: H(k ^ ipad || m). + * + * We now need to complete the HMAC calculation with the OPAD step, + * that is, we need to compute H(k ^ opad || digest), where digest is + * the digest we just obtained, i.e., H(k ^ ipad || m). + */ + if (rctx->flags & REQ_FLAGS_HMAC_SW) { + /* + * Compute k ^ opad and store it in the request buffer (which + * is not used anymore at this point). + * Note: key has been padded / hashed already (so keylen == + * blksz) . + */ + WARN_ON(tctx->key_len != rctx->blk_sz); + for (i = 0; i < rctx->blk_sz; i++) + rctx->buffer[i] = tctx->key[i] ^ HMAC_OPAD_VALUE; + /* Now append the digest to the rest of the buffer. */ + for (i = 0; (i < rctx->dig_sz); i++) + rctx->buffer[rctx->blk_sz + i] = req->result[i]; + + /* Now hash the buffer to obtain the final HMAC. */ + rc = ocs_hcu_digest(hcu_dev, rctx->algo, rctx->buffer, + rctx->blk_sz + rctx->dig_sz, req->result, + rctx->dig_sz); + if (rc) + goto error; + } + + /* Perform secure clean-up. */ + kmb_ocs_hcu_secure_cleanup(req); +done: + crypto_finalize_hash_request(hcu_dev->engine, req, 0); + + return 0; + +error: + kmb_ocs_hcu_secure_cleanup(req); + return rc; +} + +static int kmb_ocs_hcu_init(struct ahash_request *req) +{ + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + + if (!hcu_dev) + return -ENOENT; + + /* Initialize entire request context to zero. */ + memset(rctx, 0, sizeof(*rctx)); + + rctx->hcu_dev = hcu_dev; + rctx->dig_sz = crypto_ahash_digestsize(tfm); + + switch (rctx->dig_sz) { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + case SHA224_DIGEST_SIZE: + rctx->blk_sz = SHA224_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA224; + break; +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ + case SHA256_DIGEST_SIZE: + rctx->blk_sz = SHA256_BLOCK_SIZE; + /* + * SHA256 and SM3 have the same digest size: use info from tfm + * context to find out which one we should use. + */ + rctx->algo = ctx->is_sm3_tfm ? OCS_HCU_ALGO_SM3 : + OCS_HCU_ALGO_SHA256; + break; + case SHA384_DIGEST_SIZE: + rctx->blk_sz = SHA384_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA384; + break; + case SHA512_DIGEST_SIZE: + rctx->blk_sz = SHA512_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA512; + break; + default: + return -EINVAL; + } + + /* Initialize intermediate data. */ + ocs_hcu_hash_init(&rctx->hash_ctx, rctx->algo); + + /* If this a HMAC request, set HMAC flag. */ + if (ctx->is_hmac_tfm) + rctx->flags |= REQ_FLAGS_HMAC; + + return 0; +} + +static int kmb_ocs_hcu_update(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + int rc; + + if (!req->nbytes) + return 0; + + rctx->sg_data_total = req->nbytes; + rctx->sg_data_offset = 0; + rctx->sg = req->src; + + /* + * If we are doing HMAC, then we must use SW-assisted HMAC, since HW + * HMAC does not support context switching (there it can only be used + * with finup() or digest()). + */ + if (rctx->flags & REQ_FLAGS_HMAC && + !(rctx->flags & REQ_FLAGS_HMAC_SW)) { + rctx->flags |= REQ_FLAGS_HMAC_SW; + rc = prepare_ipad(req); + if (rc) + return rc; + } + + /* + * If remaining sg_data fits into ctx buffer, just copy it there; we'll + * process it at the next update() or final(). + */ + if (rctx->sg_data_total <= (sizeof(rctx->buffer) - rctx->buf_cnt)) + return flush_sg_to_ocs_buffer(rctx); + + return kmb_ocs_hcu_handle_queue(req); +} + +/* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */ +static int kmb_ocs_hcu_fin_common(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + int rc; + + rctx->flags |= REQ_FINAL; + + /* + * If this is a HMAC request and, so far, we didn't have to switch to + * SW HMAC, check if we can use HW HMAC. + */ + if (rctx->flags & REQ_FLAGS_HMAC && + !(rctx->flags & REQ_FLAGS_HMAC_SW)) { + /* + * If we are here, it means we never processed any data so far, + * so we can use HW HMAC, but only if there is some data to + * process (since OCS HW MAC does not support zero-length + * messages) and the key length is supported by the hardware + * (OCS HCU HW only supports length <= 64); if HW HMAC cannot + * be used, fall back to SW-assisted HMAC. + */ + if (kmb_get_total_data(rctx) && + ctx->key_len <= OCS_HCU_HW_KEY_LEN) { + rctx->flags |= REQ_FLAGS_HMAC_HW; + } else { + rctx->flags |= REQ_FLAGS_HMAC_SW; + rc = prepare_ipad(req); + if (rc) + return rc; + } + } + + return kmb_ocs_hcu_handle_queue(req); +} + +static int kmb_ocs_hcu_final(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + + rctx->sg_data_total = 0; + rctx->sg_data_offset = 0; + rctx->sg = NULL; + + return kmb_ocs_hcu_fin_common(req); +} + +static int kmb_ocs_hcu_finup(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + + rctx->sg_data_total = req->nbytes; + rctx->sg_data_offset = 0; + rctx->sg = req->src; + + return kmb_ocs_hcu_fin_common(req); +} + +static int kmb_ocs_hcu_digest(struct ahash_request *req) +{ + int rc = 0; + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + + if (!hcu_dev) + return -ENOENT; + + rc = kmb_ocs_hcu_init(req); + if (rc) + return rc; + + rc = kmb_ocs_hcu_finup(req); + + return rc; +} + +static int kmb_ocs_hcu_export(struct ahash_request *req, void *out) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + + /* Intermediate data is always stored and applied per request. */ + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int kmb_ocs_hcu_import(struct ahash_request *req, const void *in) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + + /* Intermediate data is always stored and applied per request. */ + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +static int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + unsigned int digestsize = crypto_ahash_digestsize(tfm); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + size_t blk_sz = crypto_ahash_blocksize(tfm); + struct crypto_ahash *ahash_tfm; + struct ahash_request *req; + struct crypto_wait wait; + struct scatterlist sg; + const char *alg_name; + int rc; + + /* + * Key length must be equal to block size: + * - If key is shorter, we are done for now (the key will be padded + * later on); this is to maximize the use of HW HMAC (which works + * only for keys <= 64 bytes). + * - If key is longer, we hash it. + */ + if (keylen <= blk_sz) { + memcpy(ctx->key, key, keylen); + ctx->key_len = keylen; + return 0; + } + + switch (digestsize) { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + case SHA224_DIGEST_SIZE: + alg_name = "sha224-keembay-ocs"; + break; +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ + case SHA256_DIGEST_SIZE: + alg_name = ctx->is_sm3_tfm ? "sm3-keembay-ocs" : + "sha256-keembay-ocs"; + break; + case SHA384_DIGEST_SIZE: + alg_name = "sha384-keembay-ocs"; + break; + case SHA512_DIGEST_SIZE: + alg_name = "sha512-keembay-ocs"; + break; + default: + return -EINVAL; + } + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); + if (!req) { + rc = -ENOMEM; + goto err_free_ahash; + } + + crypto_init_wait(&wait); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + crypto_ahash_clear_flags(ahash_tfm, ~0); + + sg_init_one(&sg, key, keylen); + ahash_request_set_crypt(req, &sg, ctx->key, keylen); + + rc = crypto_wait_req(crypto_ahash_digest(req), &wait); + if (rc == 0) + ctx->key_len = digestsize; + + ahash_request_free(req); +err_free_ahash: + crypto_free_ahash(ahash_tfm); + + return rc; +} + +/* Set request size and initialize tfm context. */ +static void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct ocs_hcu_rctx)); + + /* Init context to 0. */ + memzero_explicit(ctx, sizeof(*ctx)); + /* Set engine ops. */ + ctx->engine_ctx.op.do_one_request = kmb_ocs_hcu_do_one_request; +} + +static int kmb_ocs_hcu_sha_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + return 0; +} + +static int kmb_ocs_hcu_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_sm3_tfm = true; + + return 0; +} + +static int kmb_ocs_hcu_hmac_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_sm3_tfm = true; + ctx->is_hmac_tfm = true; + + return 0; +} + +static int kmb_ocs_hcu_hmac_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_hmac_tfm = true; + + return 0; +} + +/* Function called when 'tfm' is de-initialized. */ +static void kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + /* Clear the key. */ + memzero_explicit(ctx->key, sizeof(ctx->key)); +} + +static struct ahash_alg ocs_hcu_algs[] = { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "hmac-sha224-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "hmac-sha256-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sm3", + .cra_driver_name = "sm3-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sm3_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sm3)", + .cra_driver_name = "hmac-sm3-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_sm3_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "hmac-sha384-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "hmac-sha512-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +}; + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_hcu_of_match[] = { + { + .compatible = "intel,keembay-ocs-hcu", + }, + {} +}; + +static int kmb_ocs_hcu_remove(struct platform_device *pdev) +{ + struct ocs_hcu_dev *hcu_dev; + int rc; + + hcu_dev = platform_get_drvdata(pdev); + if (!hcu_dev) + return -ENODEV; + + crypto_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); + + rc = crypto_engine_exit(hcu_dev->engine); + + spin_lock_bh(&ocs_hcu.lock); + list_del(&hcu_dev->list); + spin_unlock_bh(&ocs_hcu.lock); + + return rc; +} + +static int kmb_ocs_hcu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_hcu_dev *hcu_dev; + struct resource *hcu_mem; + int rc; + + hcu_dev = devm_kzalloc(dev, sizeof(*hcu_dev), GFP_KERNEL); + if (!hcu_dev) + return -ENOMEM; + + hcu_dev->dev = dev; + + platform_set_drvdata(pdev, hcu_dev); + rc = dma_set_mask_and_coherent(&pdev->dev, OCS_HCU_DMA_BIT_MASK); + if (rc) + return rc; + + /* Get the memory address and remap. */ + hcu_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!hcu_mem) { + dev_err(dev, "Could not retrieve io mem resource.\n"); + return -ENODEV; + } + + hcu_dev->io_base = devm_ioremap_resource(dev, hcu_mem); + if (IS_ERR(hcu_dev->io_base)) { + dev_err(dev, "Could not io-remap mem resource.\n"); + return PTR_ERR(hcu_dev->io_base); + } + + init_completion(&hcu_dev->irq_done); + + /* Get and request IRQ. */ + hcu_dev->irq = platform_get_irq(pdev, 0); + if (hcu_dev->irq < 0) + return hcu_dev->irq; + + rc = devm_request_threaded_irq(&pdev->dev, hcu_dev->irq, + ocs_hcu_irq_handler, NULL, 0, + "keembay-ocs-hcu", hcu_dev); + if (rc < 0) { + dev_err(dev, "Could not request IRQ.\n"); + return rc; + } + + INIT_LIST_HEAD(&hcu_dev->list); + + spin_lock_bh(&ocs_hcu.lock); + list_add_tail(&hcu_dev->list, &ocs_hcu.dev_list); + spin_unlock_bh(&ocs_hcu.lock); + + /* Initialize crypto engine */ + hcu_dev->engine = crypto_engine_alloc_init(dev, 1); + if (!hcu_dev->engine) + goto list_del; + + rc = crypto_engine_start(hcu_dev->engine); + if (rc) { + dev_err(dev, "Could not start engine.\n"); + goto cleanup; + } + + /* Security infrastructure guarantees OCS clock is enabled. */ + + rc = crypto_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); + if (rc) { + dev_err(dev, "Could not register algorithms.\n"); + goto cleanup; + } + + return 0; + +cleanup: + crypto_engine_exit(hcu_dev->engine); +list_del: + spin_lock_bh(&ocs_hcu.lock); + list_del(&hcu_dev->list); + spin_unlock_bh(&ocs_hcu.lock); + + return rc; +} + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_hcu_driver = { + .probe = kmb_ocs_hcu_probe, + .remove = kmb_ocs_hcu_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_hcu_of_match, + }, +}; + +module_platform_driver(kmb_ocs_hcu_driver); + +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/keembay/ocs-aes.c b/drivers/crypto/keembay/ocs-aes.c index cc286adb1c4a..be9f32fc8f42 100644 --- a/drivers/crypto/keembay/ocs-aes.c +++ b/drivers/crypto/keembay/ocs-aes.c @@ -958,14 +958,14 @@ int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, ocs_aes_write_last_data_blk_len(aes_dev, src_size); /* Write ciphertext bit length */ - bit_len = src_size * 8; + bit_len = (u64)src_size * 8; val = bit_len & 0xFFFFFFFF; iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); val = bit_len >> 32; iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); /* Write aad bit length */ - bit_len = aad_size * 8; + bit_len = (u64)aad_size * 8; val = bit_len & 0xFFFFFFFF; iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); val = bit_len >> 32; @@ -1080,15 +1080,15 @@ static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, /* * q is the octet length of Q. * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as - * q - 1 == iv[0] + * q - 1 == iv[0] & 0x7; */ b0[0] |= iv[0] & 0x7; /* * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] * and must be copied to b0[1]..b0[15-q]. - * q == iv[0] + 1 + * q == (iv[0] & 0x7) + 1 */ - q = iv[0] + 1; + q = (iv[0] & 0x7) + 1; for (i = 1; i <= 15 - q; i++) b0[i] = iv[i]; /* diff --git a/drivers/crypto/keembay/ocs-hcu.c b/drivers/crypto/keembay/ocs-hcu.c new file mode 100644 index 000000000000..81eecacf603a --- /dev/null +++ b/drivers/crypto/keembay/ocs-hcu.c @@ -0,0 +1,840 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocs-hcu.h" + +/* Registers. */ +#define OCS_HCU_MODE 0x00 +#define OCS_HCU_CHAIN 0x04 +#define OCS_HCU_OPERATION 0x08 +#define OCS_HCU_KEY_0 0x0C +#define OCS_HCU_ISR 0x50 +#define OCS_HCU_IER 0x54 +#define OCS_HCU_STATUS 0x58 +#define OCS_HCU_MSG_LEN_LO 0x60 +#define OCS_HCU_MSG_LEN_HI 0x64 +#define OCS_HCU_KEY_BYTE_ORDER_CFG 0x80 +#define OCS_HCU_DMA_SRC_ADDR 0x400 +#define OCS_HCU_DMA_SRC_SIZE 0x408 +#define OCS_HCU_DMA_DST_SIZE 0x40C +#define OCS_HCU_DMA_DMA_MODE 0x410 +#define OCS_HCU_DMA_NEXT_SRC_DESCR 0x418 +#define OCS_HCU_DMA_MSI_ISR 0x480 +#define OCS_HCU_DMA_MSI_IER 0x484 +#define OCS_HCU_DMA_MSI_MASK 0x488 + +/* Register bit definitions. */ +#define HCU_MODE_ALGO_SHIFT 16 +#define HCU_MODE_HMAC_SHIFT 22 + +#define HCU_STATUS_BUSY BIT(0) + +#define HCU_BYTE_ORDER_SWAP BIT(0) + +#define HCU_IRQ_HASH_DONE BIT(2) +#define HCU_IRQ_HASH_ERR_MASK (BIT(3) | BIT(1) | BIT(0)) + +#define HCU_DMA_IRQ_SRC_DONE BIT(0) +#define HCU_DMA_IRQ_SAI_ERR BIT(2) +#define HCU_DMA_IRQ_BAD_COMP_ERR BIT(3) +#define HCU_DMA_IRQ_INBUF_RD_ERR BIT(4) +#define HCU_DMA_IRQ_INBUF_WD_ERR BIT(5) +#define HCU_DMA_IRQ_OUTBUF_WR_ERR BIT(6) +#define HCU_DMA_IRQ_OUTBUF_RD_ERR BIT(7) +#define HCU_DMA_IRQ_CRD_ERR BIT(8) +#define HCU_DMA_IRQ_ERR_MASK (HCU_DMA_IRQ_SAI_ERR | \ + HCU_DMA_IRQ_BAD_COMP_ERR | \ + HCU_DMA_IRQ_INBUF_RD_ERR | \ + HCU_DMA_IRQ_INBUF_WD_ERR | \ + HCU_DMA_IRQ_OUTBUF_WR_ERR | \ + HCU_DMA_IRQ_OUTBUF_RD_ERR | \ + HCU_DMA_IRQ_CRD_ERR) + +#define HCU_DMA_SNOOP_MASK (0x7 << 28) +#define HCU_DMA_SRC_LL_EN BIT(25) +#define HCU_DMA_EN BIT(31) + +#define OCS_HCU_ENDIANNESS_VALUE 0x2A + +#define HCU_DMA_MSI_UNMASK BIT(0) +#define HCU_DMA_MSI_DISABLE 0 +#define HCU_IRQ_DISABLE 0 + +#define OCS_HCU_START BIT(0) +#define OCS_HCU_TERMINATE BIT(1) + +#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) + +#define OCS_HCU_HW_KEY_LEN_U32 (OCS_HCU_HW_KEY_LEN / sizeof(u32)) + +#define HCU_DATA_WRITE_ENDIANNESS_OFFSET 26 + +#define OCS_HCU_NUM_CHAINS_SHA256_224_SM3 (SHA256_DIGEST_SIZE / sizeof(u32)) +#define OCS_HCU_NUM_CHAINS_SHA384_512 (SHA512_DIGEST_SIZE / sizeof(u32)) + +/* + * While polling on a busy HCU, wait maximum 200us between one check and the + * other. + */ +#define OCS_HCU_WAIT_BUSY_RETRY_DELAY_US 200 +/* Wait on a busy HCU for maximum 1 second. */ +#define OCS_HCU_WAIT_BUSY_TIMEOUT_US 1000000 + +/** + * struct ocs_hcu_dma_list - An entry in an OCS DMA linked list. + * @src_addr: Source address of the data. + * @src_len: Length of data to be fetched. + * @nxt_desc: Next descriptor to fetch. + * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. + */ +struct ocs_hcu_dma_entry { + u32 src_addr; + u32 src_len; + u32 nxt_desc; + u32 ll_flags; +}; + +/** + * struct ocs_dma_list - OCS-specific DMA linked list. + * @head: The head of the list (points to the array backing the list). + * @tail: The current tail of the list; NULL if the list is empty. + * @dma_addr: The DMA address of @head (i.e., the DMA address of the backing + * array). + * @max_nents: Maximum number of entries in the list (i.e., number of elements + * in the backing array). + * + * The OCS DMA list is an array-backed list of OCS DMA descriptors. The array + * backing the list is allocated with dma_alloc_coherent() and pointed by + * @head. + */ +struct ocs_hcu_dma_list { + struct ocs_hcu_dma_entry *head; + struct ocs_hcu_dma_entry *tail; + dma_addr_t dma_addr; + size_t max_nents; +}; + +static inline u32 ocs_hcu_num_chains(enum ocs_hcu_algo algo) +{ + switch (algo) { + case OCS_HCU_ALGO_SHA224: + case OCS_HCU_ALGO_SHA256: + case OCS_HCU_ALGO_SM3: + return OCS_HCU_NUM_CHAINS_SHA256_224_SM3; + case OCS_HCU_ALGO_SHA384: + case OCS_HCU_ALGO_SHA512: + return OCS_HCU_NUM_CHAINS_SHA384_512; + default: + return 0; + }; +} + +static inline u32 ocs_hcu_digest_size(enum ocs_hcu_algo algo) +{ + switch (algo) { + case OCS_HCU_ALGO_SHA224: + return SHA224_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA256: + case OCS_HCU_ALGO_SM3: + /* SM3 shares the same block size. */ + return SHA256_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA384: + return SHA384_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA512: + return SHA512_DIGEST_SIZE; + default: + return 0; + } +} + +/** + * ocs_hcu_wait_busy() - Wait for HCU OCS hardware to became usable. + * @hcu_dev: OCS HCU device to wait for. + * + * Return: 0 if device free, -ETIMEOUT if device busy and internal timeout has + * expired. + */ +static int ocs_hcu_wait_busy(struct ocs_hcu_dev *hcu_dev) +{ + long val; + + return readl_poll_timeout(hcu_dev->io_base + OCS_HCU_STATUS, val, + !(val & HCU_STATUS_BUSY), + OCS_HCU_WAIT_BUSY_RETRY_DELAY_US, + OCS_HCU_WAIT_BUSY_TIMEOUT_US); +} + +static void ocs_hcu_done_irq_en(struct ocs_hcu_dev *hcu_dev) +{ + /* Clear any pending interrupts. */ + writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_ISR); + hcu_dev->irq_err = false; + /* Enable error and HCU done interrupts. */ + writel(HCU_IRQ_HASH_DONE | HCU_IRQ_HASH_ERR_MASK, + hcu_dev->io_base + OCS_HCU_IER); +} + +static void ocs_hcu_dma_irq_en(struct ocs_hcu_dev *hcu_dev) +{ + /* Clear any pending interrupts. */ + writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + hcu_dev->irq_err = false; + /* Only operating on DMA source completion and error interrupts. */ + writel(HCU_DMA_IRQ_ERR_MASK | HCU_DMA_IRQ_SRC_DONE, + hcu_dev->io_base + OCS_HCU_DMA_MSI_IER); + /* Unmask */ + writel(HCU_DMA_MSI_UNMASK, hcu_dev->io_base + OCS_HCU_DMA_MSI_MASK); +} + +static void ocs_hcu_irq_dis(struct ocs_hcu_dev *hcu_dev) +{ + writel(HCU_IRQ_DISABLE, hcu_dev->io_base + OCS_HCU_IER); + writel(HCU_DMA_MSI_DISABLE, hcu_dev->io_base + OCS_HCU_DMA_MSI_IER); +} + +static int ocs_hcu_wait_and_disable_irq(struct ocs_hcu_dev *hcu_dev) +{ + int rc; + + rc = wait_for_completion_interruptible(&hcu_dev->irq_done); + if (rc) + goto exit; + + if (hcu_dev->irq_err) { + /* Unset flag and return error. */ + hcu_dev->irq_err = false; + rc = -EIO; + goto exit; + } + +exit: + ocs_hcu_irq_dis(hcu_dev); + + return rc; +} + +/** + * ocs_hcu_get_intermediate_data() - Get intermediate data. + * @hcu_dev: The target HCU device. + * @data: Where to store the intermediate. + * @algo: The algorithm being used. + * + * This function is used to save the current hashing process state in order to + * continue it in the future. + * + * Note: once all data has been processed, the intermediate data actually + * contains the hashing result. So this function is also used to retrieve the + * final result of a hashing process. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_get_intermediate_data(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_idata *data, + enum ocs_hcu_algo algo) +{ + const int n = ocs_hcu_num_chains(algo); + u32 *chain; + int rc; + int i; + + /* Data not requested. */ + if (!data) + return -EINVAL; + + chain = (u32 *)data->digest; + + /* Ensure that the OCS is no longer busy before reading the chains. */ + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + /* + * This loops is safe because data->digest is an array of + * SHA512_DIGEST_SIZE bytes and the maximum value returned by + * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal + * to SHA512_DIGEST_SIZE / sizeof(u32). + */ + for (i = 0; i < n; i++) + chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN); + + data->msg_len_lo = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_LO); + data->msg_len_hi = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_HI); + + return 0; +} + +/** + * ocs_hcu_set_intermediate_data() - Set intermediate data. + * @hcu_dev: The target HCU device. + * @data: The intermediate data to be set. + * @algo: The algorithm being used. + * + * This function is used to continue a previous hashing process. + */ +static void ocs_hcu_set_intermediate_data(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_idata *data, + enum ocs_hcu_algo algo) +{ + const int n = ocs_hcu_num_chains(algo); + u32 *chain = (u32 *)data->digest; + int i; + + /* + * This loops is safe because data->digest is an array of + * SHA512_DIGEST_SIZE bytes and the maximum value returned by + * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal + * to SHA512_DIGEST_SIZE / sizeof(u32). + */ + for (i = 0; i < n; i++) + writel(chain[i], hcu_dev->io_base + OCS_HCU_CHAIN); + + writel(data->msg_len_lo, hcu_dev->io_base + OCS_HCU_MSG_LEN_LO); + writel(data->msg_len_hi, hcu_dev->io_base + OCS_HCU_MSG_LEN_HI); +} + +static int ocs_hcu_get_digest(struct ocs_hcu_dev *hcu_dev, + enum ocs_hcu_algo algo, u8 *dgst, size_t dgst_len) +{ + u32 *chain; + int rc; + int i; + + if (!dgst) + return -EINVAL; + + /* Length of the output buffer must match the algo digest size. */ + if (dgst_len != ocs_hcu_digest_size(algo)) + return -EINVAL; + + /* Ensure that the OCS is no longer busy before reading the chains. */ + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + chain = (u32 *)dgst; + for (i = 0; i < dgst_len / sizeof(u32); i++) + chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN); + + return 0; +} + +/** + * ocs_hcu_hw_cfg() - Configure the HCU hardware. + * @hcu_dev: The HCU device to configure. + * @algo: The algorithm to be used by the HCU device. + * @use_hmac: Whether or not HW HMAC should be used. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_hw_cfg(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + bool use_hmac) +{ + u32 cfg; + int rc; + + if (algo != OCS_HCU_ALGO_SHA256 && algo != OCS_HCU_ALGO_SHA224 && + algo != OCS_HCU_ALGO_SHA384 && algo != OCS_HCU_ALGO_SHA512 && + algo != OCS_HCU_ALGO_SM3) + return -EINVAL; + + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + /* Ensure interrupts are disabled. */ + ocs_hcu_irq_dis(hcu_dev); + + /* Configure endianness, hashing algorithm and HW HMAC (if needed) */ + cfg = OCS_HCU_ENDIANNESS_VALUE << HCU_DATA_WRITE_ENDIANNESS_OFFSET; + cfg |= algo << HCU_MODE_ALGO_SHIFT; + if (use_hmac) + cfg |= BIT(HCU_MODE_HMAC_SHIFT); + + writel(cfg, hcu_dev->io_base + OCS_HCU_MODE); + + return 0; +} + +/** + * ocs_hcu_clear_key() - Clear key stored in OCS HMAC KEY registers. + * @hcu_dev: The OCS HCU device whose key registers should be cleared. + */ +static void ocs_hcu_clear_key(struct ocs_hcu_dev *hcu_dev) +{ + int reg_off; + + /* Clear OCS_HCU_KEY_[0..15] */ + for (reg_off = 0; reg_off < OCS_HCU_HW_KEY_LEN; reg_off += sizeof(u32)) + writel(0, hcu_dev->io_base + OCS_HCU_KEY_0 + reg_off); +} + +/** + * ocs_hcu_write_key() - Write key to OCS HMAC KEY registers. + * @hcu_dev: The OCS HCU device the key should be written to. + * @key: The key to be written. + * @len: The size of the key to write. It must be OCS_HCU_HW_KEY_LEN. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_write_key(struct ocs_hcu_dev *hcu_dev, const u8 *key, size_t len) +{ + u32 key_u32[OCS_HCU_HW_KEY_LEN_U32]; + int i; + + if (len > OCS_HCU_HW_KEY_LEN) + return -EINVAL; + + /* Copy key into temporary u32 array. */ + memcpy(key_u32, key, len); + + /* + * Hardware requires all the bytes of the HW Key vector to be + * written. So pad with zero until we reach OCS_HCU_HW_KEY_LEN. + */ + memzero_explicit((u8 *)key_u32 + len, OCS_HCU_HW_KEY_LEN - len); + + /* + * OCS hardware expects the MSB of the key to be written at the highest + * address of the HCU Key vector; in other word, the key must be + * written in reverse order. + * + * Therefore, we first enable byte swapping for the HCU key vector; + * so that bytes of 32-bit word written to OCS_HCU_KEY_[0..15] will be + * swapped: + * 3 <---> 0, 2 <---> 1. + */ + writel(HCU_BYTE_ORDER_SWAP, + hcu_dev->io_base + OCS_HCU_KEY_BYTE_ORDER_CFG); + /* + * And then we write the 32-bit words composing the key starting from + * the end of the key. + */ + for (i = 0; i < OCS_HCU_HW_KEY_LEN_U32; i++) + writel(key_u32[OCS_HCU_HW_KEY_LEN_U32 - 1 - i], + hcu_dev->io_base + OCS_HCU_KEY_0 + (sizeof(u32) * i)); + + memzero_explicit(key_u32, OCS_HCU_HW_KEY_LEN); + + return 0; +} + +/** + * ocs_hcu_ll_dma_start() - Start OCS HCU hashing via DMA + * @hcu_dev: The OCS HCU device to use. + * @dma_list: The OCS DMA list mapping the data to hash. + * @finalize: Whether or not this is the last hashing operation and therefore + * the final hash should be compute even if data is not + * block-aligned. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_ll_dma_start(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_dma_list *dma_list, + bool finalize) +{ + u32 cfg = HCU_DMA_SNOOP_MASK | HCU_DMA_SRC_LL_EN | HCU_DMA_EN; + int rc; + + if (!dma_list) + return -EINVAL; + + /* + * For final requests we use HCU_DONE IRQ to be notified when all input + * data has been processed by the HCU; however, we cannot do so for + * non-final requests, because we don't get a HCU_DONE IRQ when we + * don't terminate the operation. + * + * Therefore, for non-final requests, we use the DMA IRQ, which + * triggers when DMA has finishing feeding all the input data to the + * HCU, but the HCU may still be processing it. This is fine, since we + * will wait for the HCU processing to be completed when we try to read + * intermediate results, in ocs_hcu_get_intermediate_data(). + */ + if (finalize) + ocs_hcu_done_irq_en(hcu_dev); + else + ocs_hcu_dma_irq_en(hcu_dev); + + reinit_completion(&hcu_dev->irq_done); + writel(dma_list->dma_addr, hcu_dev->io_base + OCS_HCU_DMA_NEXT_SRC_DESCR); + writel(0, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE); + writel(0, hcu_dev->io_base + OCS_HCU_DMA_DST_SIZE); + + writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION); + + writel(cfg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE); + + if (finalize) + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + return 0; +} + +struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev, + int max_nents) +{ + struct ocs_hcu_dma_list *dma_list; + + dma_list = kmalloc(sizeof(*dma_list), GFP_KERNEL); + if (!dma_list) + return NULL; + + /* Total size of the DMA list to allocate. */ + dma_list->head = dma_alloc_coherent(hcu_dev->dev, + sizeof(*dma_list->head) * max_nents, + &dma_list->dma_addr, GFP_KERNEL); + if (!dma_list->head) { + kfree(dma_list); + return NULL; + } + dma_list->max_nents = max_nents; + dma_list->tail = NULL; + + return dma_list; +} + +void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list) +{ + if (!dma_list) + return; + + dma_free_coherent(hcu_dev->dev, + sizeof(*dma_list->head) * dma_list->max_nents, + dma_list->head, dma_list->dma_addr); + + kfree(dma_list); +} + +/* Add a new DMA entry at the end of the OCS DMA list. */ +int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list, + dma_addr_t addr, u32 len) +{ + struct device *dev = hcu_dev->dev; + struct ocs_hcu_dma_entry *old_tail; + struct ocs_hcu_dma_entry *new_tail; + + if (!len) + return 0; + + if (!dma_list) + return -EINVAL; + + if (addr & ~OCS_HCU_DMA_BIT_MASK) { + dev_err(dev, + "Unexpected error: Invalid DMA address for OCS HCU\n"); + return -EINVAL; + } + + old_tail = dma_list->tail; + new_tail = old_tail ? old_tail + 1 : dma_list->head; + + /* Check if list is full. */ + if (new_tail - dma_list->head >= dma_list->max_nents) + return -ENOMEM; + + /* + * If there was an old tail (i.e., this is not the first element we are + * adding), un-terminate the old tail and make it point to the new one. + */ + if (old_tail) { + old_tail->ll_flags &= ~OCS_LL_DMA_FLAG_TERMINATE; + /* + * The old tail 'nxt_desc' must point to the DMA address of the + * new tail. + */ + old_tail->nxt_desc = dma_list->dma_addr + + sizeof(*dma_list->tail) * (new_tail - + dma_list->head); + } + + new_tail->src_addr = (u32)addr; + new_tail->src_len = (u32)len; + new_tail->ll_flags = OCS_LL_DMA_FLAG_TERMINATE; + new_tail->nxt_desc = 0; + + /* Update list tail with new tail. */ + dma_list->tail = new_tail; + + return 0; +} + +/** + * ocs_hcu_hash_init() - Initialize hash operation context. + * @ctx: The context to initialize. + * @algo: The hashing algorithm to use. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo) +{ + if (!ctx) + return -EINVAL; + + ctx->algo = algo; + ctx->idata.msg_len_lo = 0; + ctx->idata.msg_len_hi = 0; + /* No need to set idata.digest to 0. */ + + return 0; +} + +/** + * ocs_hcu_digest() - Perform a hashing iteration. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dma_list: The OCS DMA list mapping the input data to process. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* Start linked-list DMA hashing. */ + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, false); + if (rc) + return rc; + + /* Update idata and return. */ + return ocs_hcu_get_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); +} + +/** + * ocs_hcu_hash_final() - Update and finalize hash computation. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dma_list: The OCS DMA list mapping the input data to process. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* Start linked-list DMA hashing. */ + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true); + if (rc) + return rc; + + /* Get digest and return. */ + return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len); +} + +/** + * ocs_hcu_hash_final() - Finalize hash computation. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, u8 *dgst, + size_t dgst_len) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* + * Enable HCU interrupts, so that HCU_DONE will be triggered once the + * final hash is computed. + */ + ocs_hcu_done_irq_en(hcu_dev); + reinit_completion(&hcu_dev->irq_done); + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + /* Get digest and return. */ + return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len); +} + +/** + * ocs_hcu_digest() - Compute hash digest. + * @hcu_dev: The OCS HCU device to use. + * @algo: The hash algorithm to use. + * @data: The input data to process. + * @data_len: The length of @data. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + void *data, size_t data_len, u8 *dgst, size_t dgst_len) +{ + struct device *dev = hcu_dev->dev; + dma_addr_t dma_handle; + u32 reg; + int rc; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, algo, false); + if (rc) + return rc; + + dma_handle = dma_map_single(dev, data, data_len, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_handle)) + return -EIO; + + reg = HCU_DMA_SNOOP_MASK | HCU_DMA_EN; + + ocs_hcu_done_irq_en(hcu_dev); + + reinit_completion(&hcu_dev->irq_done); + + writel(dma_handle, hcu_dev->io_base + OCS_HCU_DMA_SRC_ADDR); + writel(data_len, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE); + writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION); + writel(reg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE); + + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + dma_unmap_single(dev, dma_handle, data_len, DMA_TO_DEVICE); + + return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len); +} + +/** + * ocs_hcu_hmac() - Compute HMAC. + * @hcu_dev: The OCS HCU device to use. + * @algo: The hash algorithm to use with HMAC. + * @key: The key to use. + * @dma_list: The OCS DMA list mapping the input data to process. + * @key_len: The length of @key. + * @dgst: The buffer where to save the computed HMAC. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + const u8 *key, size_t key_len, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len) +{ + int rc; + + /* Ensure 'key' is not NULL. */ + if (!key || key_len == 0) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, algo, true); + if (rc) + return rc; + + rc = ocs_hcu_write_key(hcu_dev, key, key_len); + if (rc) + return rc; + + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true); + + /* Clear HW key before processing return code. */ + ocs_hcu_clear_key(hcu_dev); + + if (rc) + return rc; + + return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len); +} + +irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id) +{ + struct ocs_hcu_dev *hcu_dev = dev_id; + u32 hcu_irq; + u32 dma_irq; + + /* Read and clear the HCU interrupt. */ + hcu_irq = readl(hcu_dev->io_base + OCS_HCU_ISR); + writel(hcu_irq, hcu_dev->io_base + OCS_HCU_ISR); + + /* Read and clear the HCU DMA interrupt. */ + dma_irq = readl(hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + writel(dma_irq, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + + /* Check for errors. */ + if (hcu_irq & HCU_IRQ_HASH_ERR_MASK || dma_irq & HCU_DMA_IRQ_ERR_MASK) { + hcu_dev->irq_err = true; + goto complete; + } + + /* Check for DONE IRQs. */ + if (hcu_irq & HCU_IRQ_HASH_DONE || dma_irq & HCU_DMA_IRQ_SRC_DONE) + goto complete; + + return IRQ_NONE; + +complete: + complete(&hcu_dev->irq_done); + + return IRQ_HANDLED; +} + +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/keembay/ocs-hcu.h b/drivers/crypto/keembay/ocs-hcu.h new file mode 100644 index 000000000000..fbbbb92a0592 --- /dev/null +++ b/drivers/crypto/keembay/ocs-hcu.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include + +#ifndef _CRYPTO_OCS_HCU_H +#define _CRYPTO_OCS_HCU_H + +#define OCS_HCU_DMA_BIT_MASK DMA_BIT_MASK(32) + +#define OCS_HCU_HW_KEY_LEN 64 + +struct ocs_hcu_dma_list; + +enum ocs_hcu_algo { + OCS_HCU_ALGO_SHA256 = 2, + OCS_HCU_ALGO_SHA224 = 3, + OCS_HCU_ALGO_SHA384 = 4, + OCS_HCU_ALGO_SHA512 = 5, + OCS_HCU_ALGO_SM3 = 6, +}; + +/** + * struct ocs_hcu_dev - OCS HCU device context. + * @list: List of device contexts. + * @dev: OCS HCU device. + * @io_base: Base address of OCS HCU registers. + * @engine: Crypto engine for the device. + * @irq: IRQ number. + * @irq_done: Completion for IRQ. + * @irq_err: Flag indicating an IRQ error has happened. + */ +struct ocs_hcu_dev { + struct list_head list; + struct device *dev; + void __iomem *io_base; + struct crypto_engine *engine; + int irq; + struct completion irq_done; + bool irq_err; +}; + +/** + * struct ocs_hcu_idata - Intermediate data generated by the HCU. + * @msg_len_lo: Length of data the HCU has operated on in bits, low 32b. + * @msg_len_hi: Length of data the HCU has operated on in bits, high 32b. + * @digest: The digest read from the HCU. If the HCU is terminated, it will + * contain the actual hash digest. Otherwise it is the intermediate + * state. + */ +struct ocs_hcu_idata { + u32 msg_len_lo; + u32 msg_len_hi; + u8 digest[SHA512_DIGEST_SIZE]; +}; + +/** + * struct ocs_hcu_hash_ctx - Context for OCS HCU hashing operation. + * @algo: The hashing algorithm being used. + * @idata: The current intermediate data. + */ +struct ocs_hcu_hash_ctx { + enum ocs_hcu_algo algo; + struct ocs_hcu_idata idata; +}; + +irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id); + +struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev, + int max_nents); + +void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list); + +int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list, + dma_addr_t addr, u32 len); + +int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo); + +int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list); + +int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len); + +int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, u8 *dgst, + size_t dgst_len); + +int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + void *data, size_t data_len, u8 *dgst, size_t dgst_len); + +int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + const u8 *key, size_t key_len, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len); + +#endif /* _CRYPTO_OCS_HCU_H */ diff --git a/drivers/crypto/marvell/Kconfig b/drivers/crypto/marvell/Kconfig index 13063384f958..9125199f1702 100644 --- a/drivers/crypto/marvell/Kconfig +++ b/drivers/crypto/marvell/Kconfig @@ -35,3 +35,18 @@ config CRYPTO_DEV_OCTEONTX_CPT To compile this driver as module, choose M here: the modules will be called octeontx-cpt and octeontx-cptvf + +config CRYPTO_DEV_OCTEONTX2_CPT + tristate "Marvell OcteonTX2 CPT driver" + depends on ARCH_THUNDER2 || COMPILE_TEST + depends on PCI_MSI && 64BIT + depends on CRYPTO_LIB_AES + depends on NET_VENDOR_MARVELL + select OCTEONTX2_MBOX + select CRYPTO_DEV_MARVELL + select CRYPTO_SKCIPHER + select CRYPTO_HASH + select CRYPTO_AEAD + help + This driver allows you to utilize the Marvell Cryptographic + Accelerator Unit(CPT) found in OcteonTX2 series of processors. diff --git a/drivers/crypto/marvell/Makefile b/drivers/crypto/marvell/Makefile index 6c6a1519b0f1..39db6d9c0aaf 100644 --- a/drivers/crypto/marvell/Makefile +++ b/drivers/crypto/marvell/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += cesa/ obj-$(CONFIG_CRYPTO_DEV_OCTEONTX_CPT) += octeontx/ +obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += octeontx2/ diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index 06211858bf2e..f14aac532f53 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -381,10 +381,10 @@ static int mv_cesa_get_sram(struct platform_device *pdev, int idx) engine->pool = of_gen_pool_get(cesa->dev->of_node, "marvell,crypto-srams", idx); if (engine->pool) { - engine->sram = gen_pool_dma_alloc(engine->pool, - cesa->sram_size, - &engine->sram_dma); - if (engine->sram) + engine->sram_pool = gen_pool_dma_alloc(engine->pool, + cesa->sram_size, + &engine->sram_dma); + if (engine->sram_pool) return 0; engine->pool = NULL; @@ -422,7 +422,7 @@ static void mv_cesa_put_sram(struct platform_device *pdev, int idx) struct mv_cesa_engine *engine = &cesa->engines[idx]; if (engine->pool) - gen_pool_free(engine->pool, (unsigned long)engine->sram, + gen_pool_free(engine->pool, (unsigned long)engine->sram_pool, cesa->sram_size); else dma_unmap_resource(cesa->dev, engine->sram_dma, diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h index fa56b45620c7..c1007f2ba79c 100644 --- a/drivers/crypto/marvell/cesa/cesa.h +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -428,6 +428,7 @@ struct mv_cesa_dev { * @id: engine id * @regs: engine registers * @sram: SRAM memory region + * @sram_pool: SRAM memory region from pool * @sram_dma: DMA address of the SRAM memory region * @lock: engine lock * @req: current crypto request @@ -448,7 +449,10 @@ struct mv_cesa_dev { struct mv_cesa_engine { int id; void __iomem *regs; - void __iomem *sram; + union { + void __iomem *sram; + void *sram_pool; + }; dma_addr_t sram_dma; spinlock_t lock; struct crypto_async_request *req; @@ -867,6 +871,31 @@ int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, struct mv_cesa_sg_dma_iter *sgiter, gfp_t gfp_flags); +size_t mv_cesa_sg_copy(struct mv_cesa_engine *engine, + struct scatterlist *sgl, unsigned int nents, + unsigned int sram_off, size_t buflen, off_t skip, + bool to_sram); + +static inline size_t mv_cesa_sg_copy_to_sram(struct mv_cesa_engine *engine, + struct scatterlist *sgl, + unsigned int nents, + unsigned int sram_off, + size_t buflen, off_t skip) +{ + return mv_cesa_sg_copy(engine, sgl, nents, sram_off, buflen, skip, + true); +} + +static inline size_t mv_cesa_sg_copy_from_sram(struct mv_cesa_engine *engine, + struct scatterlist *sgl, + unsigned int nents, + unsigned int sram_off, + size_t buflen, off_t skip) +{ + return mv_cesa_sg_copy(engine, sgl, nents, sram_off, buflen, skip, + false); +} + /* Algorithm definitions */ extern struct ahash_alg mv_md5_alg; diff --git a/drivers/crypto/marvell/cesa/cipher.c b/drivers/crypto/marvell/cesa/cipher.c index b4a6ff9dd6d5..b739d3b873dc 100644 --- a/drivers/crypto/marvell/cesa/cipher.c +++ b/drivers/crypto/marvell/cesa/cipher.c @@ -89,22 +89,29 @@ static void mv_cesa_skcipher_std_step(struct skcipher_request *req) CESA_SA_SRAM_PAYLOAD_SIZE); mv_cesa_adjust_op(engine, &sreq->op); - memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); + if (engine->pool) + memcpy(engine->sram_pool, &sreq->op, sizeof(sreq->op)); + else + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); - len = sg_pcopy_to_buffer(req->src, creq->src_nents, - engine->sram + CESA_SA_DATA_SRAM_OFFSET, - len, sreq->offset); + len = mv_cesa_sg_copy_to_sram(engine, req->src, creq->src_nents, + CESA_SA_DATA_SRAM_OFFSET, len, + sreq->offset); sreq->size = len; mv_cesa_set_crypt_op_len(&sreq->op, len); /* FIXME: only update enc_len field */ if (!sreq->skip_ctx) { - memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); + if (engine->pool) + memcpy(engine->sram_pool, &sreq->op, sizeof(sreq->op)); + else + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); sreq->skip_ctx = true; - } else { + } else if (engine->pool) + memcpy(engine->sram_pool, &sreq->op, sizeof(sreq->op.desc)); + else memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op.desc)); - } mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); @@ -121,9 +128,9 @@ static int mv_cesa_skcipher_std_process(struct skcipher_request *req, struct mv_cesa_engine *engine = creq->base.engine; size_t len; - len = sg_pcopy_from_buffer(req->dst, creq->dst_nents, - engine->sram + CESA_SA_DATA_SRAM_OFFSET, - sreq->size, sreq->offset); + len = mv_cesa_sg_copy_from_sram(engine, req->dst, creq->dst_nents, + CESA_SA_DATA_SRAM_OFFSET, sreq->size, + sreq->offset); sreq->offset += len; if (sreq->offset < req->cryptlen) @@ -214,11 +221,14 @@ mv_cesa_skcipher_complete(struct crypto_async_request *req) basereq = &creq->base; memcpy(skreq->iv, basereq->chain.last->op->ctx.skcipher.iv, ivsize); - } else { + } else if (engine->pool) + memcpy(skreq->iv, + engine->sram_pool + CESA_SA_CRYPT_IV_SRAM_OFFSET, + ivsize); + else memcpy_fromio(skreq->iv, engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, ivsize); - } } static const struct mv_cesa_req_ops mv_cesa_skcipher_req_ops = { diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index 8cf9fd518d86..c72b0672fc71 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -168,7 +168,12 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) int i; mv_cesa_adjust_op(engine, &creq->op_tmpl); - memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); + if (engine->pool) + memcpy(engine->sram_pool, &creq->op_tmpl, + sizeof(creq->op_tmpl)); + else + memcpy_toio(engine->sram, &creq->op_tmpl, + sizeof(creq->op_tmpl)); if (!sreq->offset) { digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); @@ -177,9 +182,14 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) engine->regs + CESA_IVDIG(i)); } - if (creq->cache_ptr) - memcpy_toio(engine->sram + CESA_SA_DATA_SRAM_OFFSET, - creq->cache, creq->cache_ptr); + if (creq->cache_ptr) { + if (engine->pool) + memcpy(engine->sram_pool + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, creq->cache_ptr); + else + memcpy_toio(engine->sram + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, creq->cache_ptr); + } len = min_t(size_t, req->nbytes + creq->cache_ptr - sreq->offset, CESA_SA_SRAM_PAYLOAD_SIZE); @@ -190,12 +200,10 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) } if (len - creq->cache_ptr) - sreq->offset += sg_pcopy_to_buffer(req->src, creq->src_nents, - engine->sram + - CESA_SA_DATA_SRAM_OFFSET + - creq->cache_ptr, - len - creq->cache_ptr, - sreq->offset); + sreq->offset += mv_cesa_sg_copy_to_sram( + engine, req->src, creq->src_nents, + CESA_SA_DATA_SRAM_OFFSET + creq->cache_ptr, + len - creq->cache_ptr, sreq->offset); op = &creq->op_tmpl; @@ -220,16 +228,28 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) if (len + trailerlen > CESA_SA_SRAM_PAYLOAD_SIZE) { len &= CESA_HASH_BLOCK_SIZE_MSK; new_cache_ptr = 64 - trailerlen; - memcpy_fromio(creq->cache, - engine->sram + - CESA_SA_DATA_SRAM_OFFSET + len, - new_cache_ptr); + if (engine->pool) + memcpy(creq->cache, + engine->sram_pool + + CESA_SA_DATA_SRAM_OFFSET + len, + new_cache_ptr); + else + memcpy_fromio(creq->cache, + engine->sram + + CESA_SA_DATA_SRAM_OFFSET + + len, + new_cache_ptr); } else { i = mv_cesa_ahash_pad_req(creq, creq->cache); len += i; - memcpy_toio(engine->sram + len + - CESA_SA_DATA_SRAM_OFFSET, - creq->cache, i); + if (engine->pool) + memcpy(engine->sram_pool + len + + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, i); + else + memcpy_toio(engine->sram + len + + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, i); } if (frag_mode == CESA_SA_DESC_CFG_LAST_FRAG) @@ -243,7 +263,10 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) mv_cesa_update_op_cfg(op, frag_mode, CESA_SA_DESC_CFG_FRAG_MSK); /* FIXME: only update enc_len field */ - memcpy_toio(engine->sram, op, sizeof(*op)); + if (engine->pool) + memcpy(engine->sram_pool, op, sizeof(*op)); + else + memcpy_toio(engine->sram, op, sizeof(*op)); if (frag_mode == CESA_SA_DESC_CFG_FIRST_FRAG) mv_cesa_update_op_cfg(op, CESA_SA_DESC_CFG_MID_FRAG, diff --git a/drivers/crypto/marvell/cesa/tdma.c b/drivers/crypto/marvell/cesa/tdma.c index 5d9c48fb72b2..f0b5537038c2 100644 --- a/drivers/crypto/marvell/cesa/tdma.c +++ b/drivers/crypto/marvell/cesa/tdma.c @@ -177,7 +177,7 @@ int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) /* * Save the last request in error to engine->req, so that the core - * knows which request was fautly + * knows which request was faulty */ if (res) { spin_lock_bh(&engine->lock); @@ -350,3 +350,53 @@ int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, return 0; } + +size_t mv_cesa_sg_copy(struct mv_cesa_engine *engine, + struct scatterlist *sgl, unsigned int nents, + unsigned int sram_off, size_t buflen, off_t skip, + bool to_sram) +{ + unsigned int sg_flags = SG_MITER_ATOMIC; + struct sg_mapping_iter miter; + unsigned int offset = 0; + + if (to_sram) + sg_flags |= SG_MITER_FROM_SG; + else + sg_flags |= SG_MITER_TO_SG; + + sg_miter_start(&miter, sgl, nents, sg_flags); + + if (!sg_miter_skip(&miter, skip)) + return 0; + + while ((offset < buflen) && sg_miter_next(&miter)) { + unsigned int len; + + len = min(miter.length, buflen - offset); + + if (to_sram) { + if (engine->pool) + memcpy(engine->sram_pool + sram_off + offset, + miter.addr, len); + else + memcpy_toio(engine->sram + sram_off + offset, + miter.addr, len); + } else { + if (engine->pool) + memcpy(miter.addr, + engine->sram_pool + sram_off + offset, + len); + else + memcpy_fromio(miter.addr, + engine->sram + sram_off + offset, + len); + } + + offset += len; + } + + sg_miter_stop(&miter); + + return offset; +} diff --git a/drivers/crypto/marvell/octeontx2/Makefile b/drivers/crypto/marvell/octeontx2/Makefile new file mode 100644 index 000000000000..b9c6201019e0 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += octeontx2-cpt.o octeontx2-cptvf.o + +octeontx2-cpt-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \ + otx2_cpt_mbox_common.o otx2_cptpf_ucode.o otx2_cptlf.o +octeontx2-cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o otx2_cptlf.o \ + otx2_cpt_mbox_common.o otx2_cptvf_reqmgr.o \ + otx2_cptvf_algs.o + +ccflags-y += -I$(srctree)/drivers/net/ethernet/marvell/octeontx2/af diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h new file mode 100644 index 000000000000..3518fac29834 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_COMMON_H +#define __OTX2_CPT_COMMON_H + +#include +#include +#include +#include +#include +#include "otx2_cpt_hw_types.h" +#include "rvu.h" +#include "mbox.h" + +#define OTX2_CPT_MAX_VFS_NUM 128 +#define OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs) \ + (((blk) << 20) | ((slot) << 12) | (offs)) +#define OTX2_CPT_RVU_PFFUNC(pf, func) \ + ((((pf) & RVU_PFVF_PF_MASK) << RVU_PFVF_PF_SHIFT) | \ + (((func) & RVU_PFVF_FUNC_MASK) << RVU_PFVF_FUNC_SHIFT)) + +#define OTX2_CPT_INVALID_CRYPTO_ENG_GRP 0xFF +#define OTX2_CPT_NAME_LENGTH 64 +#define OTX2_CPT_DMA_MINALIGN 128 + +#define BAD_OTX2_CPT_ENG_TYPE OTX2_CPT_MAX_ENG_TYPES + +enum otx2_cpt_eng_type { + OTX2_CPT_AE_TYPES = 1, + OTX2_CPT_SE_TYPES = 2, + OTX2_CPT_IE_TYPES = 3, + OTX2_CPT_MAX_ENG_TYPES, +}; + +/* Take mbox id from end of CPT mbox range in AF (range 0xA00 - 0xBFF) */ +#define MBOX_MSG_GET_ENG_GRP_NUM 0xBFF +#define MBOX_MSG_GET_CAPS 0xBFD +#define MBOX_MSG_GET_KVF_LIMITS 0xBFC + +/* + * Message request and response to get engine group number + * which has attached a given type of engines (SE, AE, IE) + * This messages are only used between CPT PF <=> CPT VF + */ +struct otx2_cpt_egrp_num_msg { + struct mbox_msghdr hdr; + u8 eng_type; +}; + +struct otx2_cpt_egrp_num_rsp { + struct mbox_msghdr hdr; + u8 eng_type; + u8 eng_grp_num; +}; + +/* + * Message request and response to get kernel crypto limits + * This messages are only used between CPT PF <-> CPT VF + */ +struct otx2_cpt_kvf_limits_msg { + struct mbox_msghdr hdr; +}; + +struct otx2_cpt_kvf_limits_rsp { + struct mbox_msghdr hdr; + u8 kvf_limits; +}; + +/* CPT HW capabilities */ +union otx2_cpt_eng_caps { + u64 u; + struct { + u64 reserved_0_4:5; + u64 mul:1; + u64 sha1_sha2:1; + u64 chacha20:1; + u64 zuc_snow3g:1; + u64 sha3:1; + u64 aes:1; + u64 kasumi:1; + u64 des:1; + u64 crc:1; + u64 reserved_14_63:50; + }; +}; + +/* + * Message request and response to get HW capabilities for each + * engine type (SE, IE, AE). + * This messages are only used between CPT PF <=> CPT VF + */ +struct otx2_cpt_caps_msg { + struct mbox_msghdr hdr; +}; + +struct otx2_cpt_caps_rsp { + struct mbox_msghdr hdr; + u16 cpt_pf_drv_version; + u8 cpt_revision; + union otx2_cpt_eng_caps eng_caps[OTX2_CPT_MAX_ENG_TYPES]; +}; + +static inline void otx2_cpt_write64(void __iomem *reg_base, u64 blk, u64 slot, + u64 offs, u64 val) +{ + writeq_relaxed(val, reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs)); +} + +static inline u64 otx2_cpt_read64(void __iomem *reg_base, u64 blk, u64 slot, + u64 offs) +{ + return readq_relaxed(reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs)); +} + +int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); +int otx2_cpt_send_mbox_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); + +int otx2_cpt_send_af_reg_requests(struct otx2_mbox *mbox, + struct pci_dev *pdev); +int otx2_cpt_add_read_af_reg(struct otx2_mbox *mbox, + struct pci_dev *pdev, u64 reg, u64 *val); +int otx2_cpt_add_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val); +int otx2_cpt_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 *val); +int otx2_cpt_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val); +struct otx2_cptlfs_info; +int otx2_cpt_attach_rscrs_msg(struct otx2_cptlfs_info *lfs); +int otx2_cpt_detach_rsrcs_msg(struct otx2_cptlfs_info *lfs); +int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs); + +#endif /* __OTX2_CPT_COMMON_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h new file mode 100644 index 000000000000..ecafc42f37a2 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h @@ -0,0 +1,464 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_HW_TYPES_H +#define __OTX2_CPT_HW_TYPES_H + +#include + +/* Device IDs */ +#define OTX2_CPT_PCI_PF_DEVICE_ID 0xA0FD +#define OTX2_CPT_PCI_VF_DEVICE_ID 0xA0FE + +/* Mailbox interrupts offset */ +#define OTX2_CPT_PF_MBOX_INT 6 +#define OTX2_CPT_PF_INT_VEC_E_MBOXX(x, a) ((x) + (a)) + +/* Maximum supported microcode groups */ +#define OTX2_CPT_MAX_ENGINE_GROUPS 8 + +/* CPT instruction size in bytes */ +#define OTX2_CPT_INST_SIZE 64 +/* + * CPT VF MSIX vectors and their offsets + */ +#define OTX2_CPT_VF_MSIX_VECTORS 1 +#define OTX2_CPT_VF_INTR_MBOX_MASK BIT(0) + +/* CPT LF MSIX vectors */ +#define OTX2_CPT_LF_MSIX_VECTORS 2 + +/* OcteonTX2 CPT PF registers */ +#define OTX2_CPT_PF_CONSTANTS (0x0) +#define OTX2_CPT_PF_RESET (0x100) +#define OTX2_CPT_PF_DIAG (0x120) +#define OTX2_CPT_PF_BIST_STATUS (0x160) +#define OTX2_CPT_PF_ECC0_CTL (0x200) +#define OTX2_CPT_PF_ECC0_FLIP (0x210) +#define OTX2_CPT_PF_ECC0_INT (0x220) +#define OTX2_CPT_PF_ECC0_INT_W1S (0x230) +#define OTX2_CPT_PF_ECC0_ENA_W1S (0x240) +#define OTX2_CPT_PF_ECC0_ENA_W1C (0x250) +#define OTX2_CPT_PF_MBOX_INTX(b) (0x400 | (b) << 3) +#define OTX2_CPT_PF_MBOX_INT_W1SX(b) (0x420 | (b) << 3) +#define OTX2_CPT_PF_MBOX_ENA_W1CX(b) (0x440 | (b) << 3) +#define OTX2_CPT_PF_MBOX_ENA_W1SX(b) (0x460 | (b) << 3) +#define OTX2_CPT_PF_EXEC_INT (0x500) +#define OTX2_CPT_PF_EXEC_INT_W1S (0x520) +#define OTX2_CPT_PF_EXEC_ENA_W1C (0x540) +#define OTX2_CPT_PF_EXEC_ENA_W1S (0x560) +#define OTX2_CPT_PF_GX_EN(b) (0x600 | (b) << 3) +#define OTX2_CPT_PF_EXEC_INFO (0x700) +#define OTX2_CPT_PF_EXEC_BUSY (0x800) +#define OTX2_CPT_PF_EXEC_INFO0 (0x900) +#define OTX2_CPT_PF_EXEC_INFO1 (0x910) +#define OTX2_CPT_PF_INST_REQ_PC (0x10000) +#define OTX2_CPT_PF_INST_LATENCY_PC (0x10020) +#define OTX2_CPT_PF_RD_REQ_PC (0x10040) +#define OTX2_CPT_PF_RD_LATENCY_PC (0x10060) +#define OTX2_CPT_PF_RD_UC_PC (0x10080) +#define OTX2_CPT_PF_ACTIVE_CYCLES_PC (0x10100) +#define OTX2_CPT_PF_EXE_CTL (0x4000000) +#define OTX2_CPT_PF_EXE_STATUS (0x4000008) +#define OTX2_CPT_PF_EXE_CLK (0x4000010) +#define OTX2_CPT_PF_EXE_DBG_CTL (0x4000018) +#define OTX2_CPT_PF_EXE_DBG_DATA (0x4000020) +#define OTX2_CPT_PF_EXE_BIST_STATUS (0x4000028) +#define OTX2_CPT_PF_EXE_REQ_TIMER (0x4000030) +#define OTX2_CPT_PF_EXE_MEM_CTL (0x4000038) +#define OTX2_CPT_PF_EXE_PERF_CTL (0x4001000) +#define OTX2_CPT_PF_EXE_DBG_CNTX(b) (0x4001100 | (b) << 3) +#define OTX2_CPT_PF_EXE_PERF_EVENT_CNT (0x4001180) +#define OTX2_CPT_PF_EXE_EPCI_INBX_CNT(b) (0x4001200 | (b) << 3) +#define OTX2_CPT_PF_EXE_EPCI_OUTBX_CNT(b) (0x4001240 | (b) << 3) +#define OTX2_CPT_PF_ENGX_UCODE_BASE(b) (0x4002000 | (b) << 3) +#define OTX2_CPT_PF_QX_CTL(b) (0x8000000 | (b) << 20) +#define OTX2_CPT_PF_QX_GMCTL(b) (0x8000020 | (b) << 20) +#define OTX2_CPT_PF_QX_CTL2(b) (0x8000100 | (b) << 20) +#define OTX2_CPT_PF_VFX_MBOXX(b, c) (0x8001000 | (b) << 20 | \ + (c) << 8) + +/* OcteonTX2 CPT LF registers */ +#define OTX2_CPT_LF_CTL (0x10) +#define OTX2_CPT_LF_DONE_WAIT (0x30) +#define OTX2_CPT_LF_INPROG (0x40) +#define OTX2_CPT_LF_DONE (0x50) +#define OTX2_CPT_LF_DONE_ACK (0x60) +#define OTX2_CPT_LF_DONE_INT_ENA_W1S (0x90) +#define OTX2_CPT_LF_DONE_INT_ENA_W1C (0xa0) +#define OTX2_CPT_LF_MISC_INT (0xb0) +#define OTX2_CPT_LF_MISC_INT_W1S (0xc0) +#define OTX2_CPT_LF_MISC_INT_ENA_W1S (0xd0) +#define OTX2_CPT_LF_MISC_INT_ENA_W1C (0xe0) +#define OTX2_CPT_LF_Q_BASE (0xf0) +#define OTX2_CPT_LF_Q_SIZE (0x100) +#define OTX2_CPT_LF_Q_INST_PTR (0x110) +#define OTX2_CPT_LF_Q_GRP_PTR (0x120) +#define OTX2_CPT_LF_NQX(a) (0x400 | (a) << 3) +#define OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT 20 +/* LMT LF registers */ +#define OTX2_CPT_LMT_LFBASE BIT_ULL(OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT) +#define OTX2_CPT_LMT_LF_LMTLINEX(a) (OTX2_CPT_LMT_LFBASE | 0x000 | \ + (a) << 12) +/* RVU VF registers */ +#define OTX2_RVU_VF_INT (0x20) +#define OTX2_RVU_VF_INT_W1S (0x28) +#define OTX2_RVU_VF_INT_ENA_W1S (0x30) +#define OTX2_RVU_VF_INT_ENA_W1C (0x38) + +/* + * Enumeration otx2_cpt_ucode_error_code_e + * + * Enumerates ucode errors + */ +enum otx2_cpt_ucode_comp_code_e { + OTX2_CPT_UCC_SUCCESS = 0x00, + OTX2_CPT_UCC_INVALID_OPCODE = 0x01, + + /* Scatter gather */ + OTX2_CPT_UCC_SG_WRITE_LENGTH = 0x02, + OTX2_CPT_UCC_SG_LIST = 0x03, + OTX2_CPT_UCC_SG_NOT_SUPPORTED = 0x04, + +}; + +/* + * Enumeration otx2_cpt_comp_e + * + * OcteonTX2 CPT Completion Enumeration + * Enumerates the values of CPT_RES_S[COMPCODE]. + */ +enum otx2_cpt_comp_e { + OTX2_CPT_COMP_E_NOTDONE = 0x00, + OTX2_CPT_COMP_E_GOOD = 0x01, + OTX2_CPT_COMP_E_FAULT = 0x02, + OTX2_CPT_COMP_E_HWERR = 0x04, + OTX2_CPT_COMP_E_INSTERR = 0x05, + OTX2_CPT_COMP_E_LAST_ENTRY = 0x06 +}; + +/* + * Enumeration otx2_cpt_vf_int_vec_e + * + * OcteonTX2 CPT VF MSI-X Vector Enumeration + * Enumerates the MSI-X interrupt vectors. + */ +enum otx2_cpt_vf_int_vec_e { + OTX2_CPT_VF_INT_VEC_E_MBOX = 0x00 +}; + +/* + * Enumeration otx2_cpt_lf_int_vec_e + * + * OcteonTX2 CPT LF MSI-X Vector Enumeration + * Enumerates the MSI-X interrupt vectors. + */ +enum otx2_cpt_lf_int_vec_e { + OTX2_CPT_LF_INT_VEC_E_MISC = 0x00, + OTX2_CPT_LF_INT_VEC_E_DONE = 0x01 +}; + +/* + * Structure otx2_cpt_inst_s + * + * CPT Instruction Structure + * This structure specifies the instruction layout. Instructions are + * stored in memory as little-endian unless CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_inst_s_s + * Word 0 + * doneint:1 Done interrupt. + * 0 = No interrupts related to this instruction. + * 1 = When the instruction completes, CPT()_VQ()_DONE[DONE] will be + * incremented,and based on the rules described there an interrupt may + * occur. + * Word 1 + * res_addr [127: 64] Result IOVA. + * If nonzero, specifies where to write CPT_RES_S. + * If zero, no result structure will be written. + * Address must be 16-byte aligned. + * Bits <63:49> are ignored by hardware; software should use a + * sign-extended bit <48> for forward compatibility. + * Word 2 + * grp:10 [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to use when + * CPT submits work SSO. + * For the SSO to not discard the add-work request, FPA_PF_MAP() must map + * [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid. + * tt:2 [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use when CPT + * submits work to SSO + * tag:32 [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when CPT + * submits work to SSO. + * Word 3 + * wq_ptr [255:192] If [WQ_PTR] is nonzero, it is a pointer to a + * work-queue entry that CPT submits work to SSO after all context, + * output data, and result write operations are visible to other + * CNXXXX units and the cores. Bits <2:0> must be zero. + * Bits <63:49> are ignored by hardware; software should + * use a sign-extended bit <48> for forward compatibility. + * Internal: + * Bits <63:49>, <2:0> are ignored by hardware, treated as always 0x0. + * Word 4 + * ei0; [319:256] Engine instruction word 0. Passed to the AE/SE. + * Word 5 + * ei1; [383:320] Engine instruction word 1. Passed to the AE/SE. + * Word 6 + * ei2; [447:384] Engine instruction word 1. Passed to the AE/SE. + * Word 7 + * ei3; [511:448] Engine instruction word 1. Passed to the AE/SE. + * + */ +union otx2_cpt_inst_s { + u64 u[8]; + + struct { + /* Word 0 */ + u64 nixtxl:3; + u64 doneint:1; + u64 nixtx_addr:60; + /* Word 1 */ + u64 res_addr; + /* Word 2 */ + u64 tag:32; + u64 tt:2; + u64 grp:10; + u64 reserved_172_175:4; + u64 rvu_pf_func:16; + /* Word 3 */ + u64 qord:1; + u64 reserved_194_193:2; + u64 wq_ptr:61; + /* Word 4 */ + u64 ei0; + /* Word 5 */ + u64 ei1; + /* Word 6 */ + u64 ei2; + /* Word 7 */ + u64 ei3; + } s; +}; + +/* + * Structure otx2_cpt_res_s + * + * CPT Result Structure + * The CPT coprocessor writes the result structure after it completes a + * CPT_INST_S instruction. The result structure is exactly 16 bytes, and + * each instruction completion produces exactly one result structure. + * + * This structure is stored in memory as little-endian unless + * CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_res_s_s + * Word 0 + * doneint:1 [16:16] Done interrupt. This bit is copied from the + * corresponding instruction's CPT_INST_S[DONEINT]. + * compcode:8 [7:0] Indicates completion/error status of the CPT coprocessor + * for the associated instruction, as enumerated by CPT_COMP_E. + * Core software may write the memory location containing [COMPCODE] to + * 0x0 before ringing the doorbell, and then poll for completion by + * checking for a nonzero value. + * Once the core observes a nonzero [COMPCODE] value in this case,the CPT + * coprocessor will have also completed L2/DRAM write operations. + * Word 1 + * reserved + * + */ +union otx2_cpt_res_s { + u64 u[2]; + + struct { + u64 compcode:8; + u64 uc_compcode:8; + u64 doneint:1; + u64 reserved_17_63:47; + u64 reserved_64_127; + } s; +}; + +/* + * Register (RVU_PF_BAR0) cpt#_af_constants1 + * + * CPT AF Constants Register + * This register contains implementation-related parameters of CPT. + */ +union otx2_cptx_af_constants1 { + u64 u; + struct otx2_cptx_af_constants1_s { + u64 se:16; + u64 ie:16; + u64 ae:16; + u64 reserved_48_63:16; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_misc_int + * + * This register contain the per-queue miscellaneous interrupts. + * + */ +union otx2_cptx_lf_misc_int { + u64 u; + struct otx2_cptx_lf_misc_int_s { + u64 reserved_0:1; + u64 nqerr:1; + u64 irde:1; + u64 nwrp:1; + u64 reserved_4:1; + u64 hwerr:1; + u64 fault:1; + u64 reserved_7_63:57; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_misc_int_ena_w1s + * + * This register sets interrupt enable bits. + * + */ +union otx2_cptx_lf_misc_int_ena_w1s { + u64 u; + struct otx2_cptx_lf_misc_int_ena_w1s_s { + u64 reserved_0:1; + u64 nqerr:1; + u64 irde:1; + u64 nwrp:1; + u64 reserved_4:1; + u64 hwerr:1; + u64 fault:1; + u64 reserved_7_63:57; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_ctl + * + * This register configures the queue. + * + * When the queue is not execution-quiescent (see CPT_LF_INPROG[EENA,INFLIGHT]), + * software must only write this register with [ENA]=0. + */ +union otx2_cptx_lf_ctl { + u64 u; + struct otx2_cptx_lf_ctl_s { + u64 ena:1; + u64 fc_ena:1; + u64 fc_up_crossing:1; + u64 reserved_3:1; + u64 fc_hyst_bits:4; + u64 reserved_8_63:56; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_done_wait + * + * This register specifies the per-queue interrupt coalescing settings. + */ +union otx2_cptx_lf_done_wait { + u64 u; + struct otx2_cptx_lf_done_wait_s { + u64 num_wait:20; + u64 reserved_20_31:12; + u64 time_wait:16; + u64 reserved_48_63:16; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_done + * + * This register contain the per-queue instruction done count. + */ +union otx2_cptx_lf_done { + u64 u; + struct otx2_cptx_lf_done_s { + u64 done:20; + u64 reserved_20_63:44; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_inprog + * + * These registers contain the per-queue instruction in flight registers. + * + */ +union otx2_cptx_lf_inprog { + u64 u; + struct otx2_cptx_lf_inprog_s { + u64 inflight:9; + u64 reserved_9_15:7; + u64 eena:1; + u64 grp_drp:1; + u64 reserved_18_30:13; + u64 grb_partial:1; + u64 grb_cnt:8; + u64 gwb_cnt:8; + u64 reserved_48_63:16; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_q_base + * + * CPT initializes these CSR fields to these values on any CPT_LF_Q_BASE write: + * _ CPT_LF_Q_INST_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_INST_PTR[NQ_PTR]=2. + * _ CPT_LF_Q_INST_PTR[DQ_PTR]=2. + * _ CPT_LF_Q_GRP_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_GRP_PTR[NQ_PTR]=1. + * _ CPT_LF_Q_GRP_PTR[DQ_PTR]=1. + */ +union otx2_cptx_lf_q_base { + u64 u; + struct otx2_cptx_lf_q_base_s { + u64 fault:1; + u64 reserved_1_6:6; + u64 addr:46; + u64 reserved_53_63:11; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_q_size + * + * CPT initializes these CSR fields to these values on any CPT_LF_Q_SIZE write: + * _ CPT_LF_Q_INST_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_INST_PTR[NQ_PTR]=2. + * _ CPT_LF_Q_INST_PTR[DQ_PTR]=2. + * _ CPT_LF_Q_GRP_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_GRP_PTR[NQ_PTR]=1. + * _ CPT_LF_Q_GRP_PTR[DQ_PTR]=1. + */ +union otx2_cptx_lf_q_size { + u64 u; + struct otx2_cptx_lf_q_size_s { + u64 size_div40:15; + u64 reserved_15_63:49; + } s; +}; + +/* + * RVU_PF_BAR0 - cpt_af_lf_ctl + * + * This register configures queues. This register should be written only + * when the queue is execution-quiescent (see CPT_LF_INPROG[INFLIGHT]). + */ +union otx2_cptx_af_lf_ctrl { + u64 u; + struct otx2_cptx_af_lf_ctrl_s { + u64 pri:1; + u64 reserved_1_8:8; + u64 pf_func_inst:1; + u64 cont_err:1; + u64 reserved_11_15:5; + u64 nixtx_en:1; + u64 reserved_17_47:31; + u64 grp:8; + u64 reserved_56_63:8; + } s; +}; + +#endif /* __OTX2_CPT_HW_TYPES_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c new file mode 100644 index 000000000000..51cb6404ded7 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptlf.h" + +int otx2_cpt_send_mbox_msg(struct otx2_mbox *mbox, struct pci_dev *pdev) +{ + int ret; + + otx2_mbox_msg_send(mbox, 0); + ret = otx2_mbox_wait_for_rsp(mbox, 0); + if (ret == -EIO) { + dev_err(&pdev->dev, "RVU MBOX timeout.\n"); + return ret; + } else if (ret) { + dev_err(&pdev->dev, "RVU MBOX error: %d.\n", ret); + return -EFAULT; + } + return ret; +} + +int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev) +{ + struct mbox_msghdr *req; + + req = otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct ready_msg_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->id = MBOX_MSG_READY; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = 0; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cpt_send_af_reg_requests(struct otx2_mbox *mbox, struct pci_dev *pdev) +{ + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cpt_add_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 *val) +{ + struct cpt_rd_wr_reg_msg *reg_msg; + + reg_msg = (struct cpt_rd_wr_reg_msg *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*reg_msg), + sizeof(*reg_msg)); + if (reg_msg == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + reg_msg->hdr.id = MBOX_MSG_CPT_RD_WR_REGISTER; + reg_msg->hdr.sig = OTX2_MBOX_REQ_SIG; + reg_msg->hdr.pcifunc = 0; + + reg_msg->is_write = 0; + reg_msg->reg_offset = reg; + reg_msg->ret_val = val; + + return 0; +} + +int otx2_cpt_add_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val) +{ + struct cpt_rd_wr_reg_msg *reg_msg; + + reg_msg = (struct cpt_rd_wr_reg_msg *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*reg_msg), + sizeof(*reg_msg)); + if (reg_msg == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + reg_msg->hdr.id = MBOX_MSG_CPT_RD_WR_REGISTER; + reg_msg->hdr.sig = OTX2_MBOX_REQ_SIG; + reg_msg->hdr.pcifunc = 0; + + reg_msg->is_write = 1; + reg_msg->reg_offset = reg; + reg_msg->val = val; + + return 0; +} + +int otx2_cpt_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 *val) +{ + int ret; + + ret = otx2_cpt_add_read_af_reg(mbox, pdev, reg, val); + if (ret) + return ret; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cpt_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val) +{ + int ret; + + ret = otx2_cpt_add_write_af_reg(mbox, pdev, reg, val); + if (ret) + return ret; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cpt_attach_rscrs_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct rsrc_attach *req; + int ret; + + req = (struct rsrc_attach *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (req == NULL) { + dev_err(&lfs->pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_ATTACH_RESOURCES; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + req->cptlfs = lfs->lfs_num; + ret = otx2_cpt_send_mbox_msg(mbox, lfs->pdev); + if (ret) + return ret; + + if (!lfs->are_lfs_attached) + ret = -EINVAL; + + return ret; +} + +int otx2_cpt_detach_rsrcs_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct rsrc_detach *req; + int ret; + + req = (struct rsrc_detach *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (req == NULL) { + dev_err(&lfs->pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_DETACH_RESOURCES; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + ret = otx2_cpt_send_mbox_msg(mbox, lfs->pdev); + if (ret) + return ret; + + if (lfs->are_lfs_attached) + ret = -EINVAL; + + return ret; +} + +int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct pci_dev *pdev = lfs->pdev; + struct mbox_msghdr *req; + int ret, i; + + req = otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msix_offset_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->id = MBOX_MSG_MSIX_OFFSET; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = 0; + ret = otx2_cpt_send_mbox_msg(mbox, pdev); + if (ret) + return ret; + + for (i = 0; i < lfs->lfs_num; i++) { + if (lfs->lf[i].msix_offset == MSIX_VECTOR_INVALID) { + dev_err(&pdev->dev, + "Invalid msix offset %d for LF %d\n", + lfs->lf[i].msix_offset, i); + return -EINVAL; + } + } + return ret; +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h new file mode 100644 index 000000000000..dbb1ee746f4c --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_REQMGR_H +#define __OTX2_CPT_REQMGR_H + +#include "otx2_cpt_common.h" + +/* Completion code size and initial value */ +#define OTX2_CPT_COMPLETION_CODE_SIZE 8 +#define OTX2_CPT_COMPLETION_CODE_INIT OTX2_CPT_COMP_E_NOTDONE +/* + * Maximum total number of SG buffers is 100, we divide it equally + * between input and output + */ +#define OTX2_CPT_MAX_SG_IN_CNT 50 +#define OTX2_CPT_MAX_SG_OUT_CNT 50 + +/* DMA mode direct or SG */ +#define OTX2_CPT_DMA_MODE_DIRECT 0 +#define OTX2_CPT_DMA_MODE_SG 1 + +/* Context source CPTR or DPTR */ +#define OTX2_CPT_FROM_CPTR 0 +#define OTX2_CPT_FROM_DPTR 1 + +#define OTX2_CPT_MAX_REQ_SIZE 65535 + +union otx2_cpt_opcode { + u16 flags; + struct { + u8 major; + u8 minor; + } s; +}; + +struct otx2_cptvf_request { + u32 param1; + u32 param2; + u16 dlen; + union otx2_cpt_opcode opcode; +}; + +/* + * CPT_INST_S software command definitions + * Words EI (0-3) + */ +union otx2_cpt_iq_cmd_word0 { + u64 u; + struct { + __be16 opcode; + __be16 param1; + __be16 param2; + __be16 dlen; + } s; +}; + +union otx2_cpt_iq_cmd_word3 { + u64 u; + struct { + u64 cptr:61; + u64 grp:3; + } s; +}; + +struct otx2_cpt_iq_command { + union otx2_cpt_iq_cmd_word0 cmd; + u64 dptr; + u64 rptr; + union otx2_cpt_iq_cmd_word3 cptr; +}; + +struct otx2_cpt_pending_entry { + void *completion_addr; /* Completion address */ + void *info; + /* Kernel async request callback */ + void (*callback)(int status, void *arg1, void *arg2); + struct crypto_async_request *areq; /* Async request callback arg */ + u8 resume_sender; /* Notify sender to resume sending requests */ + u8 busy; /* Entry status (free/busy) */ +}; + +struct otx2_cpt_pending_queue { + struct otx2_cpt_pending_entry *head; /* Head of the queue */ + u32 front; /* Process work from here */ + u32 rear; /* Append new work here */ + u32 pending_count; /* Pending requests count */ + u32 qlen; /* Queue length */ + spinlock_t lock; /* Queue lock */ +}; + +struct otx2_cpt_buf_ptr { + u8 *vptr; + dma_addr_t dma_addr; + u16 size; +}; + +union otx2_cpt_ctrl_info { + u32 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u32 reserved_6_31:26; + u32 grp:3; /* Group bits */ + u32 dma_mode:2; /* DMA mode */ + u32 se_req:1; /* To SE core */ +#else + u32 se_req:1; /* To SE core */ + u32 dma_mode:2; /* DMA mode */ + u32 grp:3; /* Group bits */ + u32 reserved_6_31:26; +#endif + } s; +}; + +struct otx2_cpt_req_info { + /* Kernel async request callback */ + void (*callback)(int status, void *arg1, void *arg2); + struct crypto_async_request *areq; /* Async request callback arg */ + struct otx2_cptvf_request req;/* Request information (core specific) */ + union otx2_cpt_ctrl_info ctrl;/* User control information */ + struct otx2_cpt_buf_ptr in[OTX2_CPT_MAX_SG_IN_CNT]; + struct otx2_cpt_buf_ptr out[OTX2_CPT_MAX_SG_OUT_CNT]; + u8 *iv_out; /* IV to send back */ + u16 rlen; /* Output length */ + u8 in_cnt; /* Number of input buffers */ + u8 out_cnt; /* Number of output buffers */ + u8 req_type; /* Type of request */ + u8 is_enc; /* Is a request an encryption request */ + u8 is_trunc_hmac;/* Is truncated hmac used */ +}; + +struct otx2_cpt_inst_info { + struct otx2_cpt_pending_entry *pentry; + struct otx2_cpt_req_info *req; + struct pci_dev *pdev; + void *completion_addr; + u8 *out_buffer; + u8 *in_buffer; + dma_addr_t dptr_baddr; + dma_addr_t rptr_baddr; + dma_addr_t comp_baddr; + unsigned long time_in; + u32 dlen; + u32 dma_len; + u8 extra_time; +}; + +struct otx2_cpt_sglist_component { + __be16 len0; + __be16 len1; + __be16 len2; + __be16 len3; + __be64 ptr0; + __be64 ptr1; + __be64 ptr2; + __be64 ptr3; +}; + +static inline void otx2_cpt_info_destroy(struct pci_dev *pdev, + struct otx2_cpt_inst_info *info) +{ + struct otx2_cpt_req_info *req; + int i; + + if (info->dptr_baddr) + dma_unmap_single(&pdev->dev, info->dptr_baddr, + info->dma_len, DMA_BIDIRECTIONAL); + + if (info->req) { + req = info->req; + for (i = 0; i < req->out_cnt; i++) { + if (req->out[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->out[i].dma_addr, + req->out[i].size, + DMA_BIDIRECTIONAL); + } + + for (i = 0; i < req->in_cnt; i++) { + if (req->in[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->in[i].dma_addr, + req->in[i].size, + DMA_BIDIRECTIONAL); + } + } + kfree(info); +} + +struct otx2_cptlf_wqe; +int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + int cpu_num); +void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe); +int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev); + +#endif /* __OTX2_CPT_REQMGR_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c new file mode 100644 index 000000000000..823a4571fd67 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptlf.h" +#include "rvu_reg.h" + +#define CPT_TIMER_HOLD 0x03F +#define CPT_COUNT_HOLD 32 + +static void cptlf_do_set_done_time_wait(struct otx2_cptlf_info *lf, + int time_wait) +{ + union otx2_cptx_lf_done_wait done_wait; + + done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_WAIT); + done_wait.s.time_wait = time_wait; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_WAIT, done_wait.u); +} + +static void cptlf_do_set_done_num_wait(struct otx2_cptlf_info *lf, int num_wait) +{ + union otx2_cptx_lf_done_wait done_wait; + + done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_WAIT); + done_wait.s.num_wait = num_wait; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_WAIT, done_wait.u); +} + +static void cptlf_set_done_time_wait(struct otx2_cptlfs_info *lfs, + int time_wait) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + cptlf_do_set_done_time_wait(&lfs->lf[slot], time_wait); +} + +static void cptlf_set_done_num_wait(struct otx2_cptlfs_info *lfs, int num_wait) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + cptlf_do_set_done_num_wait(&lfs->lf[slot], num_wait); +} + +static int cptlf_set_pri(struct otx2_cptlf_info *lf, int pri) +{ + struct otx2_cptlfs_info *lfs = lf->lfs; + union otx2_cptx_af_lf_ctrl lf_ctrl; + int ret; + + ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + &lf_ctrl.u); + if (ret) + return ret; + + lf_ctrl.s.pri = pri ? 1 : 0; + + ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + lf_ctrl.u); + return ret; +} + +static int cptlf_set_eng_grps_mask(struct otx2_cptlf_info *lf, + int eng_grps_mask) +{ + struct otx2_cptlfs_info *lfs = lf->lfs; + union otx2_cptx_af_lf_ctrl lf_ctrl; + int ret; + + ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + &lf_ctrl.u); + if (ret) + return ret; + + lf_ctrl.s.grp = eng_grps_mask; + + ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + lf_ctrl.u); + return ret; +} + +static int cptlf_set_grp_and_pri(struct otx2_cptlfs_info *lfs, + int eng_grp_mask, int pri) +{ + int slot, ret = 0; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + ret = cptlf_set_pri(&lfs->lf[slot], pri); + if (ret) + return ret; + + ret = cptlf_set_eng_grps_mask(&lfs->lf[slot], eng_grp_mask); + if (ret) + return ret; + } + return ret; +} + +static void cptlf_hw_init(struct otx2_cptlfs_info *lfs) +{ + /* Disable instruction queues */ + otx2_cptlf_disable_iqueues(lfs); + + /* Set instruction queues base addresses */ + otx2_cptlf_set_iqueues_base_addr(lfs); + + /* Set instruction queues sizes */ + otx2_cptlf_set_iqueues_size(lfs); + + /* Set done interrupts time wait */ + cptlf_set_done_time_wait(lfs, CPT_TIMER_HOLD); + + /* Set done interrupts num wait */ + cptlf_set_done_num_wait(lfs, CPT_COUNT_HOLD); + + /* Enable instruction queues */ + otx2_cptlf_enable_iqueues(lfs); +} + +static void cptlf_hw_cleanup(struct otx2_cptlfs_info *lfs) +{ + /* Disable instruction queues */ + otx2_cptlf_disable_iqueues(lfs); +} + +static void cptlf_set_misc_intrs(struct otx2_cptlfs_info *lfs, u8 enable) +{ + union otx2_cptx_lf_misc_int_ena_w1s irq_misc = { .u = 0x0 }; + u64 reg = enable ? OTX2_CPT_LF_MISC_INT_ENA_W1S : + OTX2_CPT_LF_MISC_INT_ENA_W1C; + int slot; + + irq_misc.s.fault = 0x1; + irq_misc.s.hwerr = 0x1; + irq_misc.s.irde = 0x1; + irq_misc.s.nqerr = 0x1; + irq_misc.s.nwrp = 0x1; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, reg, + irq_misc.u); +} + +static void cptlf_enable_intrs(struct otx2_cptlfs_info *lfs) +{ + int slot; + + /* Enable done interrupts */ + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, + OTX2_CPT_LF_DONE_INT_ENA_W1S, 0x1); + /* Enable Misc interrupts */ + cptlf_set_misc_intrs(lfs, true); +} + +static void cptlf_disable_intrs(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, + OTX2_CPT_LF_DONE_INT_ENA_W1C, 0x1); + cptlf_set_misc_intrs(lfs, false); +} + +static inline int cptlf_read_done_cnt(struct otx2_cptlf_info *lf) +{ + union otx2_cptx_lf_done irq_cnt; + + irq_cnt.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE); + return irq_cnt.s.done; +} + +static irqreturn_t cptlf_misc_intr_handler(int __always_unused irq, void *arg) +{ + union otx2_cptx_lf_misc_int irq_misc, irq_misc_ack; + struct otx2_cptlf_info *lf = arg; + struct device *dev; + + dev = &lf->lfs->pdev->dev; + irq_misc.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_MISC_INT); + irq_misc_ack.u = 0x0; + + if (irq_misc.s.fault) { + dev_err(dev, "Memory error detected while executing CPT_INST_S, LF %d.\n", + lf->slot); + irq_misc_ack.s.fault = 0x1; + + } else if (irq_misc.s.hwerr) { + dev_err(dev, "HW error from an engine executing CPT_INST_S, LF %d.", + lf->slot); + irq_misc_ack.s.hwerr = 0x1; + + } else if (irq_misc.s.nwrp) { + dev_err(dev, "SMMU fault while writing CPT_RES_S to CPT_INST_S[RES_ADDR], LF %d.\n", + lf->slot); + irq_misc_ack.s.nwrp = 0x1; + + } else if (irq_misc.s.irde) { + dev_err(dev, "Memory error when accessing instruction memory queue CPT_LF_Q_BASE[ADDR].\n"); + irq_misc_ack.s.irde = 0x1; + + } else if (irq_misc.s.nqerr) { + dev_err(dev, "Error enqueuing an instruction received at CPT_LF_NQ.\n"); + irq_misc_ack.s.nqerr = 0x1; + + } else { + dev_err(dev, "Unhandled interrupt in CPT LF %d\n", lf->slot); + return IRQ_NONE; + } + + /* Acknowledge interrupts */ + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_MISC_INT, irq_misc_ack.u); + + return IRQ_HANDLED; +} + +static irqreturn_t cptlf_done_intr_handler(int irq, void *arg) +{ + union otx2_cptx_lf_done_wait done_wait; + struct otx2_cptlf_info *lf = arg; + int irq_cnt; + + /* Read the number of completed requests */ + irq_cnt = cptlf_read_done_cnt(lf); + if (irq_cnt) { + done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, + lf->slot, OTX2_CPT_LF_DONE_WAIT); + /* Acknowledge the number of completed requests */ + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_ACK, irq_cnt); + + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_WAIT, done_wait.u); + if (unlikely(!lf->wqe)) { + dev_err(&lf->lfs->pdev->dev, "No work for LF %d\n", + lf->slot); + return IRQ_NONE; + } + + /* Schedule processing of completed requests */ + tasklet_hi_schedule(&lf->wqe->work); + } + return IRQ_HANDLED; +} + +void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs) +{ + int i, offs, vector; + + for (i = 0; i < lfs->lfs_num; i++) { + for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) { + if (!lfs->lf[i].is_irq_reg[offs]) + continue; + + vector = pci_irq_vector(lfs->pdev, + lfs->lf[i].msix_offset + offs); + free_irq(vector, &lfs->lf[i]); + lfs->lf[i].is_irq_reg[offs] = false; + } + } + cptlf_disable_intrs(lfs); +} + +static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs, + int lf_num, int irq_offset, + irq_handler_t handler) +{ + int ret, vector; + + vector = pci_irq_vector(lfs->pdev, lfs->lf[lf_num].msix_offset + + irq_offset); + ret = request_irq(vector, handler, 0, + lfs->lf[lf_num].irq_name[irq_offset], + &lfs->lf[lf_num]); + if (ret) + return ret; + + lfs->lf[lf_num].is_irq_reg[irq_offset] = true; + + return ret; +} + +int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs) +{ + int irq_offs, ret, i; + + for (i = 0; i < lfs->lfs_num; i++) { + irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC; + snprintf(lfs->lf[i].irq_name[irq_offs], 32, "CPTLF Misc%d", i); + ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, + cptlf_misc_intr_handler); + if (ret) + goto free_irq; + + irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE; + snprintf(lfs->lf[i].irq_name[irq_offs], 32, "OTX2_CPTLF Done%d", + i); + ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, + cptlf_done_intr_handler); + if (ret) + goto free_irq; + } + cptlf_enable_intrs(lfs); + return 0; + +free_irq: + otx2_cptlf_unregister_interrupts(lfs); + return ret; +} + +void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs) +{ + int slot, offs; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) + irq_set_affinity_hint(pci_irq_vector(lfs->pdev, + lfs->lf[slot].msix_offset + + offs), NULL); + free_cpumask_var(lfs->lf[slot].affinity_mask); + } +} + +int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs) +{ + struct otx2_cptlf_info *lf = lfs->lf; + int slot, offs, ret; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + if (!zalloc_cpumask_var(&lf[slot].affinity_mask, GFP_KERNEL)) { + dev_err(&lfs->pdev->dev, + "cpumask allocation failed for LF %d", slot); + ret = -ENOMEM; + goto free_affinity_mask; + } + + cpumask_set_cpu(cpumask_local_spread(slot, + dev_to_node(&lfs->pdev->dev)), + lf[slot].affinity_mask); + + for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) { + ret = irq_set_affinity_hint(pci_irq_vector(lfs->pdev, + lf[slot].msix_offset + offs), + lf[slot].affinity_mask); + if (ret) + goto free_affinity_mask; + } + } + return 0; + +free_affinity_mask: + otx2_cptlf_free_irqs_affinity(lfs); + return ret; +} + +int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri, + int lfs_num) +{ + int slot, ret; + + if (!lfs->pdev || !lfs->reg_base) + return -EINVAL; + + lfs->lfs_num = lfs_num; + for (slot = 0; slot < lfs->lfs_num; slot++) { + lfs->lf[slot].lfs = lfs; + lfs->lf[slot].slot = slot; + lfs->lf[slot].lmtline = lfs->reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot, + OTX2_CPT_LMT_LF_LMTLINEX(0)); + lfs->lf[slot].ioreg = lfs->reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_CPT0, slot, + OTX2_CPT_LF_NQX(0)); + } + /* Send request to attach LFs */ + ret = otx2_cpt_attach_rscrs_msg(lfs); + if (ret) + goto clear_lfs_num; + + ret = otx2_cpt_alloc_instruction_queues(lfs); + if (ret) { + dev_err(&lfs->pdev->dev, + "Allocating instruction queues failed\n"); + goto detach_rsrcs; + } + cptlf_hw_init(lfs); + /* + * Allow each LF to execute requests destined to any of 8 engine + * groups and set queue priority of each LF to high + */ + ret = cptlf_set_grp_and_pri(lfs, eng_grp_mask, pri); + if (ret) + goto free_iq; + + return 0; + +free_iq: + otx2_cpt_free_instruction_queues(lfs); + cptlf_hw_cleanup(lfs); +detach_rsrcs: + otx2_cpt_detach_rsrcs_msg(lfs); +clear_lfs_num: + lfs->lfs_num = 0; + return ret; +} + +void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs) +{ + lfs->lfs_num = 0; + /* Cleanup LFs hardware side */ + cptlf_hw_cleanup(lfs); + /* Send request to detach LFs */ + otx2_cpt_detach_rsrcs_msg(lfs); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h new file mode 100644 index 000000000000..314e97354100 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h @@ -0,0 +1,353 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ +#ifndef __OTX2_CPTLF_H +#define __OTX2_CPTLF_H + +#include +#include +#include +#include "otx2_cpt_common.h" +#include "otx2_cpt_reqmgr.h" + +/* + * CPT instruction and pending queues user requested length in CPT_INST_S msgs + */ +#define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200 + +/* + * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S + * messages. + */ +#define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40) + +/* + * CPT instruction and pending queues length in CPT_INST_S messages + */ +#define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40) + +/* CPT instruction queue length in bytes */ +#define OTX2_CPT_INST_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 40 * \ + OTX2_CPT_INST_SIZE) + +/* CPT instruction group queue length in bytes */ +#define OTX2_CPT_INST_GRP_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 16) + +/* CPT FC length in bytes */ +#define OTX2_CPT_Q_FC_LEN 128 + +/* CPT instruction queue alignment */ +#define OTX2_CPT_INST_Q_ALIGNMENT 128 + +/* Mask which selects all engine groups */ +#define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF + +/* Maximum LFs supported in OcteonTX2 for CPT */ +#define OTX2_CPT_MAX_LFS_NUM 64 + +/* Queue priority */ +#define OTX2_CPT_QUEUE_HI_PRIO 0x1 +#define OTX2_CPT_QUEUE_LOW_PRIO 0x0 + +enum otx2_cptlf_state { + OTX2_CPTLF_IN_RESET, + OTX2_CPTLF_STARTED, +}; + +struct otx2_cpt_inst_queue { + u8 *vaddr; + u8 *real_vaddr; + dma_addr_t dma_addr; + dma_addr_t real_dma_addr; + u32 size; +}; + +struct otx2_cptlfs_info; +struct otx2_cptlf_wqe { + struct tasklet_struct work; + struct otx2_cptlfs_info *lfs; + u8 lf_num; +}; + +struct otx2_cptlf_info { + struct otx2_cptlfs_info *lfs; /* Ptr to cptlfs_info struct */ + void __iomem *lmtline; /* Address of LMTLINE */ + void __iomem *ioreg; /* LMTLINE send register */ + int msix_offset; /* MSI-X interrupts offset */ + cpumask_var_t affinity_mask; /* IRQs affinity mask */ + u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */ + u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS]; /* Is interrupt registered */ + u8 slot; /* Slot number of this LF */ + + struct otx2_cpt_inst_queue iqueue;/* Instruction queue */ + struct otx2_cpt_pending_queue pqueue; /* Pending queue */ + struct otx2_cptlf_wqe *wqe; /* Tasklet work info */ +}; + +struct otx2_cptlfs_info { + /* Registers start address of VF/PF LFs are attached to */ + void __iomem *reg_base; + struct pci_dev *pdev; /* Device LFs are attached to */ + struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; + struct otx2_mbox *mbox; + u8 are_lfs_attached; /* Whether CPT LFs are attached */ + u8 lfs_num; /* Number of CPT LFs */ + u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */ + u8 kvf_limits; /* Kernel crypto limits */ + atomic_t state; /* LF's state. started/reset */ +}; + +static inline void otx2_cpt_free_instruction_queues( + struct otx2_cptlfs_info *lfs) +{ + struct otx2_cpt_inst_queue *iq; + int i; + + for (i = 0; i < lfs->lfs_num; i++) { + iq = &lfs->lf[i].iqueue; + if (iq->real_vaddr) + dma_free_coherent(&lfs->pdev->dev, + iq->size, + iq->real_vaddr, + iq->real_dma_addr); + iq->real_vaddr = NULL; + iq->vaddr = NULL; + } +} + +static inline int otx2_cpt_alloc_instruction_queues( + struct otx2_cptlfs_info *lfs) +{ + struct otx2_cpt_inst_queue *iq; + int ret = 0, i; + + if (!lfs->lfs_num) + return -EINVAL; + + for (i = 0; i < lfs->lfs_num; i++) { + iq = &lfs->lf[i].iqueue; + iq->size = OTX2_CPT_INST_QLEN_BYTES + + OTX2_CPT_Q_FC_LEN + + OTX2_CPT_INST_GRP_QLEN_BYTES + + OTX2_CPT_INST_Q_ALIGNMENT; + iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size, + &iq->real_dma_addr, GFP_KERNEL); + if (!iq->real_vaddr) { + ret = -ENOMEM; + goto error; + } + iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES; + iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES; + + /* Align pointers */ + iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT); + iq->dma_addr = PTR_ALIGN(iq->dma_addr, + OTX2_CPT_INST_Q_ALIGNMENT); + } + return 0; + +error: + otx2_cpt_free_instruction_queues(lfs); + return ret; +} + +static inline void otx2_cptlf_set_iqueues_base_addr( + struct otx2_cptlfs_info *lfs) +{ + union otx2_cptx_lf_q_base lf_q_base; + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + lf_q_base.u = lfs->lf[slot].iqueue.dma_addr; + otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, + OTX2_CPT_LF_Q_BASE, lf_q_base.u); + } +} + +static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf) +{ + union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 }; + + lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_Q_SIZE, lf_q_size.u); +} + +static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]); +} + +static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf) +{ + union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 }; + union otx2_cptx_lf_inprog lf_inprog; + int timeout = 20; + + /* Disable instructions enqueuing */ + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_CTL, lf_ctl.u); + + /* Wait for instruction queue to become empty */ + do { + lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, + lf->slot, OTX2_CPT_LF_INPROG); + if (!lf_inprog.s.inflight) + break; + + usleep_range(10000, 20000); + if (timeout-- < 0) { + dev_err(&lf->lfs->pdev->dev, + "Error LF %d is still busy.\n", lf->slot); + break; + } + + } while (1); + + /* + * Disable executions in the LF's queue, + * the queue should be empty at this point + */ + lf_inprog.s.eena = 0x0; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_INPROG, lf_inprog.u); +} + +static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]); +} + +static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf, + bool enable) +{ + union otx2_cptx_lf_ctl lf_ctl; + + lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_CTL); + + /* Set iqueue's enqueuing */ + lf_ctl.s.ena = enable ? 0x1 : 0x0; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_CTL, lf_ctl.u); +} + +static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf) +{ + otx2_cptlf_set_iqueue_enq(lf, true); +} + +static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf, + bool enable) +{ + union otx2_cptx_lf_inprog lf_inprog; + + lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_INPROG); + + /* Set iqueue's execution */ + lf_inprog.s.eena = enable ? 0x1 : 0x0; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_INPROG, lf_inprog.u); +} + +static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf) +{ + otx2_cptlf_set_iqueue_exec(lf, true); +} + +static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf) +{ + otx2_cptlf_set_iqueue_exec(lf, false); +} + +static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]); + otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]); + } +} + +static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst, + struct otx2_cpt_iq_command *iq_cmd, + u64 comp_baddr) +{ + cptinst->u[0] = 0x0; + cptinst->s.doneint = true; + cptinst->s.res_addr = comp_baddr; + cptinst->u[2] = 0x0; + cptinst->u[3] = 0x0; + cptinst->s.ei0 = iq_cmd->cmd.u; + cptinst->s.ei1 = iq_cmd->dptr; + cptinst->s.ei2 = iq_cmd->rptr; + cptinst->s.ei3 = iq_cmd->cptr.u; +} + +/* + * On OcteonTX2 platform the parameter insts_num is used as a count of + * instructions to be enqueued. The valid values for insts_num are: + * 1 - 1 CPT instruction will be enqueued during LMTST operation + * 2 - 2 CPT instructions will be enqueued during LMTST operation + */ +static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, + u32 insts_num, struct otx2_cptlf_info *lf) +{ + void __iomem *lmtline = lf->lmtline; + long ret; + + /* + * Make sure memory areas pointed in CPT_INST_S + * are flushed before the instruction is sent to CPT + */ + dma_wmb(); + + do { + /* Copy CPT command to LMTLINE */ + memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); + + /* + * LDEOR initiates atomic transfer to I/O device + * The following will cause the LMTST to fail (the LDEOR + * returns zero): + * - No stores have been performed to the LMTLINE since it was + * last invalidated. + * - The bytes which have been stored to LMTLINE since it was + * last invalidated form a pattern that is non-contiguous, does + * not start at byte 0, or does not end on a 8-byte boundary. + * (i.e.comprises a formation of other than 1–16 8-byte + * words.) + * + * These rules are designed such that an operating system + * context switch or hypervisor guest switch need have no + * knowledge of the LMTST operations; the switch code does not + * need to store to LMTCANCEL. Also note as LMTLINE data cannot + * be read, there is no information leakage between processes. + */ + ret = otx2_lmt_flush(lf->ioreg); + + } while (!ret); +} + +static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs) +{ + return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED; +} + +int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri, + int lfs_num); +void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs); +int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs); +void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs); +void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs); +int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs); + +#endif /* __OTX2_CPTLF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h new file mode 100644 index 000000000000..8c899ad531a5 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPTPF_H +#define __OTX2_CPTPF_H + +#include "otx2_cpt_common.h" +#include "otx2_cptpf_ucode.h" +#include "otx2_cptlf.h" + +struct otx2_cptpf_dev; +struct otx2_cptvf_info { + struct otx2_cptpf_dev *cptpf; /* PF pointer this VF belongs to */ + struct work_struct vfpf_mbox_work; + struct pci_dev *vf_dev; + int vf_id; + int intr_idx; +}; + +struct cptpf_flr_work { + struct work_struct work; + struct otx2_cptpf_dev *pf; +}; + +struct otx2_cptpf_dev { + void __iomem *reg_base; /* CPT PF registers start address */ + void __iomem *afpf_mbox_base; /* PF-AF mbox start address */ + void __iomem *vfpf_mbox_base; /* VF-PF mbox start address */ + struct pci_dev *pdev; /* PCI device handle */ + struct otx2_cptvf_info vf[OTX2_CPT_MAX_VFS_NUM]; + struct otx2_cpt_eng_grps eng_grps;/* Engine groups information */ + struct otx2_cptlfs_info lfs; /* CPT LFs attached to this PF */ + /* HW capabilities for each engine type */ + union otx2_cpt_eng_caps eng_caps[OTX2_CPT_MAX_ENG_TYPES]; + bool is_eng_caps_discovered; + + /* AF <=> PF mbox */ + struct otx2_mbox afpf_mbox; + struct work_struct afpf_mbox_work; + struct workqueue_struct *afpf_mbox_wq; + + /* VF <=> PF mbox */ + struct otx2_mbox vfpf_mbox; + struct workqueue_struct *vfpf_mbox_wq; + + struct workqueue_struct *flr_wq; + struct cptpf_flr_work *flr_work; + + u8 pf_id; /* RVU PF number */ + u8 max_vfs; /* Maximum number of VFs supported by CPT */ + u8 enabled_vfs; /* Number of enabled VFs */ + u8 kvf_limits; /* Kernel crypto limits */ +}; + +irqreturn_t otx2_cptpf_afpf_mbox_intr(int irq, void *arg); +void otx2_cptpf_afpf_mbox_handler(struct work_struct *work); +irqreturn_t otx2_cptpf_vfpf_mbox_intr(int irq, void *arg); +void otx2_cptpf_vfpf_mbox_handler(struct work_struct *work); + +#endif /* __OTX2_CPTPF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c new file mode 100644 index 000000000000..5277e04badd9 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -0,0 +1,713 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include +#include "otx2_cpt_hw_types.h" +#include "otx2_cpt_common.h" +#include "otx2_cptpf_ucode.h" +#include "otx2_cptpf.h" +#include "rvu_reg.h" + +#define OTX2_CPT_DRV_NAME "octeontx2-cpt" +#define OTX2_CPT_DRV_STRING "Marvell OcteonTX2 CPT Physical Function Driver" + +static void cptpf_enable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + int ena_bits; + + /* Clear any pending interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(0), ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(1), ~0x0ULL); + + /* Enable VF interrupts for VFs from 0 to 63 */ + ena_bits = ((num_vfs - 1) % 64); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), + GENMASK_ULL(ena_bits, 0)); + + if (num_vfs > 64) { + /* Enable VF interrupts for VFs from 64 to 127 */ + ena_bits = num_vfs - 64 - 1; + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1), + GENMASK_ULL(ena_bits, 0)); + } +} + +static void cptpf_disable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + int vector; + + /* Disable VF-PF interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), ~0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), ~0ULL); + /* Clear any pending interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(0), ~0ULL); + + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0); + free_irq(vector, cptpf); + + if (num_vfs > 64) { + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(1), ~0ULL); + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFPF_MBOX1); + free_irq(vector, cptpf); + } +} + +static void cptpf_enable_vf_flr_intrs(struct otx2_cptpf_dev *cptpf) +{ + /* Clear interrupt if any */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(0), + ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1), + ~0x0ULL); + + /* Enable VF FLR interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1SX(0), ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1SX(1), ~0x0ULL); +} + +static void cptpf_disable_vf_flr_intrs(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + int vector; + + /* Disable VF FLR interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1CX(0), ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1CX(1), ~0x0ULL); + + /* Clear interrupt if any */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(0), + ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1), + ~0x0ULL); + + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR0); + free_irq(vector, cptpf); + + if (num_vfs > 64) { + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR1); + free_irq(vector, cptpf); + } +} + +static void cptpf_flr_wq_handler(struct work_struct *work) +{ + struct cptpf_flr_work *flr_work; + struct otx2_cptpf_dev *pf; + struct mbox_msghdr *req; + struct otx2_mbox *mbox; + int vf, reg = 0; + + flr_work = container_of(work, struct cptpf_flr_work, work); + pf = flr_work->pf; + mbox = &pf->afpf_mbox; + + vf = flr_work - pf->flr_work; + + req = otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (!req) + return; + + req->sig = OTX2_MBOX_REQ_SIG; + req->id = MBOX_MSG_VF_FLR; + req->pcifunc &= RVU_PFVF_FUNC_MASK; + req->pcifunc |= (vf + 1) & RVU_PFVF_FUNC_MASK; + + otx2_cpt_send_mbox_msg(mbox, pf->pdev); + + if (vf >= 64) { + reg = 1; + vf = vf - 64; + } + /* Clear transaction pending register */ + otx2_cpt_write64(pf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFTRPENDX(reg), BIT_ULL(vf)); + otx2_cpt_write64(pf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf)); +} + +static irqreturn_t cptpf_vf_flr_intr(int __always_unused irq, void *arg) +{ + int reg, dev, vf, start_vf, num_reg = 1; + struct otx2_cptpf_dev *cptpf = arg; + u64 intr; + + if (cptpf->max_vfs > 64) + num_reg = 2; + + for (reg = 0; reg < num_reg; reg++) { + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INTX(reg)); + if (!intr) + continue; + start_vf = 64 * reg; + for (vf = 0; vf < 64; vf++) { + if (!(intr & BIT_ULL(vf))) + continue; + dev = vf + start_vf; + queue_work(cptpf->flr_wq, &cptpf->flr_work[dev].work); + /* Clear interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf)); + /* Disable the interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1CX(reg), + BIT_ULL(vf)); + } + } + return IRQ_HANDLED; +} + +static void cptpf_unregister_vfpf_intr(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + cptpf_disable_vfpf_mbox_intr(cptpf, num_vfs); + cptpf_disable_vf_flr_intrs(cptpf, num_vfs); +} + +static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs) +{ + struct pci_dev *pdev = cptpf->pdev; + struct device *dev = &pdev->dev; + int ret, vector; + + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX0); + /* Register VF-PF mailbox interrupt handler */ + ret = request_irq(vector, otx2_cptpf_vfpf_mbox_intr, 0, "CPTVFPF Mbox0", + cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for PFVF mbox0 irq\n"); + return ret; + } + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR0); + /* Register VF FLR interrupt handler */ + ret = request_irq(vector, cptpf_vf_flr_intr, 0, "CPTPF FLR0", cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for VFFLR0 irq\n"); + goto free_mbox0_irq; + } + if (num_vfs > 64) { + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1); + ret = request_irq(vector, otx2_cptpf_vfpf_mbox_intr, 0, + "CPTVFPF Mbox1", cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for PFVF mbox1 irq\n"); + goto free_flr0_irq; + } + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR1); + /* Register VF FLR interrupt handler */ + ret = request_irq(vector, cptpf_vf_flr_intr, 0, "CPTPF FLR1", + cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for VFFLR1 irq\n"); + goto free_mbox1_irq; + } + } + cptpf_enable_vfpf_mbox_intr(cptpf, num_vfs); + cptpf_enable_vf_flr_intrs(cptpf); + + return 0; + +free_mbox1_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1); + free_irq(vector, cptpf); +free_flr0_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR0); + free_irq(vector, cptpf); +free_mbox0_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX0); + free_irq(vector, cptpf); + return ret; +} + +static void cptpf_flr_wq_destroy(struct otx2_cptpf_dev *pf) +{ + if (!pf->flr_wq) + return; + destroy_workqueue(pf->flr_wq); + pf->flr_wq = NULL; + kfree(pf->flr_work); +} + +static int cptpf_flr_wq_init(struct otx2_cptpf_dev *cptpf, int num_vfs) +{ + int vf; + + cptpf->flr_wq = alloc_ordered_workqueue("cptpf_flr_wq", 0); + if (!cptpf->flr_wq) + return -ENOMEM; + + cptpf->flr_work = kcalloc(num_vfs, sizeof(struct cptpf_flr_work), + GFP_KERNEL); + if (!cptpf->flr_work) + goto destroy_wq; + + for (vf = 0; vf < num_vfs; vf++) { + cptpf->flr_work[vf].pf = cptpf; + INIT_WORK(&cptpf->flr_work[vf].work, cptpf_flr_wq_handler); + } + return 0; + +destroy_wq: + destroy_workqueue(cptpf->flr_wq); + return -ENOMEM; +} + +static int cptpf_vfpf_mbox_init(struct otx2_cptpf_dev *cptpf, int num_vfs) +{ + struct device *dev = &cptpf->pdev->dev; + u64 vfpf_mbox_base; + int err, i; + + cptpf->vfpf_mbox_wq = alloc_workqueue("cpt_vfpf_mailbox", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, 1); + if (!cptpf->vfpf_mbox_wq) + return -ENOMEM; + + /* Map VF-PF mailbox memory */ + vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_BAR4_ADDR); + if (!vfpf_mbox_base) { + dev_err(dev, "VF-PF mailbox address not configured\n"); + err = -ENOMEM; + goto free_wqe; + } + cptpf->vfpf_mbox_base = devm_ioremap_wc(dev, vfpf_mbox_base, + MBOX_SIZE * cptpf->max_vfs); + if (!cptpf->vfpf_mbox_base) { + dev_err(dev, "Mapping of VF-PF mailbox address failed\n"); + err = -ENOMEM; + goto free_wqe; + } + err = otx2_mbox_init(&cptpf->vfpf_mbox, cptpf->vfpf_mbox_base, + cptpf->pdev, cptpf->reg_base, MBOX_DIR_PFVF, + num_vfs); + if (err) + goto free_wqe; + + for (i = 0; i < num_vfs; i++) { + cptpf->vf[i].vf_id = i; + cptpf->vf[i].cptpf = cptpf; + cptpf->vf[i].intr_idx = i % 64; + INIT_WORK(&cptpf->vf[i].vfpf_mbox_work, + otx2_cptpf_vfpf_mbox_handler); + } + return 0; + +free_wqe: + destroy_workqueue(cptpf->vfpf_mbox_wq); + return err; +} + +static void cptpf_vfpf_mbox_destroy(struct otx2_cptpf_dev *cptpf) +{ + destroy_workqueue(cptpf->vfpf_mbox_wq); + otx2_mbox_destroy(&cptpf->vfpf_mbox); +} + +static void cptpf_disable_afpf_mbox_intr(struct otx2_cptpf_dev *cptpf) +{ + /* Disable AF-PF interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT_ENA_W1C, + 0x1ULL); + /* Clear interrupt if any */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, 0x1ULL); +} + +static int cptpf_register_afpf_mbox_intr(struct otx2_cptpf_dev *cptpf) +{ + struct pci_dev *pdev = cptpf->pdev; + struct device *dev = &pdev->dev; + int ret, irq; + + irq = pci_irq_vector(pdev, RVU_PF_INT_VEC_AFPF_MBOX); + /* Register AF-PF mailbox interrupt handler */ + ret = devm_request_irq(dev, irq, otx2_cptpf_afpf_mbox_intr, 0, + "CPTAFPF Mbox", cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for PFAF mbox irq\n"); + return ret; + } + /* Clear interrupt if any, to avoid spurious interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, 0x1ULL); + /* Enable AF-PF interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT_ENA_W1S, + 0x1ULL); + + ret = otx2_cpt_send_ready_msg(&cptpf->afpf_mbox, cptpf->pdev); + if (ret) { + dev_warn(dev, + "AF not responding to mailbox, deferring probe\n"); + cptpf_disable_afpf_mbox_intr(cptpf); + return -EPROBE_DEFER; + } + return 0; +} + +static int cptpf_afpf_mbox_init(struct otx2_cptpf_dev *cptpf) +{ + int err; + + cptpf->afpf_mbox_wq = alloc_workqueue("cpt_afpf_mailbox", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, 1); + if (!cptpf->afpf_mbox_wq) + return -ENOMEM; + + err = otx2_mbox_init(&cptpf->afpf_mbox, cptpf->afpf_mbox_base, + cptpf->pdev, cptpf->reg_base, MBOX_DIR_PFAF, 1); + if (err) + goto error; + + INIT_WORK(&cptpf->afpf_mbox_work, otx2_cptpf_afpf_mbox_handler); + return 0; + +error: + destroy_workqueue(cptpf->afpf_mbox_wq); + return err; +} + +static void cptpf_afpf_mbox_destroy(struct otx2_cptpf_dev *cptpf) +{ + destroy_workqueue(cptpf->afpf_mbox_wq); + otx2_mbox_destroy(&cptpf->afpf_mbox); +} + +static ssize_t kvf_limits_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", cptpf->kvf_limits); +} + +static ssize_t kvf_limits_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); + int lfs_num; + + if (kstrtoint(buf, 0, &lfs_num)) { + dev_err(dev, "lfs count %d must be in range [1 - %d]\n", + lfs_num, num_online_cpus()); + return -EINVAL; + } + if (lfs_num < 1 || lfs_num > num_online_cpus()) { + dev_err(dev, "lfs count %d must be in range [1 - %d]\n", + lfs_num, num_online_cpus()); + return -EINVAL; + } + cptpf->kvf_limits = lfs_num; + + return count; +} + +static DEVICE_ATTR_RW(kvf_limits); +static struct attribute *cptpf_attrs[] = { + &dev_attr_kvf_limits.attr, + NULL +}; + +static const struct attribute_group cptpf_sysfs_group = { + .attrs = cptpf_attrs, +}; + +static int cpt_is_pf_usable(struct otx2_cptpf_dev *cptpf) +{ + u64 rev; + + rev = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM)); + rev = (rev >> 12) & 0xFF; + /* + * Check if AF has setup revision for RVUM block, otherwise + * driver probe should be deferred until AF driver comes up + */ + if (!rev) { + dev_warn(&cptpf->pdev->dev, + "AF is not initialized, deferring probe\n"); + return -EPROBE_DEFER; + } + return 0; +} + +static int cptpf_device_reset(struct otx2_cptpf_dev *cptpf) +{ + int timeout = 10, ret; + u64 reg = 0; + + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_BLK_RST, 0x1); + if (ret) + return ret; + + do { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_BLK_RST, ®); + if (ret) + return ret; + + if (!((reg >> 63) & 0x1)) + break; + + usleep_range(10000, 20000); + if (timeout-- < 0) + return -EBUSY; + } while (1); + + return ret; +} + +static int cptpf_device_init(struct otx2_cptpf_dev *cptpf) +{ + union otx2_cptx_af_constants1 af_cnsts1 = {0}; + int ret = 0; + + /* Reset the CPT PF device */ + ret = cptpf_device_reset(cptpf); + if (ret) + return ret; + + /* Get number of SE, IE and AE engines */ + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_CONSTANTS1, &af_cnsts1.u); + if (ret) + return ret; + + cptpf->eng_grps.avail.max_se_cnt = af_cnsts1.s.se; + cptpf->eng_grps.avail.max_ie_cnt = af_cnsts1.s.ie; + cptpf->eng_grps.avail.max_ae_cnt = af_cnsts1.s.ae; + + /* Disable all cores */ + ret = otx2_cpt_disable_all_cores(cptpf); + + return ret; +} + +static int cptpf_sriov_disable(struct pci_dev *pdev) +{ + struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev); + int num_vfs = pci_num_vf(pdev); + + if (!num_vfs) + return 0; + + pci_disable_sriov(pdev); + cptpf_unregister_vfpf_intr(cptpf, num_vfs); + cptpf_flr_wq_destroy(cptpf); + cptpf_vfpf_mbox_destroy(cptpf); + module_put(THIS_MODULE); + cptpf->enabled_vfs = 0; + + return 0; +} + +static int cptpf_sriov_enable(struct pci_dev *pdev, int num_vfs) +{ + struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev); + int ret; + + /* Initialize VF<=>PF mailbox */ + ret = cptpf_vfpf_mbox_init(cptpf, num_vfs); + if (ret) + return ret; + + ret = cptpf_flr_wq_init(cptpf, num_vfs); + if (ret) + goto destroy_mbox; + /* Register VF<=>PF mailbox interrupt */ + ret = cptpf_register_vfpf_intr(cptpf, num_vfs); + if (ret) + goto destroy_flr; + + /* Get CPT HW capabilities using LOAD_FVC operation. */ + ret = otx2_cpt_discover_eng_capabilities(cptpf); + if (ret) + goto disable_intr; + + ret = otx2_cpt_create_eng_grps(cptpf->pdev, &cptpf->eng_grps); + if (ret) + goto disable_intr; + + cptpf->enabled_vfs = num_vfs; + ret = pci_enable_sriov(pdev, num_vfs); + if (ret) + goto disable_intr; + + dev_notice(&cptpf->pdev->dev, "VFs enabled: %d\n", num_vfs); + + try_module_get(THIS_MODULE); + return num_vfs; + +disable_intr: + cptpf_unregister_vfpf_intr(cptpf, num_vfs); + cptpf->enabled_vfs = 0; +destroy_flr: + cptpf_flr_wq_destroy(cptpf); +destroy_mbox: + cptpf_vfpf_mbox_destroy(cptpf); + return ret; +} + +static int otx2_cptpf_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + if (num_vfs > 0) { + return cptpf_sriov_enable(pdev, num_vfs); + } else { + return cptpf_sriov_disable(pdev); + } +} + +static int otx2_cptpf_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + resource_size_t offset, size; + struct otx2_cptpf_dev *cptpf; + int err; + + cptpf = devm_kzalloc(dev, sizeof(*cptpf), GFP_KERNEL); + if (!cptpf) + return -ENOMEM; + + err = pcim_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + goto clear_drvdata; + } + + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable DMA configuration\n"); + goto clear_drvdata; + } + /* Map PF's configuration registers */ + err = pcim_iomap_regions_request_all(pdev, 1 << PCI_PF_REG_BAR_NUM, + OTX2_CPT_DRV_NAME); + if (err) { + dev_err(dev, "Couldn't get PCI resources 0x%x\n", err); + goto clear_drvdata; + } + pci_set_master(pdev); + pci_set_drvdata(pdev, cptpf); + cptpf->pdev = pdev; + + cptpf->reg_base = pcim_iomap_table(pdev)[PCI_PF_REG_BAR_NUM]; + + /* Check if AF driver is up, otherwise defer probe */ + err = cpt_is_pf_usable(cptpf); + if (err) + goto clear_drvdata; + + offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); + size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); + /* Map AF-PF mailbox memory */ + cptpf->afpf_mbox_base = devm_ioremap_wc(dev, offset, size); + if (!cptpf->afpf_mbox_base) { + dev_err(&pdev->dev, "Unable to map BAR4\n"); + err = -ENODEV; + goto clear_drvdata; + } + err = pci_alloc_irq_vectors(pdev, RVU_PF_INT_VEC_CNT, + RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX); + if (err < 0) { + dev_err(dev, "Request for %d msix vectors failed\n", + RVU_PF_INT_VEC_CNT); + goto clear_drvdata; + } + /* Initialize AF-PF mailbox */ + err = cptpf_afpf_mbox_init(cptpf); + if (err) + goto clear_drvdata; + /* Register mailbox interrupt */ + err = cptpf_register_afpf_mbox_intr(cptpf); + if (err) + goto destroy_afpf_mbox; + + cptpf->max_vfs = pci_sriov_get_totalvfs(pdev); + + /* Initialize CPT PF device */ + err = cptpf_device_init(cptpf); + if (err) + goto unregister_intr; + + /* Initialize engine groups */ + err = otx2_cpt_init_eng_grps(pdev, &cptpf->eng_grps); + if (err) + goto unregister_intr; + + err = sysfs_create_group(&dev->kobj, &cptpf_sysfs_group); + if (err) + goto cleanup_eng_grps; + return 0; + +cleanup_eng_grps: + otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps); +unregister_intr: + cptpf_disable_afpf_mbox_intr(cptpf); +destroy_afpf_mbox: + cptpf_afpf_mbox_destroy(cptpf); +clear_drvdata: + pci_set_drvdata(pdev, NULL); + return err; +} + +static void otx2_cptpf_remove(struct pci_dev *pdev) +{ + struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev); + + if (!cptpf) + return; + + cptpf_sriov_disable(pdev); + /* Delete sysfs entry created for kernel VF limits */ + sysfs_remove_group(&pdev->dev.kobj, &cptpf_sysfs_group); + /* Cleanup engine groups */ + otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps); + /* Disable AF-PF mailbox interrupt */ + cptpf_disable_afpf_mbox_intr(cptpf); + /* Destroy AF-PF mbox */ + cptpf_afpf_mbox_destroy(cptpf); + pci_set_drvdata(pdev, NULL); +} + +/* Supported devices */ +static const struct pci_device_id otx2_cpt_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OTX2_CPT_PCI_PF_DEVICE_ID) }, + { 0, } /* end of table */ +}; + +static struct pci_driver otx2_cpt_pci_driver = { + .name = OTX2_CPT_DRV_NAME, + .id_table = otx2_cpt_id_table, + .probe = otx2_cptpf_probe, + .remove = otx2_cptpf_remove, + .sriov_configure = otx2_cptpf_sriov_configure +}; + +module_pci_driver(otx2_cpt_pci_driver); + +MODULE_AUTHOR("Marvell"); +MODULE_DESCRIPTION(OTX2_CPT_DRV_STRING); +MODULE_LICENSE("GPL v2"); +MODULE_DEVICE_TABLE(pci, otx2_cpt_id_table); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c new file mode 100644 index 000000000000..186f1c1190c1 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptpf.h" +#include "rvu_reg.h" + +/* + * CPT PF driver version, It will be incremented by 1 for every feature + * addition in CPT mailbox messages. + */ +#define OTX2_CPT_PF_DRV_VERSION 0x1 + +static int forward_to_af(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req, int size) +{ + struct mbox_msghdr *msg; + int ret; + + msg = otx2_mbox_alloc_msg(&cptpf->afpf_mbox, 0, size); + if (msg == NULL) + return -ENOMEM; + + memcpy((uint8_t *)msg + sizeof(struct mbox_msghdr), + (uint8_t *)req + sizeof(struct mbox_msghdr), size); + msg->id = req->id; + msg->pcifunc = req->pcifunc; + msg->sig = req->sig; + msg->ver = req->ver; + + otx2_mbox_msg_send(&cptpf->afpf_mbox, 0); + ret = otx2_mbox_wait_for_rsp(&cptpf->afpf_mbox, 0); + if (ret == -EIO) { + dev_err(&cptpf->pdev->dev, "RVU MBOX timeout.\n"); + return ret; + } else if (ret) { + dev_err(&cptpf->pdev->dev, "RVU MBOX error: %d.\n", ret); + return -EFAULT; + } + return 0; +} + +static int handle_msg_get_caps(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req) +{ + struct otx2_cpt_caps_rsp *rsp; + + rsp = (struct otx2_cpt_caps_rsp *) + otx2_mbox_alloc_msg(&cptpf->vfpf_mbox, vf->vf_id, + sizeof(*rsp)); + if (!rsp) + return -ENOMEM; + + rsp->hdr.id = MBOX_MSG_GET_CAPS; + rsp->hdr.sig = OTX2_MBOX_RSP_SIG; + rsp->hdr.pcifunc = req->pcifunc; + rsp->cpt_pf_drv_version = OTX2_CPT_PF_DRV_VERSION; + rsp->cpt_revision = cptpf->pdev->revision; + memcpy(&rsp->eng_caps, &cptpf->eng_caps, sizeof(rsp->eng_caps)); + + return 0; +} + +static int handle_msg_get_eng_grp_num(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req) +{ + struct otx2_cpt_egrp_num_msg *grp_req; + struct otx2_cpt_egrp_num_rsp *rsp; + + grp_req = (struct otx2_cpt_egrp_num_msg *)req; + rsp = (struct otx2_cpt_egrp_num_rsp *) + otx2_mbox_alloc_msg(&cptpf->vfpf_mbox, vf->vf_id, sizeof(*rsp)); + if (!rsp) + return -ENOMEM; + + rsp->hdr.id = MBOX_MSG_GET_ENG_GRP_NUM; + rsp->hdr.sig = OTX2_MBOX_RSP_SIG; + rsp->hdr.pcifunc = req->pcifunc; + rsp->eng_type = grp_req->eng_type; + rsp->eng_grp_num = otx2_cpt_get_eng_grp(&cptpf->eng_grps, + grp_req->eng_type); + + return 0; +} + +static int handle_msg_kvf_limits(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req) +{ + struct otx2_cpt_kvf_limits_rsp *rsp; + + rsp = (struct otx2_cpt_kvf_limits_rsp *) + otx2_mbox_alloc_msg(&cptpf->vfpf_mbox, vf->vf_id, sizeof(*rsp)); + if (!rsp) + return -ENOMEM; + + rsp->hdr.id = MBOX_MSG_GET_KVF_LIMITS; + rsp->hdr.sig = OTX2_MBOX_RSP_SIG; + rsp->hdr.pcifunc = req->pcifunc; + rsp->kvf_limits = cptpf->kvf_limits; + + return 0; +} + +static int cptpf_handle_vf_req(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req, int size) +{ + int err = 0; + + /* Check if msg is valid, if not reply with an invalid msg */ + if (req->sig != OTX2_MBOX_REQ_SIG) + goto inval_msg; + + switch (req->id) { + case MBOX_MSG_GET_ENG_GRP_NUM: + err = handle_msg_get_eng_grp_num(cptpf, vf, req); + break; + case MBOX_MSG_GET_CAPS: + err = handle_msg_get_caps(cptpf, vf, req); + break; + case MBOX_MSG_GET_KVF_LIMITS: + err = handle_msg_kvf_limits(cptpf, vf, req); + break; + default: + err = forward_to_af(cptpf, vf, req, size); + break; + } + return err; + +inval_msg: + otx2_reply_invalid_msg(&cptpf->vfpf_mbox, vf->vf_id, 0, req->id); + otx2_mbox_msg_send(&cptpf->vfpf_mbox, vf->vf_id); + return err; +} + +irqreturn_t otx2_cptpf_vfpf_mbox_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptpf_dev *cptpf = arg; + struct otx2_cptvf_info *vf; + int i, vf_idx; + u64 intr; + + /* + * Check which VF has raised an interrupt and schedule + * corresponding work queue to process the messages + */ + for (i = 0; i < 2; i++) { + /* Read the interrupt bits */ + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(i)); + + for (vf_idx = i * 64; vf_idx < cptpf->enabled_vfs; vf_idx++) { + vf = &cptpf->vf[vf_idx]; + if (intr & (1ULL << vf->intr_idx)) { + queue_work(cptpf->vfpf_mbox_wq, + &vf->vfpf_mbox_work); + /* Clear the interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, + 0, RVU_PF_VFPF_MBOX_INTX(i), + BIT_ULL(vf->intr_idx)); + } + } + } + return IRQ_HANDLED; +} + +void otx2_cptpf_vfpf_mbox_handler(struct work_struct *work) +{ + struct otx2_cptpf_dev *cptpf; + struct otx2_cptvf_info *vf; + struct otx2_mbox_dev *mdev; + struct mbox_hdr *req_hdr; + struct mbox_msghdr *msg; + struct otx2_mbox *mbox; + int offset, i, err; + + vf = container_of(work, struct otx2_cptvf_info, vfpf_mbox_work); + cptpf = vf->cptpf; + mbox = &cptpf->vfpf_mbox; + /* sync with mbox memory region */ + smp_rmb(); + mdev = &mbox->dev[vf->vf_id]; + /* Process received mbox messages */ + req_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); + offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN); + + for (i = 0; i < req_hdr->num_msgs; i++) { + msg = (struct mbox_msghdr *)(mdev->mbase + offset); + + /* Set which VF sent this message based on mbox IRQ */ + msg->pcifunc = ((u16)cptpf->pf_id << RVU_PFVF_PF_SHIFT) | + ((vf->vf_id + 1) & RVU_PFVF_FUNC_MASK); + + err = cptpf_handle_vf_req(cptpf, vf, msg, + msg->next_msgoff - offset); + /* + * Behave as the AF, drop the msg if there is + * no memory, timeout handling also goes here + */ + if (err == -ENOMEM || err == -EIO) + break; + offset = msg->next_msgoff; + } + /* Send mbox responses to VF */ + if (mdev->num_msgs) + otx2_mbox_msg_send(mbox, vf->vf_id); +} + +irqreturn_t otx2_cptpf_afpf_mbox_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptpf_dev *cptpf = arg; + u64 intr; + + /* Read the interrupt bits */ + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT); + + if (intr & 0x1ULL) { + /* Schedule work queue function to process the MBOX request */ + queue_work(cptpf->afpf_mbox_wq, &cptpf->afpf_mbox_work); + /* Clear and ack the interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, + 0x1ULL); + } + return IRQ_HANDLED; +} + +static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf, + struct mbox_msghdr *msg) +{ + struct device *dev = &cptpf->pdev->dev; + struct cpt_rd_wr_reg_msg *rsp_rd_wr; + + if (msg->id >= MBOX_MSG_MAX) { + dev_err(dev, "MBOX msg with unknown ID %d\n", msg->id); + return; + } + if (msg->sig != OTX2_MBOX_RSP_SIG) { + dev_err(dev, "MBOX msg with wrong signature %x, ID %d\n", + msg->sig, msg->id); + return; + } + + switch (msg->id) { + case MBOX_MSG_READY: + cptpf->pf_id = (msg->pcifunc >> RVU_PFVF_PF_SHIFT) & + RVU_PFVF_PF_MASK; + break; + case MBOX_MSG_CPT_RD_WR_REGISTER: + rsp_rd_wr = (struct cpt_rd_wr_reg_msg *)msg; + if (msg->rc) { + dev_err(dev, "Reg %llx rd/wr(%d) failed %d\n", + rsp_rd_wr->reg_offset, rsp_rd_wr->is_write, + msg->rc); + return; + } + if (!rsp_rd_wr->is_write) + *rsp_rd_wr->ret_val = rsp_rd_wr->val; + break; + case MBOX_MSG_ATTACH_RESOURCES: + if (!msg->rc) + cptpf->lfs.are_lfs_attached = 1; + break; + case MBOX_MSG_DETACH_RESOURCES: + if (!msg->rc) + cptpf->lfs.are_lfs_attached = 0; + break; + + default: + dev_err(dev, + "Unsupported msg %d received.\n", msg->id); + break; + } +} + +static void forward_to_vf(struct otx2_cptpf_dev *cptpf, struct mbox_msghdr *msg, + int vf_id, int size) +{ + struct otx2_mbox *vfpf_mbox; + struct mbox_msghdr *fwd; + + if (msg->id >= MBOX_MSG_MAX) { + dev_err(&cptpf->pdev->dev, + "MBOX msg with unknown ID %d\n", msg->id); + return; + } + if (msg->sig != OTX2_MBOX_RSP_SIG) { + dev_err(&cptpf->pdev->dev, + "MBOX msg with wrong signature %x, ID %d\n", + msg->sig, msg->id); + return; + } + vfpf_mbox = &cptpf->vfpf_mbox; + vf_id--; + if (vf_id >= cptpf->enabled_vfs) { + dev_err(&cptpf->pdev->dev, + "MBOX msg to unknown VF: %d >= %d\n", + vf_id, cptpf->enabled_vfs); + return; + } + if (msg->id == MBOX_MSG_VF_FLR) + return; + + fwd = otx2_mbox_alloc_msg(vfpf_mbox, vf_id, size); + if (!fwd) { + dev_err(&cptpf->pdev->dev, + "Forwarding to VF%d failed.\n", vf_id); + return; + } + memcpy((uint8_t *)fwd + sizeof(struct mbox_msghdr), + (uint8_t *)msg + sizeof(struct mbox_msghdr), size); + fwd->id = msg->id; + fwd->pcifunc = msg->pcifunc; + fwd->sig = msg->sig; + fwd->ver = msg->ver; + fwd->rc = msg->rc; +} + +/* Handle mailbox messages received from AF */ +void otx2_cptpf_afpf_mbox_handler(struct work_struct *work) +{ + struct otx2_cptpf_dev *cptpf; + struct otx2_mbox *afpf_mbox; + struct otx2_mbox_dev *mdev; + struct mbox_hdr *rsp_hdr; + struct mbox_msghdr *msg; + int offset, vf_id, i; + + cptpf = container_of(work, struct otx2_cptpf_dev, afpf_mbox_work); + afpf_mbox = &cptpf->afpf_mbox; + mdev = &afpf_mbox->dev[0]; + /* Sync mbox data into memory */ + smp_wmb(); + + rsp_hdr = (struct mbox_hdr *)(mdev->mbase + afpf_mbox->rx_start); + offset = ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); + + for (i = 0; i < rsp_hdr->num_msgs; i++) { + msg = (struct mbox_msghdr *)(mdev->mbase + afpf_mbox->rx_start + + offset); + vf_id = (msg->pcifunc >> RVU_PFVF_FUNC_SHIFT) & + RVU_PFVF_FUNC_MASK; + if (vf_id > 0) + forward_to_vf(cptpf, msg, vf_id, + msg->next_msgoff - offset); + else + process_afpf_mbox_msg(cptpf, msg); + + offset = msg->next_msgoff; + mdev->msgs_acked++; + } + otx2_mbox_reset(afpf_mbox, 0); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c new file mode 100644 index 000000000000..1dc3ba298139 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -0,0 +1,1415 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include +#include +#include "otx2_cptpf_ucode.h" +#include "otx2_cpt_common.h" +#include "otx2_cptpf.h" +#include "otx2_cptlf.h" +#include "otx2_cpt_reqmgr.h" +#include "rvu_reg.h" + +#define CSR_DELAY 30 + +#define LOADFVC_RLEN 8 +#define LOADFVC_MAJOR_OP 0x01 +#define LOADFVC_MINOR_OP 0x08 + +struct fw_info_t { + struct list_head ucodes; +}; + +static struct otx2_cpt_bitmap get_cores_bmap(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp) +{ + struct otx2_cpt_bitmap bmap = { {0} }; + bool found = false; + int i; + + if (eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) { + dev_err(dev, "unsupported number of engines %d on octeontx2\n", + eng_grp->g->engs_num); + return bmap; + } + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (eng_grp->engs[i].type) { + bitmap_or(bmap.bits, bmap.bits, + eng_grp->engs[i].bmap, + eng_grp->g->engs_num); + bmap.size = eng_grp->g->engs_num; + found = true; + } + } + + if (!found) + dev_err(dev, "No engines reserved for engine group %d\n", + eng_grp->idx); + return bmap; +} + +static int is_eng_type(int val, int eng_type) +{ + return val & (1 << eng_type); +} + +static int is_2nd_ucode_used(struct otx2_cpt_eng_grp_info *eng_grp) +{ + if (eng_grp->ucode[1].type) + return true; + else + return false; +} + +static void set_ucode_filename(struct otx2_cpt_ucode *ucode, + const char *filename) +{ + strlcpy(ucode->filename, filename, OTX2_CPT_NAME_LENGTH); +} + +static char *get_eng_type_str(int eng_type) +{ + char *str = "unknown"; + + switch (eng_type) { + case OTX2_CPT_SE_TYPES: + str = "SE"; + break; + + case OTX2_CPT_IE_TYPES: + str = "IE"; + break; + + case OTX2_CPT_AE_TYPES: + str = "AE"; + break; + } + return str; +} + +static char *get_ucode_type_str(int ucode_type) +{ + char *str = "unknown"; + + switch (ucode_type) { + case (1 << OTX2_CPT_SE_TYPES): + str = "SE"; + break; + + case (1 << OTX2_CPT_IE_TYPES): + str = "IE"; + break; + + case (1 << OTX2_CPT_AE_TYPES): + str = "AE"; + break; + + case (1 << OTX2_CPT_SE_TYPES | 1 << OTX2_CPT_IE_TYPES): + str = "SE+IPSEC"; + break; + } + return str; +} + +static int get_ucode_type(struct device *dev, + struct otx2_cpt_ucode_hdr *ucode_hdr, + int *ucode_type) +{ + struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); + char ver_str_prefix[OTX2_CPT_UCODE_VER_STR_SZ]; + char tmp_ver_str[OTX2_CPT_UCODE_VER_STR_SZ]; + struct pci_dev *pdev = cptpf->pdev; + int i, val = 0; + u8 nn; + + strlcpy(tmp_ver_str, ucode_hdr->ver_str, OTX2_CPT_UCODE_VER_STR_SZ); + for (i = 0; i < strlen(tmp_ver_str); i++) + tmp_ver_str[i] = tolower(tmp_ver_str[i]); + + sprintf(ver_str_prefix, "ocpt-%02d", pdev->revision); + if (!strnstr(tmp_ver_str, ver_str_prefix, OTX2_CPT_UCODE_VER_STR_SZ)) + return -EINVAL; + + nn = ucode_hdr->ver_num.nn; + if (strnstr(tmp_ver_str, "se-", OTX2_CPT_UCODE_VER_STR_SZ) && + (nn == OTX2_CPT_SE_UC_TYPE1 || nn == OTX2_CPT_SE_UC_TYPE2 || + nn == OTX2_CPT_SE_UC_TYPE3)) + val |= 1 << OTX2_CPT_SE_TYPES; + if (strnstr(tmp_ver_str, "ie-", OTX2_CPT_UCODE_VER_STR_SZ) && + (nn == OTX2_CPT_IE_UC_TYPE1 || nn == OTX2_CPT_IE_UC_TYPE2 || + nn == OTX2_CPT_IE_UC_TYPE3)) + val |= 1 << OTX2_CPT_IE_TYPES; + if (strnstr(tmp_ver_str, "ae", OTX2_CPT_UCODE_VER_STR_SZ) && + nn == OTX2_CPT_AE_UC_TYPE) + val |= 1 << OTX2_CPT_AE_TYPES; + + *ucode_type = val; + + if (!val) + return -EINVAL; + + return 0; +} + +static int __write_ucode_base(struct otx2_cptpf_dev *cptpf, int eng, + dma_addr_t dma_addr) +{ + return otx2_cpt_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_UCODE_BASE(eng), + (u64)dma_addr); +} + +static int cpt_set_ucode_base(struct otx2_cpt_eng_grp_info *eng_grp, void *obj) +{ + struct otx2_cptpf_dev *cptpf = obj; + struct otx2_cpt_engs_rsvd *engs; + dma_addr_t dma_addr; + int i, bit, ret; + + /* Set PF number for microcode fetches */ + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_PF_FUNC, + cptpf->pf_id << RVU_PFVF_PF_SHIFT); + if (ret) + return ret; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + + dma_addr = engs->ucode->dma; + + /* + * Set UCODE_BASE only for the cores which are not used, + * other cores should have already valid UCODE_BASE set + */ + for_each_set_bit(bit, engs->bmap, eng_grp->g->engs_num) + if (!eng_grp->g->eng_ref_cnt[bit]) { + ret = __write_ucode_base(cptpf, bit, dma_addr); + if (ret) + return ret; + } + } + return 0; +} + +static int cpt_detach_and_disable_cores(struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + struct otx2_cptpf_dev *cptpf = obj; + struct otx2_cpt_bitmap bmap; + int i, timeout = 10; + int busy, ret; + u64 reg = 0; + + bmap = get_cores_bmap(&cptpf->pdev->dev, eng_grp); + if (!bmap.size) + return -EINVAL; + + /* Detach the cores from group */ + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL2(i), ®); + if (ret) + return ret; + + if (reg & (1ull << eng_grp->idx)) { + eng_grp->g->eng_ref_cnt[i]--; + reg &= ~(1ull << eng_grp->idx); + + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL2(i), reg); + if (ret) + return ret; + } + } + + /* Wait for cores to become idle */ + do { + busy = 0; + usleep_range(10000, 20000); + if (timeout-- < 0) + return -EBUSY; + + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_STS(i), ®); + if (ret) + return ret; + + if (reg & 0x1) { + busy = 1; + break; + } + } + } while (busy); + + /* Disable the cores only if they are not used anymore */ + for_each_set_bit(i, bmap.bits, bmap.size) { + if (!eng_grp->g->eng_ref_cnt[i]) { + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL(i), 0x0); + if (ret) + return ret; + } + } + + return 0; +} + +static int cpt_attach_and_enable_cores(struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + struct otx2_cptpf_dev *cptpf = obj; + struct otx2_cpt_bitmap bmap; + u64 reg = 0; + int i, ret; + + bmap = get_cores_bmap(&cptpf->pdev->dev, eng_grp); + if (!bmap.size) + return -EINVAL; + + /* Attach the cores to the group */ + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL2(i), ®); + if (ret) + return ret; + + if (!(reg & (1ull << eng_grp->idx))) { + eng_grp->g->eng_ref_cnt[i]++; + reg |= 1ull << eng_grp->idx; + + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL2(i), reg); + if (ret) + return ret; + } + } + + /* Enable the cores */ + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_add_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL(i), 0x1); + if (ret) + return ret; + } + ret = otx2_cpt_send_af_reg_requests(&cptpf->afpf_mbox, cptpf->pdev); + + return ret; +} + +static int load_fw(struct device *dev, struct fw_info_t *fw_info, + char *filename) +{ + struct otx2_cpt_ucode_hdr *ucode_hdr; + struct otx2_cpt_uc_info_t *uc_info; + int ucode_type, ucode_size; + int ret; + + uc_info = kzalloc(sizeof(*uc_info), GFP_KERNEL); + if (!uc_info) + return -ENOMEM; + + ret = request_firmware(&uc_info->fw, filename, dev); + if (ret) + goto free_uc_info; + + ucode_hdr = (struct otx2_cpt_ucode_hdr *)uc_info->fw->data; + ret = get_ucode_type(dev, ucode_hdr, &ucode_type); + if (ret) + goto release_fw; + + ucode_size = ntohl(ucode_hdr->code_length) * 2; + if (!ucode_size) { + dev_err(dev, "Ucode %s invalid size\n", filename); + ret = -EINVAL; + goto release_fw; + } + + set_ucode_filename(&uc_info->ucode, filename); + memcpy(uc_info->ucode.ver_str, ucode_hdr->ver_str, + OTX2_CPT_UCODE_VER_STR_SZ); + uc_info->ucode.ver_num = ucode_hdr->ver_num; + uc_info->ucode.type = ucode_type; + uc_info->ucode.size = ucode_size; + list_add_tail(&uc_info->list, &fw_info->ucodes); + + return 0; + +release_fw: + release_firmware(uc_info->fw); +free_uc_info: + kfree(uc_info); + return ret; +} + +static void cpt_ucode_release_fw(struct fw_info_t *fw_info) +{ + struct otx2_cpt_uc_info_t *curr, *temp; + + if (!fw_info) + return; + + list_for_each_entry_safe(curr, temp, &fw_info->ucodes, list) { + list_del(&curr->list); + release_firmware(curr->fw); + kfree(curr); + } +} + +static struct otx2_cpt_uc_info_t *get_ucode(struct fw_info_t *fw_info, + int ucode_type) +{ + struct otx2_cpt_uc_info_t *curr; + + list_for_each_entry(curr, &fw_info->ucodes, list) { + if (!is_eng_type(curr->ucode.type, ucode_type)) + continue; + + return curr; + } + return NULL; +} + +static void print_uc_info(struct fw_info_t *fw_info) +{ + struct otx2_cpt_uc_info_t *curr; + + list_for_each_entry(curr, &fw_info->ucodes, list) { + pr_debug("Ucode filename %s\n", curr->ucode.filename); + pr_debug("Ucode version string %s\n", curr->ucode.ver_str); + pr_debug("Ucode version %d.%d.%d.%d\n", + curr->ucode.ver_num.nn, curr->ucode.ver_num.xx, + curr->ucode.ver_num.yy, curr->ucode.ver_num.zz); + pr_debug("Ucode type (%d) %s\n", curr->ucode.type, + get_ucode_type_str(curr->ucode.type)); + pr_debug("Ucode size %d\n", curr->ucode.size); + pr_debug("Ucode ptr %p\n", curr->fw->data); + } +} + +static int cpt_ucode_load_fw(struct pci_dev *pdev, struct fw_info_t *fw_info) +{ + char filename[OTX2_CPT_NAME_LENGTH]; + char eng_type[8] = {0}; + int ret, e, i; + + INIT_LIST_HEAD(&fw_info->ucodes); + + for (e = 1; e < OTX2_CPT_MAX_ENG_TYPES; e++) { + strcpy(eng_type, get_eng_type_str(e)); + for (i = 0; i < strlen(eng_type); i++) + eng_type[i] = tolower(eng_type[i]); + + snprintf(filename, sizeof(filename), "mrvl/cpt%02d/%s.out", + pdev->revision, eng_type); + /* Request firmware for each engine type */ + ret = load_fw(&pdev->dev, fw_info, filename); + if (ret) + goto release_fw; + } + print_uc_info(fw_info); + return 0; + +release_fw: + cpt_ucode_release_fw(fw_info); + return ret; +} + +static struct otx2_cpt_engs_rsvd *find_engines_by_type( + struct otx2_cpt_eng_grp_info *eng_grp, + int eng_type) +{ + int i; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!eng_grp->engs[i].type) + continue; + + if (eng_grp->engs[i].type == eng_type) + return &eng_grp->engs[i]; + } + return NULL; +} + +static int eng_grp_has_eng_type(struct otx2_cpt_eng_grp_info *eng_grp, + int eng_type) +{ + struct otx2_cpt_engs_rsvd *engs; + + engs = find_engines_by_type(eng_grp, eng_type); + + return (engs != NULL ? 1 : 0); +} + +static int update_engines_avail_count(struct device *dev, + struct otx2_cpt_engs_available *avail, + struct otx2_cpt_engs_rsvd *engs, int val) +{ + switch (engs->type) { + case OTX2_CPT_SE_TYPES: + avail->se_cnt += val; + break; + + case OTX2_CPT_IE_TYPES: + avail->ie_cnt += val; + break; + + case OTX2_CPT_AE_TYPES: + avail->ae_cnt += val; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + return 0; +} + +static int update_engines_offset(struct device *dev, + struct otx2_cpt_engs_available *avail, + struct otx2_cpt_engs_rsvd *engs) +{ + switch (engs->type) { + case OTX2_CPT_SE_TYPES: + engs->offset = 0; + break; + + case OTX2_CPT_IE_TYPES: + engs->offset = avail->max_se_cnt; + break; + + case OTX2_CPT_AE_TYPES: + engs->offset = avail->max_se_cnt + avail->max_ie_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + return 0; +} + +static int release_engines(struct device *dev, + struct otx2_cpt_eng_grp_info *grp) +{ + int i, ret = 0; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!grp->engs[i].type) + continue; + + if (grp->engs[i].count > 0) { + ret = update_engines_avail_count(dev, &grp->g->avail, + &grp->engs[i], + grp->engs[i].count); + if (ret) + return ret; + } + + grp->engs[i].type = 0; + grp->engs[i].count = 0; + grp->engs[i].offset = 0; + grp->engs[i].ucode = NULL; + bitmap_zero(grp->engs[i].bmap, grp->g->engs_num); + } + return 0; +} + +static int do_reserve_engines(struct device *dev, + struct otx2_cpt_eng_grp_info *grp, + struct otx2_cpt_engines *req_engs) +{ + struct otx2_cpt_engs_rsvd *engs = NULL; + int i, ret; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!grp->engs[i].type) { + engs = &grp->engs[i]; + break; + } + } + + if (!engs) + return -ENOMEM; + + engs->type = req_engs->type; + engs->count = req_engs->count; + + ret = update_engines_offset(dev, &grp->g->avail, engs); + if (ret) + return ret; + + if (engs->count > 0) { + ret = update_engines_avail_count(dev, &grp->g->avail, engs, + -engs->count); + if (ret) + return ret; + } + + return 0; +} + +static int check_engines_availability(struct device *dev, + struct otx2_cpt_eng_grp_info *grp, + struct otx2_cpt_engines *req_eng) +{ + int avail_cnt = 0; + + switch (req_eng->type) { + case OTX2_CPT_SE_TYPES: + avail_cnt = grp->g->avail.se_cnt; + break; + + case OTX2_CPT_IE_TYPES: + avail_cnt = grp->g->avail.ie_cnt; + break; + + case OTX2_CPT_AE_TYPES: + avail_cnt = grp->g->avail.ae_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", req_eng->type); + return -EINVAL; + } + + if (avail_cnt < req_eng->count) { + dev_err(dev, + "Error available %s engines %d < than requested %d\n", + get_eng_type_str(req_eng->type), + avail_cnt, req_eng->count); + return -EBUSY; + } + return 0; +} + +static int reserve_engines(struct device *dev, + struct otx2_cpt_eng_grp_info *grp, + struct otx2_cpt_engines *req_engs, int ucodes_cnt) +{ + int i, ret = 0; + + /* Validate if a number of requested engines are available */ + for (i = 0; i < ucodes_cnt; i++) { + ret = check_engines_availability(dev, grp, &req_engs[i]); + if (ret) + return ret; + } + + /* Reserve requested engines for this engine group */ + for (i = 0; i < ucodes_cnt; i++) { + ret = do_reserve_engines(dev, grp, &req_engs[i]); + if (ret) + return ret; + } + return 0; +} + +static void ucode_unload(struct device *dev, struct otx2_cpt_ucode *ucode) +{ + if (ucode->va) { + dma_free_coherent(dev, ucode->size, ucode->va, ucode->dma); + ucode->va = NULL; + ucode->dma = 0; + ucode->size = 0; + } + + memset(&ucode->ver_str, 0, OTX2_CPT_UCODE_VER_STR_SZ); + memset(&ucode->ver_num, 0, sizeof(struct otx2_cpt_ucode_ver_num)); + set_ucode_filename(ucode, ""); + ucode->type = 0; +} + +static int copy_ucode_to_dma_mem(struct device *dev, + struct otx2_cpt_ucode *ucode, + const u8 *ucode_data) +{ + u32 i; + + /* Allocate DMAable space */ + ucode->va = dma_alloc_coherent(dev, ucode->size, &ucode->dma, + GFP_KERNEL); + if (!ucode->va) + return -ENOMEM; + + memcpy(ucode->va, ucode_data + sizeof(struct otx2_cpt_ucode_hdr), + ucode->size); + + /* Byte swap 64-bit */ + for (i = 0; i < (ucode->size / 8); i++) + cpu_to_be64s(&((u64 *)ucode->va)[i]); + /* Ucode needs 16-bit swap */ + for (i = 0; i < (ucode->size / 2); i++) + cpu_to_be16s(&((u16 *)ucode->va)[i]); + return 0; +} + +static int enable_eng_grp(struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + int ret; + + /* Point microcode to each core of the group */ + ret = cpt_set_ucode_base(eng_grp, obj); + if (ret) + return ret; + + /* Attach the cores to the group and enable them */ + ret = cpt_attach_and_enable_cores(eng_grp, obj); + + return ret; +} + +static int disable_eng_grp(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + int i, ret; + + /* Disable all engines used by this group */ + ret = cpt_detach_and_disable_cores(eng_grp, obj); + if (ret) + return ret; + + /* Unload ucode used by this engine group */ + ucode_unload(dev, &eng_grp->ucode[0]); + ucode_unload(dev, &eng_grp->ucode[1]); + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!eng_grp->engs[i].type) + continue; + + eng_grp->engs[i].ucode = &eng_grp->ucode[0]; + } + + /* Clear UCODE_BASE register for each engine used by this group */ + ret = cpt_set_ucode_base(eng_grp, obj); + + return ret; +} + +static void setup_eng_grp_mirroring(struct otx2_cpt_eng_grp_info *dst_grp, + struct otx2_cpt_eng_grp_info *src_grp) +{ + /* Setup fields for engine group which is mirrored */ + src_grp->mirror.is_ena = false; + src_grp->mirror.idx = 0; + src_grp->mirror.ref_count++; + + /* Setup fields for mirroring engine group */ + dst_grp->mirror.is_ena = true; + dst_grp->mirror.idx = src_grp->idx; + dst_grp->mirror.ref_count = 0; +} + +static void remove_eng_grp_mirroring(struct otx2_cpt_eng_grp_info *dst_grp) +{ + struct otx2_cpt_eng_grp_info *src_grp; + + if (!dst_grp->mirror.is_ena) + return; + + src_grp = &dst_grp->g->grp[dst_grp->mirror.idx]; + + src_grp->mirror.ref_count--; + dst_grp->mirror.is_ena = false; + dst_grp->mirror.idx = 0; + dst_grp->mirror.ref_count = 0; +} + +static void update_requested_engs(struct otx2_cpt_eng_grp_info *mirror_eng_grp, + struct otx2_cpt_engines *engs, int engs_cnt) +{ + struct otx2_cpt_engs_rsvd *mirrored_engs; + int i; + + for (i = 0; i < engs_cnt; i++) { + mirrored_engs = find_engines_by_type(mirror_eng_grp, + engs[i].type); + if (!mirrored_engs) + continue; + + /* + * If mirrored group has this type of engines attached then + * there are 3 scenarios possible: + * 1) mirrored_engs.count == engs[i].count then all engines + * from mirrored engine group will be shared with this engine + * group + * 2) mirrored_engs.count > engs[i].count then only a subset of + * engines from mirrored engine group will be shared with this + * engine group + * 3) mirrored_engs.count < engs[i].count then all engines + * from mirrored engine group will be shared with this group + * and additional engines will be reserved for exclusively use + * by this engine group + */ + engs[i].count -= mirrored_engs->count; + } +} + +static struct otx2_cpt_eng_grp_info *find_mirrored_eng_grp( + struct otx2_cpt_eng_grp_info *grp) +{ + struct otx2_cpt_eng_grps *eng_grps = grp->g; + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + if (!eng_grps->grp[i].is_enabled) + continue; + if (eng_grps->grp[i].ucode[0].type && + eng_grps->grp[i].ucode[1].type) + continue; + if (grp->idx == i) + continue; + if (!strncasecmp(eng_grps->grp[i].ucode[0].ver_str, + grp->ucode[0].ver_str, + OTX2_CPT_UCODE_VER_STR_SZ)) + return &eng_grps->grp[i]; + } + + return NULL; +} + +static struct otx2_cpt_eng_grp_info *find_unused_eng_grp( + struct otx2_cpt_eng_grps *eng_grps) +{ + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + if (!eng_grps->grp[i].is_enabled) + return &eng_grps->grp[i]; + } + return NULL; +} + +static int eng_grp_update_masks(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp) +{ + struct otx2_cpt_engs_rsvd *engs, *mirrored_engs; + struct otx2_cpt_bitmap tmp_bmap = { {0} }; + int i, j, cnt, max_cnt; + int bit; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + if (engs->count <= 0) + continue; + + switch (engs->type) { + case OTX2_CPT_SE_TYPES: + max_cnt = eng_grp->g->avail.max_se_cnt; + break; + + case OTX2_CPT_IE_TYPES: + max_cnt = eng_grp->g->avail.max_ie_cnt; + break; + + case OTX2_CPT_AE_TYPES: + max_cnt = eng_grp->g->avail.max_ae_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + + cnt = engs->count; + WARN_ON(engs->offset + max_cnt > OTX2_CPT_MAX_ENGINES); + bitmap_zero(tmp_bmap.bits, eng_grp->g->engs_num); + for (j = engs->offset; j < engs->offset + max_cnt; j++) { + if (!eng_grp->g->eng_ref_cnt[j]) { + bitmap_set(tmp_bmap.bits, j, 1); + cnt--; + if (!cnt) + break; + } + } + + if (cnt) + return -ENOSPC; + + bitmap_copy(engs->bmap, tmp_bmap.bits, eng_grp->g->engs_num); + } + + if (!eng_grp->mirror.is_ena) + return 0; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + + mirrored_engs = find_engines_by_type( + &eng_grp->g->grp[eng_grp->mirror.idx], + engs->type); + WARN_ON(!mirrored_engs && engs->count <= 0); + if (!mirrored_engs) + continue; + + bitmap_copy(tmp_bmap.bits, mirrored_engs->bmap, + eng_grp->g->engs_num); + if (engs->count < 0) { + bit = find_first_bit(mirrored_engs->bmap, + eng_grp->g->engs_num); + bitmap_clear(tmp_bmap.bits, bit, -engs->count); + } + bitmap_or(engs->bmap, engs->bmap, tmp_bmap.bits, + eng_grp->g->engs_num); + } + return 0; +} + +static int delete_engine_group(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp) +{ + int ret; + + if (!eng_grp->is_enabled) + return 0; + + if (eng_grp->mirror.ref_count) + return -EINVAL; + + /* Removing engine group mirroring if enabled */ + remove_eng_grp_mirroring(eng_grp); + + /* Disable engine group */ + ret = disable_eng_grp(dev, eng_grp, eng_grp->g->obj); + if (ret) + return ret; + + /* Release all engines held by this engine group */ + ret = release_engines(dev, eng_grp); + if (ret) + return ret; + + eng_grp->is_enabled = false; + + return 0; +} + +static void update_ucode_ptrs(struct otx2_cpt_eng_grp_info *eng_grp) +{ + struct otx2_cpt_ucode *ucode; + + if (eng_grp->mirror.is_ena) + ucode = &eng_grp->g->grp[eng_grp->mirror.idx].ucode[0]; + else + ucode = &eng_grp->ucode[0]; + WARN_ON(!eng_grp->engs[0].type); + eng_grp->engs[0].ucode = ucode; + + if (eng_grp->engs[1].type) { + if (is_2nd_ucode_used(eng_grp)) + eng_grp->engs[1].ucode = &eng_grp->ucode[1]; + else + eng_grp->engs[1].ucode = ucode; + } +} + +static int create_engine_group(struct device *dev, + struct otx2_cpt_eng_grps *eng_grps, + struct otx2_cpt_engines *engs, int ucodes_cnt, + void *ucode_data[], int is_print) +{ + struct otx2_cpt_eng_grp_info *mirrored_eng_grp; + struct otx2_cpt_eng_grp_info *eng_grp; + struct otx2_cpt_uc_info_t *uc_info; + int i, ret = 0; + + /* Find engine group which is not used */ + eng_grp = find_unused_eng_grp(eng_grps); + if (!eng_grp) { + dev_err(dev, "Error all engine groups are being used\n"); + return -ENOSPC; + } + /* Load ucode */ + for (i = 0; i < ucodes_cnt; i++) { + uc_info = (struct otx2_cpt_uc_info_t *) ucode_data[i]; + eng_grp->ucode[i] = uc_info->ucode; + ret = copy_ucode_to_dma_mem(dev, &eng_grp->ucode[i], + uc_info->fw->data); + if (ret) + goto unload_ucode; + } + + /* Check if this group mirrors another existing engine group */ + mirrored_eng_grp = find_mirrored_eng_grp(eng_grp); + if (mirrored_eng_grp) { + /* Setup mirroring */ + setup_eng_grp_mirroring(eng_grp, mirrored_eng_grp); + + /* + * Update count of requested engines because some + * of them might be shared with mirrored group + */ + update_requested_engs(mirrored_eng_grp, engs, ucodes_cnt); + } + ret = reserve_engines(dev, eng_grp, engs, ucodes_cnt); + if (ret) + goto unload_ucode; + + /* Update ucode pointers used by engines */ + update_ucode_ptrs(eng_grp); + + /* Update engine masks used by this group */ + ret = eng_grp_update_masks(dev, eng_grp); + if (ret) + goto release_engs; + + /* Enable engine group */ + ret = enable_eng_grp(eng_grp, eng_grps->obj); + if (ret) + goto release_engs; + + /* + * If this engine group mirrors another engine group + * then we need to unload ucode as we will use ucode + * from mirrored engine group + */ + if (eng_grp->mirror.is_ena) + ucode_unload(dev, &eng_grp->ucode[0]); + + eng_grp->is_enabled = true; + + if (!is_print) + return 0; + + if (mirrored_eng_grp) + dev_info(dev, + "Engine_group%d: reuse microcode %s from group %d\n", + eng_grp->idx, mirrored_eng_grp->ucode[0].ver_str, + mirrored_eng_grp->idx); + else + dev_info(dev, "Engine_group%d: microcode loaded %s\n", + eng_grp->idx, eng_grp->ucode[0].ver_str); + if (is_2nd_ucode_used(eng_grp)) + dev_info(dev, "Engine_group%d: microcode loaded %s\n", + eng_grp->idx, eng_grp->ucode[1].ver_str); + + return 0; + +release_engs: + release_engines(dev, eng_grp); +unload_ucode: + ucode_unload(dev, &eng_grp->ucode[0]); + ucode_unload(dev, &eng_grp->ucode[1]); + return ret; +} + +static void delete_engine_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + int i; + + /* First delete all mirroring engine groups */ + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) + if (eng_grps->grp[i].mirror.is_ena) + delete_engine_group(&pdev->dev, &eng_grps->grp[i]); + + /* Delete remaining engine groups */ + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) + delete_engine_group(&pdev->dev, &eng_grps->grp[i]); +} + +int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type) +{ + + int eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP; + struct otx2_cpt_eng_grp_info *grp; + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + if (!grp->is_enabled) + continue; + + if (eng_type == OTX2_CPT_SE_TYPES) { + if (eng_grp_has_eng_type(grp, eng_type) && + !eng_grp_has_eng_type(grp, OTX2_CPT_IE_TYPES)) { + eng_grp_num = i; + break; + } + } else { + if (eng_grp_has_eng_type(grp, eng_type)) { + eng_grp_num = i; + break; + } + } + } + return eng_grp_num; +} + +int otx2_cpt_create_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = { }; + struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} }; + struct fw_info_t fw_info; + int ret; + + /* + * We don't create engine groups if it was already + * made (when user enabled VFs for the first time) + */ + if (eng_grps->is_grps_created) + return 0; + + ret = cpt_ucode_load_fw(pdev, &fw_info); + if (ret) + return ret; + + /* + * Create engine group with SE engines for kernel + * crypto functionality (symmetric crypto) + */ + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for SE\n"); + ret = -EINVAL; + goto release_fw; + } + engs[0].type = OTX2_CPT_SE_TYPES; + engs[0].count = eng_grps->avail.max_se_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 1); + if (ret) + goto release_fw; + + /* + * Create engine group with SE+IE engines for IPSec. + * All SE engines will be shared with engine group 0. + */ + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + uc_info[1] = get_ucode(&fw_info, OTX2_CPT_IE_TYPES); + + if (uc_info[1] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for IE"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_SE_TYPES; + engs[0].count = eng_grps->avail.max_se_cnt; + engs[1].type = OTX2_CPT_IE_TYPES; + engs[1].count = eng_grps->avail.max_ie_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 2, + (void **) uc_info, 1); + if (ret) + goto delete_eng_grp; + + /* + * Create engine group with AE engines for asymmetric + * crypto functionality. + */ + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_AE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for AE"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_AE_TYPES; + engs[0].count = eng_grps->avail.max_ae_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 1); + if (ret) + goto delete_eng_grp; + + eng_grps->is_grps_created = true; + + cpt_ucode_release_fw(&fw_info); + return 0; + +delete_eng_grp: + delete_engine_grps(pdev, eng_grps); +release_fw: + cpt_ucode_release_fw(&fw_info); + return ret; +} + +int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf) +{ + int i, ret, busy, total_cores; + int timeout = 10; + u64 reg = 0; + + total_cores = cptpf->eng_grps.avail.max_se_cnt + + cptpf->eng_grps.avail.max_ie_cnt + + cptpf->eng_grps.avail.max_ae_cnt; + + /* Disengage the cores from groups */ + for (i = 0; i < total_cores; i++) { + ret = otx2_cpt_add_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL2(i), 0x0); + if (ret) + return ret; + + cptpf->eng_grps.eng_ref_cnt[i] = 0; + } + ret = otx2_cpt_send_af_reg_requests(&cptpf->afpf_mbox, cptpf->pdev); + if (ret) + return ret; + + /* Wait for cores to become idle */ + do { + busy = 0; + usleep_range(10000, 20000); + if (timeout-- < 0) + return -EBUSY; + + for (i = 0; i < total_cores; i++) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_STS(i), ®); + if (ret) + return ret; + + if (reg & 0x1) { + busy = 1; + break; + } + } + } while (busy); + + /* Disable the cores */ + for (i = 0; i < total_cores; i++) { + ret = otx2_cpt_add_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL(i), 0x0); + if (ret) + return ret; + } + return otx2_cpt_send_af_reg_requests(&cptpf->afpf_mbox, cptpf->pdev); +} + +void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_eng_grp_info *grp; + int i, j; + + delete_engine_grps(pdev, eng_grps); + /* Release memory */ + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) { + kfree(grp->engs[j].bmap); + grp->engs[j].bmap = NULL; + } + } +} + +int otx2_cpt_init_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_eng_grp_info *grp; + int i, j, ret; + + eng_grps->obj = pci_get_drvdata(pdev); + eng_grps->avail.se_cnt = eng_grps->avail.max_se_cnt; + eng_grps->avail.ie_cnt = eng_grps->avail.max_ie_cnt; + eng_grps->avail.ae_cnt = eng_grps->avail.max_ae_cnt; + + eng_grps->engs_num = eng_grps->avail.max_se_cnt + + eng_grps->avail.max_ie_cnt + + eng_grps->avail.max_ae_cnt; + if (eng_grps->engs_num > OTX2_CPT_MAX_ENGINES) { + dev_err(&pdev->dev, + "Number of engines %d > than max supported %d\n", + eng_grps->engs_num, OTX2_CPT_MAX_ENGINES); + ret = -EINVAL; + goto cleanup_eng_grps; + } + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + grp->g = eng_grps; + grp->idx = i; + + for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) { + grp->engs[j].bmap = + kcalloc(BITS_TO_LONGS(eng_grps->engs_num), + sizeof(long), GFP_KERNEL); + if (!grp->engs[j].bmap) { + ret = -ENOMEM; + goto cleanup_eng_grps; + } + } + } + return 0; + +cleanup_eng_grps: + otx2_cpt_cleanup_eng_grps(pdev, eng_grps); + return ret; +} + +static int create_eng_caps_discovery_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = { }; + struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} }; + struct fw_info_t fw_info; + int ret; + + ret = cpt_ucode_load_fw(pdev, &fw_info); + if (ret) + return ret; + + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for AE\n"); + ret = -EINVAL; + goto release_fw; + } + engs[0].type = OTX2_CPT_AE_TYPES; + engs[0].count = 2; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 0); + if (ret) + goto release_fw; + + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for SE\n"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_SE_TYPES; + engs[0].count = 2; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 0); + if (ret) + goto delete_eng_grp; + + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_IE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for IE\n"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_IE_TYPES; + engs[0].count = 2; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 0); + if (ret) + goto delete_eng_grp; + + cpt_ucode_release_fw(&fw_info); + return 0; + +delete_eng_grp: + delete_engine_grps(pdev, eng_grps); +release_fw: + cpt_ucode_release_fw(&fw_info); + return ret; +} + +/* + * Get CPT HW capabilities using LOAD_FVC operation. + */ +int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf) +{ + struct otx2_cptlfs_info *lfs = &cptpf->lfs; + struct otx2_cpt_iq_command iq_cmd; + union otx2_cpt_opcode opcode; + union otx2_cpt_res_s *result; + union otx2_cpt_inst_s inst; + dma_addr_t rptr_baddr; + struct pci_dev *pdev; + u32 len, compl_rlen; + int ret, etype; + void *rptr; + + /* + * We don't get capabilities if it was already done + * (when user enabled VFs for the first time) + */ + if (cptpf->is_eng_caps_discovered) + return 0; + + pdev = cptpf->pdev; + /* + * Create engine groups for each type to submit LOAD_FVC op and + * get engine's capabilities. + */ + ret = create_eng_caps_discovery_grps(pdev, &cptpf->eng_grps); + if (ret) + goto delete_grps; + + lfs->pdev = pdev; + lfs->reg_base = cptpf->reg_base; + lfs->mbox = &cptpf->afpf_mbox; + ret = otx2_cptlf_init(&cptpf->lfs, OTX2_CPT_ALL_ENG_GRPS_MASK, + OTX2_CPT_QUEUE_HI_PRIO, 1); + if (ret) + goto delete_grps; + + compl_rlen = ALIGN(sizeof(union otx2_cpt_res_s), OTX2_CPT_DMA_MINALIGN); + len = compl_rlen + LOADFVC_RLEN; + + result = kzalloc(len, GFP_KERNEL); + if (!result) { + ret = -ENOMEM; + goto lf_cleanup; + } + rptr_baddr = dma_map_single(&pdev->dev, (void *)result, len, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, rptr_baddr)) { + dev_err(&pdev->dev, "DMA mapping failed\n"); + ret = -EFAULT; + goto free_result; + } + rptr = (u8 *)result + compl_rlen; + + /* Fill in the command */ + opcode.s.major = LOADFVC_MAJOR_OP; + opcode.s.minor = LOADFVC_MINOR_OP; + + iq_cmd.cmd.u = 0; + iq_cmd.cmd.s.opcode = cpu_to_be16(opcode.flags); + + /* 64-bit swap for microcode data reads, not needed for addresses */ + cpu_to_be64s(&iq_cmd.cmd.u); + iq_cmd.dptr = 0; + iq_cmd.rptr = rptr_baddr + compl_rlen; + iq_cmd.cptr.u = 0; + + for (etype = 1; etype < OTX2_CPT_MAX_ENG_TYPES; etype++) { + result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; + iq_cmd.cptr.s.grp = otx2_cpt_get_eng_grp(&cptpf->eng_grps, + etype); + otx2_cpt_fill_inst(&inst, &iq_cmd, rptr_baddr); + otx2_cpt_send_cmd(&inst, 1, &cptpf->lfs.lf[0]); + + while (result->s.compcode == OTX2_CPT_COMPLETION_CODE_INIT) + cpu_relax(); + + cptpf->eng_caps[etype].u = be64_to_cpup(rptr); + } + dma_unmap_single(&pdev->dev, rptr_baddr, len, DMA_BIDIRECTIONAL); + cptpf->is_eng_caps_discovered = true; + +free_result: + kfree(result); +lf_cleanup: + otx2_cptlf_shutdown(&cptpf->lfs); +delete_grps: + delete_engine_grps(pdev, &cptpf->eng_grps); + + return ret; +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h new file mode 100644 index 000000000000..6b0d432de0af --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPTPF_UCODE_H +#define __OTX2_CPTPF_UCODE_H + +#include +#include +#include +#include "otx2_cpt_hw_types.h" +#include "otx2_cpt_common.h" + +/* + * On OcteonTX2 platform IPSec ucode can use both IE and SE engines therefore + * IE and SE engines can be attached to the same engine group. + */ +#define OTX2_CPT_MAX_ETYPES_PER_GRP 2 + +/* CPT ucode signature size */ +#define OTX2_CPT_UCODE_SIGN_LEN 256 + +/* Microcode version string length */ +#define OTX2_CPT_UCODE_VER_STR_SZ 44 + +/* Maximum number of supported engines/cores on OcteonTX2 platform */ +#define OTX2_CPT_MAX_ENGINES 128 + +#define OTX2_CPT_ENGS_BITMASK_LEN BITS_TO_LONGS(OTX2_CPT_MAX_ENGINES) + +/* Microcode types */ +enum otx2_cpt_ucode_type { + OTX2_CPT_AE_UC_TYPE = 1, /* AE-MAIN */ + OTX2_CPT_SE_UC_TYPE1 = 20,/* SE-MAIN - combination of 21 and 22 */ + OTX2_CPT_SE_UC_TYPE2 = 21,/* Fast Path IPSec + AirCrypto */ + OTX2_CPT_SE_UC_TYPE3 = 22,/* + * Hash + HMAC + FlexiCrypto + RNG + + * Full Feature IPSec + AirCrypto + Kasumi + */ + OTX2_CPT_IE_UC_TYPE1 = 30, /* IE-MAIN - combination of 31 and 32 */ + OTX2_CPT_IE_UC_TYPE2 = 31, /* Fast Path IPSec */ + OTX2_CPT_IE_UC_TYPE3 = 32, /* + * Hash + HMAC + FlexiCrypto + RNG + + * Full Future IPSec + */ +}; + +struct otx2_cpt_bitmap { + unsigned long bits[OTX2_CPT_ENGS_BITMASK_LEN]; + int size; +}; + +struct otx2_cpt_engines { + int type; + int count; +}; + +/* Microcode version number */ +struct otx2_cpt_ucode_ver_num { + u8 nn; + u8 xx; + u8 yy; + u8 zz; +}; + +struct otx2_cpt_ucode_hdr { + struct otx2_cpt_ucode_ver_num ver_num; + u8 ver_str[OTX2_CPT_UCODE_VER_STR_SZ]; + __be32 code_length; + u32 padding[3]; +}; + +struct otx2_cpt_ucode { + u8 ver_str[OTX2_CPT_UCODE_VER_STR_SZ];/* + * ucode version in readable + * format + */ + struct otx2_cpt_ucode_ver_num ver_num;/* ucode version number */ + char filename[OTX2_CPT_NAME_LENGTH];/* ucode filename */ + dma_addr_t dma; /* phys address of ucode image */ + void *va; /* virt address of ucode image */ + u32 size; /* ucode image size */ + int type; /* ucode image type SE, IE, AE or SE+IE */ +}; + +struct otx2_cpt_uc_info_t { + struct list_head list; + struct otx2_cpt_ucode ucode;/* microcode information */ + const struct firmware *fw; +}; + +/* Maximum and current number of engines available for all engine groups */ +struct otx2_cpt_engs_available { + int max_se_cnt; + int max_ie_cnt; + int max_ae_cnt; + int se_cnt; + int ie_cnt; + int ae_cnt; +}; + +/* Engines reserved to an engine group */ +struct otx2_cpt_engs_rsvd { + int type; /* engine type */ + int count; /* number of engines attached */ + int offset; /* constant offset of engine type in the bitmap */ + unsigned long *bmap; /* attached engines bitmap */ + struct otx2_cpt_ucode *ucode; /* ucode used by these engines */ +}; + +struct otx2_cpt_mirror_info { + int is_ena; /* + * is mirroring enabled, it is set only for engine + * group which mirrors another engine group + */ + int idx; /* + * index of engine group which is mirrored by this + * group, set only for engine group which mirrors + * another group + */ + int ref_count; /* + * number of times this engine group is mirrored by + * other groups, this is set only for engine group + * which is mirrored by other group(s) + */ +}; + +struct otx2_cpt_eng_grp_info { + struct otx2_cpt_eng_grps *g; /* pointer to engine_groups structure */ + /* engines attached */ + struct otx2_cpt_engs_rsvd engs[OTX2_CPT_MAX_ETYPES_PER_GRP]; + /* ucodes information */ + struct otx2_cpt_ucode ucode[OTX2_CPT_MAX_ETYPES_PER_GRP]; + /* engine group mirroring information */ + struct otx2_cpt_mirror_info mirror; + int idx; /* engine group index */ + bool is_enabled; /* + * is engine group enabled, engine group is enabled + * when it has engines attached and ucode loaded + */ +}; + +struct otx2_cpt_eng_grps { + struct otx2_cpt_eng_grp_info grp[OTX2_CPT_MAX_ENGINE_GROUPS]; + struct otx2_cpt_engs_available avail; + void *obj; /* device specific data */ + int engs_num; /* total number of engines supported */ + u8 eng_ref_cnt[OTX2_CPT_MAX_ENGINES];/* engines reference count */ + bool is_grps_created; /* Is the engine groups are already created */ +}; +struct otx2_cptpf_dev; +int otx2_cpt_init_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps); +void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps); +int otx2_cpt_create_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps); +int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf); +int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type); +int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf); + +#endif /* __OTX2_CPTPF_UCODE_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf.h b/drivers/crypto/marvell/octeontx2/otx2_cptvf.h new file mode 100644 index 000000000000..4f0a169fddbd --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPTVF_H +#define __OTX2_CPTVF_H + +#include "mbox.h" +#include "otx2_cptlf.h" + +struct otx2_cptvf_dev { + void __iomem *reg_base; /* Register start address */ + void __iomem *pfvf_mbox_base; /* PF-VF mbox start address */ + struct pci_dev *pdev; /* PCI device handle */ + struct otx2_cptlfs_info lfs; /* CPT LFs attached to this VF */ + u8 vf_id; /* Virtual function index */ + + /* PF <=> VF mbox */ + struct otx2_mbox pfvf_mbox; + struct work_struct pfvf_mbox_work; + struct workqueue_struct *pfvf_mbox_wq; +}; + +irqreturn_t otx2_cptvf_pfvf_mbox_intr(int irq, void *arg); +void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work); +int otx2_cptvf_send_eng_grp_num_msg(struct otx2_cptvf_dev *cptvf, int eng_type); +int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf); + +#endif /* __OTX2_CPTVF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c new file mode 100644 index 000000000000..a72723455df7 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c @@ -0,0 +1,1758 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "otx2_cptvf.h" +#include "otx2_cptvf_algs.h" +#include "otx2_cpt_reqmgr.h" + +/* Size of salt in AES GCM mode */ +#define AES_GCM_SALT_SIZE 4 +/* Size of IV in AES GCM mode */ +#define AES_GCM_IV_SIZE 8 +/* Size of ICV (Integrity Check Value) in AES GCM mode */ +#define AES_GCM_ICV_SIZE 16 +/* Offset of IV in AES GCM mode */ +#define AES_GCM_IV_OFFSET 8 +#define CONTROL_WORD_LEN 8 +#define KEY2_OFFSET 48 +#define DMA_MODE_FLAG(dma_mode) \ + (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0) + +/* Truncated SHA digest size */ +#define SHA1_TRUNC_DIGEST_SIZE 12 +#define SHA256_TRUNC_DIGEST_SIZE 16 +#define SHA384_TRUNC_DIGEST_SIZE 24 +#define SHA512_TRUNC_DIGEST_SIZE 32 + +static DEFINE_MUTEX(mutex); +static int is_crypto_registered; + +struct cpt_device_desc { + struct pci_dev *dev; + int num_queues; +}; + +struct cpt_device_table { + atomic_t count; + struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM]; +}; + +static struct cpt_device_table se_devices = { + .count = ATOMIC_INIT(0) +}; + +static inline int get_se_device(struct pci_dev **pdev, int *cpu_num) +{ + int count; + + count = atomic_read(&se_devices.count); + if (count < 1) + return -ENODEV; + + *cpu_num = get_cpu(); + /* + * On OcteonTX2 platform CPT instruction queue is bound to each + * local function LF, in turn LFs can be attached to PF + * or VF therefore we always use first device. We get maximum + * performance if one CPT queue is available for each cpu + * otherwise CPT queues need to be shared between cpus. + */ + if (*cpu_num >= se_devices.desc[0].num_queues) + *cpu_num %= se_devices.desc[0].num_queues; + *pdev = se_devices.desc[0].dev; + + put_cpu(); + + return 0; +} + +static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req) +{ + struct otx2_cpt_req_ctx *rctx; + struct aead_request *req; + struct crypto_aead *tfm; + + req = container_of(cpt_req->areq, struct aead_request, base); + tfm = crypto_aead_reqtfm(req); + rctx = aead_request_ctx(req); + if (memcmp(rctx->fctx.hmac.s.hmac_calc, + rctx->fctx.hmac.s.hmac_recv, + crypto_aead_authsize(tfm)) != 0) + return -EBADMSG; + + return 0; +} + +static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2) +{ + struct otx2_cpt_inst_info *inst_info = arg2; + struct crypto_async_request *areq = arg1; + struct otx2_cpt_req_info *cpt_req; + struct pci_dev *pdev; + + if (inst_info) { + cpt_req = inst_info->req; + if (!status) { + /* + * When selected cipher is NULL we need to manually + * verify whether calculated hmac value matches + * received hmac value + */ + if (cpt_req->req_type == + OTX2_CPT_AEAD_ENC_DEC_NULL_REQ && + !cpt_req->is_enc) + status = validate_hmac_cipher_null(cpt_req); + } + pdev = inst_info->pdev; + otx2_cpt_info_destroy(pdev, inst_info); + } + if (areq) + areq->complete(areq, status); +} + +static void output_iv_copyback(struct crypto_async_request *areq) +{ + struct otx2_cpt_req_info *req_info; + struct otx2_cpt_req_ctx *rctx; + struct skcipher_request *sreq; + struct crypto_skcipher *stfm; + struct otx2_cpt_enc_ctx *ctx; + u32 start, ivsize; + + sreq = container_of(areq, struct skcipher_request, base); + stfm = crypto_skcipher_reqtfm(sreq); + ctx = crypto_skcipher_ctx(stfm); + if (ctx->cipher_type == OTX2_CPT_AES_CBC || + ctx->cipher_type == OTX2_CPT_DES3_CBC) { + rctx = skcipher_request_ctx(sreq); + req_info = &rctx->cpt_req; + ivsize = crypto_skcipher_ivsize(stfm); + start = sreq->cryptlen - ivsize; + + if (req_info->is_enc) { + scatterwalk_map_and_copy(sreq->iv, sreq->dst, start, + ivsize, 0); + } else { + if (sreq->src != sreq->dst) { + scatterwalk_map_and_copy(sreq->iv, sreq->src, + start, ivsize, 0); + } else { + memcpy(sreq->iv, req_info->iv_out, ivsize); + kfree(req_info->iv_out); + } + } + } +} + +static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2) +{ + struct otx2_cpt_inst_info *inst_info = arg2; + struct crypto_async_request *areq = arg1; + struct pci_dev *pdev; + + if (areq) { + if (!status) + output_iv_copyback(areq); + if (inst_info) { + pdev = inst_info->pdev; + otx2_cpt_info_destroy(pdev, inst_info); + } + areq->complete(areq, status); + } +} + +static inline void update_input_data(struct otx2_cpt_req_info *req_info, + struct scatterlist *inp_sg, + u32 nbytes, u32 *argcnt) +{ + req_info->req.dlen += nbytes; + + while (nbytes) { + u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length; + u8 *ptr = sg_virt(inp_sg); + + req_info->in[*argcnt].vptr = (void *)ptr; + req_info->in[*argcnt].size = len; + nbytes -= len; + ++(*argcnt); + inp_sg = sg_next(inp_sg); + } +} + +static inline void update_output_data(struct otx2_cpt_req_info *req_info, + struct scatterlist *outp_sg, + u32 offset, u32 nbytes, u32 *argcnt) +{ + u32 len, sg_len; + u8 *ptr; + + req_info->rlen += nbytes; + + while (nbytes) { + sg_len = outp_sg->length - offset; + len = (nbytes < sg_len) ? nbytes : sg_len; + ptr = sg_virt(outp_sg); + + req_info->out[*argcnt].vptr = (void *) (ptr + offset); + req_info->out[*argcnt].size = len; + nbytes -= len; + ++(*argcnt); + offset = 0; + outp_sg = sg_next(outp_sg); + } +} + +static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc, + u32 *argcnt) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; + int ivsize = crypto_skcipher_ivsize(stfm); + u32 start = req->cryptlen - ivsize; + gfp_t flags; + + flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; + req_info->ctrl.s.se_req = 1; + + req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | + DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); + if (enc) { + req_info->req.opcode.s.minor = 2; + } else { + req_info->req.opcode.s.minor = 3; + if ((ctx->cipher_type == OTX2_CPT_AES_CBC || + ctx->cipher_type == OTX2_CPT_DES3_CBC) && + req->src == req->dst) { + req_info->iv_out = kmalloc(ivsize, flags); + if (!req_info->iv_out) + return -ENOMEM; + + scatterwalk_map_and_copy(req_info->iv_out, req->src, + start, ivsize, 0); + } + } + /* Encryption data length */ + req_info->req.param1 = req->cryptlen; + /* Authentication data length */ + req_info->req.param2 = 0; + + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; + fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; + + if (ctx->cipher_type == OTX2_CPT_AES_XTS) + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); + else + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); + + memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm)); + + cpu_to_be64s(&fctx->enc.enc_ctrl.u); + + /* + * Storing Packet Data Information in offset + * Control Word First 8 bytes + */ + req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; + req_info->in[*argcnt].size = CONTROL_WORD_LEN; + req_info->req.dlen += CONTROL_WORD_LEN; + ++(*argcnt); + + req_info->in[*argcnt].vptr = (u8 *)fctx; + req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); + req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); + + ++(*argcnt); + + return 0; +} + +static inline int create_input_list(struct skcipher_request *req, u32 enc, + u32 enc_iv_len) +{ + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + int ret; + + ret = create_ctx_hdr(req, enc, &argcnt); + if (ret) + return ret; + + update_input_data(req_info, req->src, req->cryptlen, &argcnt); + req_info->in_cnt = argcnt; + + return 0; +} + +static inline void create_output_list(struct skcipher_request *req, + u32 enc_iv_len) +{ + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + + /* + * OUTPUT Buffer Processing + * AES encryption/decryption output would be + * received in the following format + * + * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| + * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] + */ + update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt); + req_info->out_cnt = argcnt; +} + +static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + int ret; + + if (ctx->fbk_cipher) { + skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher); + skcipher_request_set_callback(&rctx->sk_fbk_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src, + req->dst, req->cryptlen, req->iv); + ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) : + crypto_skcipher_decrypt(&rctx->sk_fbk_req); + } else { + ret = -EINVAL; + } + return ret; +} + +static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 enc_iv_len = crypto_skcipher_ivsize(stfm); + struct pci_dev *pdev; + int status, cpu_num; + + if (req->cryptlen == 0) + return 0; + + if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len)) + return -EINVAL; + + if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE) + return skcipher_do_fallback(req, enc); + + /* Clear control words */ + rctx->ctrl_word.flags = 0; + rctx->fctx.enc.enc_ctrl.u = 0; + + status = create_input_list(req, enc, enc_iv_len); + if (status) + return status; + create_output_list(req, enc_iv_len); + + status = get_se_device(&pdev, &cpu_num); + if (status) + return status; + + req_info->callback = otx2_cpt_skcipher_callback; + req_info->areq = &req->base; + req_info->req_type = OTX2_CPT_ENC_DEC_REQ; + req_info->is_enc = enc; + req_info->is_trunc_hmac = false; + req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); + + /* + * We perform an asynchronous send and once + * the request is completed the driver would + * intimate through registered call back functions + */ + status = otx2_cpt_do_request(pdev, req_info, cpu_num); + + return status; +} + +static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req) +{ + return cpt_enc_dec(req, true); +} + +static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req) +{ + return cpt_enc_dec(req, false); +} + +static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + const u8 *key2 = key + (keylen / 2); + const u8 *key1 = key; + int ret; + + ret = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen); + if (ret) + return ret; + ctx->key_len = keylen; + ctx->enc_align_len = 1; + memcpy(ctx->enc_key, key1, keylen / 2); + memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); + ctx->cipher_type = OTX2_CPT_AES_XTS; + switch (ctx->key_len) { + case 2 * AES_KEYSIZE_128: + ctx->key_type = OTX2_CPT_AES_128_BIT; + break; + case 2 * AES_KEYSIZE_192: + ctx->key_type = OTX2_CPT_AES_192_BIT; + break; + case 2 * AES_KEYSIZE_256: + ctx->key_type = OTX2_CPT_AES_256_BIT; + break; + default: + return -EINVAL; + } + return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); +} + +static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key, + u32 keylen, u8 cipher_type) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != DES3_EDE_KEY_SIZE) + return -EINVAL; + + ctx->key_len = keylen; + ctx->cipher_type = cipher_type; + ctx->enc_align_len = 8; + + memcpy(ctx->enc_key, key, keylen); + + return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); +} + +static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + u32 keylen, u8 cipher_type) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + switch (keylen) { + case AES_KEYSIZE_128: + ctx->key_type = OTX2_CPT_AES_128_BIT; + break; + case AES_KEYSIZE_192: + ctx->key_type = OTX2_CPT_AES_192_BIT; + break; + case AES_KEYSIZE_256: + ctx->key_type = OTX2_CPT_AES_256_BIT; + break; + default: + return -EINVAL; + } + if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB) + ctx->enc_align_len = 16; + else + ctx->enc_align_len = 1; + + ctx->key_len = keylen; + ctx->cipher_type = cipher_type; + + memcpy(ctx->enc_key, key, keylen); + + return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); +} + +static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC); +} + +static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB); +} + +static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC); +} + +static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB); +} + +static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx, + struct crypto_alg *alg) +{ + if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + ctx->fbk_cipher = + crypto_alloc_skcipher(alg->cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fbk_cipher)) { + pr_err("%s() failed to allocate fallback for %s\n", + __func__, alg->cra_name); + return PTR_ERR(ctx->fbk_cipher); + } + } + return 0; +} + +static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); + struct crypto_alg *alg = tfm->__crt_alg; + + memset(ctx, 0, sizeof(*ctx)); + /* + * Additional memory for skcipher_request is + * allocated since the cryptd daemon uses + * this memory for request_ctx information + */ + crypto_skcipher_set_reqsize(stfm, sizeof(struct otx2_cpt_req_ctx) + + sizeof(struct skcipher_request)); + + return cpt_skcipher_fallback_init(ctx, alg); +} + +static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->fbk_cipher) { + crypto_free_skcipher(ctx->fbk_cipher); + ctx->fbk_cipher = NULL; + } +} + +static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx, + struct crypto_alg *alg) +{ + if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + ctx->fbk_cipher = + crypto_alloc_aead(alg->cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fbk_cipher)) { + pr_err("%s() failed to allocate fallback for %s\n", + __func__, alg->cra_name); + return PTR_ERR(ctx->fbk_cipher); + } + } + return 0; +} + +static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(atfm); + struct crypto_tfm *tfm = crypto_aead_tfm(atfm); + struct crypto_alg *alg = tfm->__crt_alg; + + ctx->cipher_type = cipher_type; + ctx->mac_type = mac_type; + + /* + * When selected cipher is NULL we use HMAC opcode instead of + * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms + * for calculating ipad and opad + */ + if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL) { + switch (ctx->mac_type) { + case OTX2_CPT_SHA1: + ctx->hashalg = crypto_alloc_shash("sha1", 0, + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->hashalg)) + return PTR_ERR(ctx->hashalg); + break; + + case OTX2_CPT_SHA256: + ctx->hashalg = crypto_alloc_shash("sha256", 0, + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->hashalg)) + return PTR_ERR(ctx->hashalg); + break; + + case OTX2_CPT_SHA384: + ctx->hashalg = crypto_alloc_shash("sha384", 0, + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->hashalg)) + return PTR_ERR(ctx->hashalg); + break; + + case OTX2_CPT_SHA512: + ctx->hashalg = crypto_alloc_shash("sha512", 0, + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->hashalg)) + return PTR_ERR(ctx->hashalg); + break; + } + } + switch (ctx->cipher_type) { + case OTX2_CPT_AES_CBC: + case OTX2_CPT_AES_ECB: + ctx->enc_align_len = 16; + break; + case OTX2_CPT_DES3_CBC: + case OTX2_CPT_DES3_ECB: + ctx->enc_align_len = 8; + break; + case OTX2_CPT_AES_GCM: + case OTX2_CPT_CIPHER_NULL: + ctx->enc_align_len = 1; + break; + } + crypto_aead_set_reqsize(atfm, sizeof(struct otx2_cpt_req_ctx)); + + return cpt_aead_fallback_init(ctx, alg); +} + +static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1); +} + +static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256); +} + +static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384); +} + +static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512); +} + +static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1); +} + +static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256); +} + +static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384); +} + +static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512); +} + +static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL); +} + +static void otx2_cpt_aead_exit(struct crypto_aead *tfm) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + + kfree(ctx->ipad); + kfree(ctx->opad); + if (ctx->hashalg) + crypto_free_shash(ctx->hashalg); + kfree(ctx->sdesc); + + if (ctx->fbk_cipher) { + crypto_free_aead(ctx->fbk_cipher); + ctx->fbk_cipher = NULL; + } +} + +static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + + if (crypto_rfc4106_check_authsize(authsize)) + return -EINVAL; + + tfm->authsize = authsize; + /* Set authsize for fallback case */ + if (ctx->fbk_cipher) + ctx->fbk_cipher->authsize = authsize; + + return 0; +} + +static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + tfm->authsize = authsize; + + return 0; +} + +static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + + ctx->is_trunc_hmac = true; + tfm->authsize = authsize; + + return 0; +} + +static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg) +{ + struct otx2_cpt_sdesc *sdesc; + int size; + + size = sizeof(struct shash_desc) + crypto_shash_descsize(alg); + sdesc = kmalloc(size, GFP_KERNEL); + if (!sdesc) + return NULL; + + sdesc->shash.tfm = alg; + + return sdesc; +} + +static inline void swap_data32(void *buf, u32 len) +{ + cpu_to_be32_array(buf, buf, len / 4); +} + +static inline void swap_data64(void *buf, u32 len) +{ + u64 *src = buf; + int i = 0; + + for (i = 0 ; i < len / 8; i++, src++) + cpu_to_be64s(src); +} + +static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad) +{ + struct sha512_state *sha512; + struct sha256_state *sha256; + struct sha1_state *sha1; + + switch (mac_type) { + case OTX2_CPT_SHA1: + sha1 = (struct sha1_state *) in_pad; + swap_data32(sha1->state, SHA1_DIGEST_SIZE); + memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE); + break; + + case OTX2_CPT_SHA256: + sha256 = (struct sha256_state *) in_pad; + swap_data32(sha256->state, SHA256_DIGEST_SIZE); + memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE); + break; + + case OTX2_CPT_SHA384: + case OTX2_CPT_SHA512: + sha512 = (struct sha512_state *) in_pad; + swap_data64(sha512->state, SHA512_DIGEST_SIZE); + memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE); + break; + + default: + return -EINVAL; + } + + return 0; +} + +static int aead_hmac_init(struct crypto_aead *cipher) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); + int state_size = crypto_shash_statesize(ctx->hashalg); + int ds = crypto_shash_digestsize(ctx->hashalg); + int bs = crypto_shash_blocksize(ctx->hashalg); + int authkeylen = ctx->auth_key_len; + u8 *ipad = NULL, *opad = NULL; + int ret = 0, icount = 0; + + ctx->sdesc = alloc_sdesc(ctx->hashalg); + if (!ctx->sdesc) + return -ENOMEM; + + ctx->ipad = kzalloc(bs, GFP_KERNEL); + if (!ctx->ipad) { + ret = -ENOMEM; + goto calc_fail; + } + + ctx->opad = kzalloc(bs, GFP_KERNEL); + if (!ctx->opad) { + ret = -ENOMEM; + goto calc_fail; + } + + ipad = kzalloc(state_size, GFP_KERNEL); + if (!ipad) { + ret = -ENOMEM; + goto calc_fail; + } + + opad = kzalloc(state_size, GFP_KERNEL); + if (!opad) { + ret = -ENOMEM; + goto calc_fail; + } + + if (authkeylen > bs) { + ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key, + authkeylen, ipad); + if (ret) + goto calc_fail; + + authkeylen = ds; + } else { + memcpy(ipad, ctx->key, authkeylen); + } + + memset(ipad + authkeylen, 0, bs - authkeylen); + memcpy(opad, ipad, bs); + + for (icount = 0; icount < bs; icount++) { + ipad[icount] ^= 0x36; + opad[icount] ^= 0x5c; + } + + /* + * Partial Hash calculated from the software + * algorithm is retrieved for IPAD & OPAD + */ + + /* IPAD Calculation */ + crypto_shash_init(&ctx->sdesc->shash); + crypto_shash_update(&ctx->sdesc->shash, ipad, bs); + crypto_shash_export(&ctx->sdesc->shash, ipad); + ret = copy_pad(ctx->mac_type, ctx->ipad, ipad); + if (ret) + goto calc_fail; + + /* OPAD Calculation */ + crypto_shash_init(&ctx->sdesc->shash); + crypto_shash_update(&ctx->sdesc->shash, opad, bs); + crypto_shash_export(&ctx->sdesc->shash, opad); + ret = copy_pad(ctx->mac_type, ctx->opad, opad); + if (ret) + goto calc_fail; + + kfree(ipad); + kfree(opad); + + return 0; + +calc_fail: + kfree(ctx->ipad); + ctx->ipad = NULL; + kfree(ctx->opad); + ctx->opad = NULL; + kfree(ipad); + kfree(opad); + kfree(ctx->sdesc); + ctx->sdesc = NULL; + + return ret; +} + +static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); + struct crypto_authenc_key_param *param; + int enckeylen = 0, authkeylen = 0; + struct rtattr *rta = (void *)key; + int status; + + if (!RTA_OK(rta, keylen)) + return -EINVAL; + + if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) + return -EINVAL; + + if (RTA_PAYLOAD(rta) < sizeof(*param)) + return -EINVAL; + + param = RTA_DATA(rta); + enckeylen = be32_to_cpu(param->enckeylen); + key += RTA_ALIGN(rta->rta_len); + keylen -= RTA_ALIGN(rta->rta_len); + if (keylen < enckeylen) + return -EINVAL; + + if (keylen > OTX2_CPT_MAX_KEY_SIZE) + return -EINVAL; + + authkeylen = keylen - enckeylen; + memcpy(ctx->key, key, keylen); + + switch (enckeylen) { + case AES_KEYSIZE_128: + ctx->key_type = OTX2_CPT_AES_128_BIT; + break; + case AES_KEYSIZE_192: + ctx->key_type = OTX2_CPT_AES_192_BIT; + break; + case AES_KEYSIZE_256: + ctx->key_type = OTX2_CPT_AES_256_BIT; + break; + default: + /* Invalid key length */ + return -EINVAL; + } + + ctx->enc_key_len = enckeylen; + ctx->auth_key_len = authkeylen; + + status = aead_hmac_init(cipher); + if (status) + return status; + + return 0; +} + +static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); + struct crypto_authenc_key_param *param; + struct rtattr *rta = (void *)key; + int enckeylen = 0; + + if (!RTA_OK(rta, keylen)) + return -EINVAL; + + if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) + return -EINVAL; + + if (RTA_PAYLOAD(rta) < sizeof(*param)) + return -EINVAL; + + param = RTA_DATA(rta); + enckeylen = be32_to_cpu(param->enckeylen); + key += RTA_ALIGN(rta->rta_len); + keylen -= RTA_ALIGN(rta->rta_len); + if (enckeylen != 0) + return -EINVAL; + + if (keylen > OTX2_CPT_MAX_KEY_SIZE) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->enc_key_len = enckeylen; + ctx->auth_key_len = keylen; + + return 0; +} + +static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); + + /* + * For aes gcm we expect to get encryption key (16, 24, 32 bytes) + * and salt (4 bytes) + */ + switch (keylen) { + case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX2_CPT_AES_128_BIT; + ctx->enc_key_len = AES_KEYSIZE_128; + break; + case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX2_CPT_AES_192_BIT; + ctx->enc_key_len = AES_KEYSIZE_192; + break; + case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX2_CPT_AES_256_BIT; + ctx->enc_key_len = AES_KEYSIZE_256; + break; + default: + /* Invalid key and salt length */ + return -EINVAL; + } + + /* Store encryption key and salt */ + memcpy(ctx->key, key, keylen); + + return crypto_aead_setkey(ctx->fbk_cipher, key, keylen); +} + +static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc, + u32 *argcnt) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; + int mac_len = crypto_aead_authsize(tfm); + int ds; + + rctx->ctrl_word.e.enc_data_offset = req->assoclen; + + switch (ctx->cipher_type) { + case OTX2_CPT_AES_CBC: + if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8)) + return -EINVAL; + + fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; + /* Copy encryption key to context */ + memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len, + ctx->enc_key_len); + /* Copy IV to context */ + memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm)); + + ds = crypto_shash_digestsize(ctx->hashalg); + if (ctx->mac_type == OTX2_CPT_SHA384) + ds = SHA512_DIGEST_SIZE; + if (ctx->ipad) + memcpy(fctx->hmac.e.ipad, ctx->ipad, ds); + if (ctx->opad) + memcpy(fctx->hmac.e.opad, ctx->opad, ds); + break; + + case OTX2_CPT_AES_GCM: + if (crypto_ipsec_check_assoclen(req->assoclen)) + return -EINVAL; + + fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR; + /* Copy encryption key to context */ + memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len); + /* Copy salt to context */ + memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len, + AES_GCM_SALT_SIZE); + + rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET; + break; + + default: + /* Unknown cipher type */ + return -EINVAL; + } + cpu_to_be64s(&rctx->ctrl_word.flags); + + req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; + req_info->ctrl.s.se_req = 1; + req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | + DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); + if (enc) { + req_info->req.opcode.s.minor = 2; + req_info->req.param1 = req->cryptlen; + req_info->req.param2 = req->cryptlen + req->assoclen; + } else { + req_info->req.opcode.s.minor = 3; + req_info->req.param1 = req->cryptlen - mac_len; + req_info->req.param2 = req->cryptlen + req->assoclen - mac_len; + } + + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; + fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type; + fctx->enc.enc_ctrl.e.mac_len = mac_len; + cpu_to_be64s(&fctx->enc.enc_ctrl.u); + + /* + * Storing Packet Data Information in offset + * Control Word First 8 bytes + */ + req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; + req_info->in[*argcnt].size = CONTROL_WORD_LEN; + req_info->req.dlen += CONTROL_WORD_LEN; + ++(*argcnt); + + req_info->in[*argcnt].vptr = (u8 *)fctx; + req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); + req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); + ++(*argcnt); + + return 0; +} + +static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt, + u32 enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + + req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; + req_info->ctrl.s.se_req = 1; + req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC | + DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); + req_info->is_trunc_hmac = ctx->is_trunc_hmac; + + req_info->req.opcode.s.minor = 0; + req_info->req.param1 = ctx->auth_key_len; + req_info->req.param2 = ctx->mac_type << 8; + + /* Add authentication key */ + req_info->in[*argcnt].vptr = ctx->key; + req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8); + req_info->req.dlen += round_up(ctx->auth_key_len, 8); + ++(*argcnt); +} + +static inline int create_aead_input_list(struct aead_request *req, u32 enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 inputlen = req->cryptlen + req->assoclen; + u32 status, argcnt = 0; + + status = create_aead_ctx_hdr(req, enc, &argcnt); + if (status) + return status; + update_input_data(req_info, req->src, inputlen, &argcnt); + req_info->in_cnt = argcnt; + + return 0; +} + +static inline void create_aead_output_list(struct aead_request *req, u32 enc, + u32 mac_len) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0, outputlen = 0; + + if (enc) + outputlen = req->cryptlen + req->assoclen + mac_len; + else + outputlen = req->cryptlen + req->assoclen - mac_len; + + update_output_data(req_info, req->dst, 0, outputlen, &argcnt); + req_info->out_cnt = argcnt; +} + +static inline void create_aead_null_input_list(struct aead_request *req, + u32 enc, u32 mac_len) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 inputlen, argcnt = 0; + + if (enc) + inputlen = req->cryptlen + req->assoclen; + else + inputlen = req->cryptlen + req->assoclen - mac_len; + + create_hmac_ctx_hdr(req, &argcnt, enc); + update_input_data(req_info, req->src, inputlen, &argcnt); + req_info->in_cnt = argcnt; +} + +static inline int create_aead_null_output_list(struct aead_request *req, + u32 enc, u32 mac_len) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct scatterlist *dst; + u8 *ptr = NULL; + int argcnt = 0, status, offset; + u32 inputlen; + + if (enc) + inputlen = req->cryptlen + req->assoclen; + else + inputlen = req->cryptlen + req->assoclen - mac_len; + + /* + * If source and destination are different + * then copy payload to destination + */ + if (req->src != req->dst) { + + ptr = kmalloc(inputlen, (req_info->areq->flags & + CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC); + if (!ptr) + return -ENOMEM; + + status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr, + inputlen); + if (status != inputlen) { + status = -EINVAL; + goto error_free; + } + status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr, + inputlen); + if (status != inputlen) { + status = -EINVAL; + goto error_free; + } + kfree(ptr); + } + + if (enc) { + /* + * In an encryption scenario hmac needs + * to be appended after payload + */ + dst = req->dst; + offset = inputlen; + while (offset >= dst->length) { + offset -= dst->length; + dst = sg_next(dst); + if (!dst) + return -ENOENT; + } + + update_output_data(req_info, dst, offset, mac_len, &argcnt); + } else { + /* + * In a decryption scenario calculated hmac for received + * payload needs to be compare with hmac received + */ + status = sg_copy_buffer(req->src, sg_nents(req->src), + rctx->fctx.hmac.s.hmac_recv, mac_len, + inputlen, true); + if (status != mac_len) + return -EINVAL; + + req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc; + req_info->out[argcnt].size = mac_len; + argcnt++; + } + + req_info->out_cnt = argcnt; + return 0; + +error_free: + kfree(ptr); + return status; +} + +static int aead_do_fallback(struct aead_request *req, bool is_enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(aead); + int ret; + + if (ctx->fbk_cipher) { + /* Store the cipher tfm and then use the fallback tfm */ + aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher); + aead_request_set_callback(&rctx->fbk_req, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(&rctx->fbk_req, req->src, + req->dst, req->cryptlen, req->iv); + ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) : + crypto_aead_decrypt(&rctx->fbk_req); + } else { + ret = -EINVAL; + } + + return ret; +} + +static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct pci_dev *pdev; + int status, cpu_num; + + /* Clear control words */ + rctx->ctrl_word.flags = 0; + rctx->fctx.enc.enc_ctrl.u = 0; + + req_info->callback = otx2_cpt_aead_callback; + req_info->areq = &req->base; + req_info->req_type = reg_type; + req_info->is_enc = enc; + req_info->is_trunc_hmac = false; + + switch (reg_type) { + case OTX2_CPT_AEAD_ENC_DEC_REQ: + status = create_aead_input_list(req, enc); + if (status) + return status; + create_aead_output_list(req, enc, crypto_aead_authsize(tfm)); + break; + + case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ: + create_aead_null_input_list(req, enc, + crypto_aead_authsize(tfm)); + status = create_aead_null_output_list(req, enc, + crypto_aead_authsize(tfm)); + if (status) + return status; + break; + + default: + return -EINVAL; + } + if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len)) + return -EINVAL; + + if (!req_info->req.param2 || + (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) || + (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE)) + return aead_do_fallback(req, enc); + + status = get_se_device(&pdev, &cpu_num); + if (status) + return status; + + req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); + + /* + * We perform an asynchronous send and once + * the request is completed the driver would + * intimate through registered call back functions + */ + return otx2_cpt_do_request(pdev, req_info, cpu_num); +} + +static int otx2_cpt_aead_encrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true); +} + +static int otx2_cpt_aead_decrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false); +} + +static int otx2_cpt_aead_null_encrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true); +} + +static int otx2_cpt_aead_null_decrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false); +} + +static struct skcipher_alg otx2_cpt_skciphers[] = { { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "cpt_xts_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .setkey = otx2_cpt_skcipher_xts_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cpt_cbc_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = otx2_cpt_skcipher_cbc_aes_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "cpt_ecb_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .ivsize = 0, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = otx2_cpt_skcipher_ecb_aes_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cpt_cbc_des3_ede", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = otx2_cpt_skcipher_cbc_des3_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "cpt_ecb_des3_ede", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = 0, + .setkey = otx2_cpt_skcipher_ecb_des3_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +} }; + +static struct aead_alg otx2_cpt_aeads[] = { { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha1_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha1_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha256_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha256_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha384_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha384_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha512_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha512_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha1),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha1_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha1_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA1_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha256),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha256_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha256_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA256_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha384),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha384_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha384_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA384_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha512),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha512_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha512_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA512_DIGEST_SIZE, +}, { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "cpt_rfc4106_gcm_aes", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_gcm_aes_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_gcm_aes_setkey, + .setauthsize = otx2_cpt_aead_gcm_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_GCM_IV_SIZE, + .maxauthsize = AES_GCM_ICV_SIZE, +} }; + +static inline int cpt_register_algs(void) +{ + int i, err = 0; + + if (!IS_ENABLED(CONFIG_DM_CRYPT)) { + for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++) + otx2_cpt_skciphers[i].base.cra_flags &= + ~CRYPTO_ALG_DEAD; + + err = crypto_register_skciphers(otx2_cpt_skciphers, + ARRAY_SIZE(otx2_cpt_skciphers)); + if (err) + return err; + } + + for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++) + otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; + + err = crypto_register_aeads(otx2_cpt_aeads, + ARRAY_SIZE(otx2_cpt_aeads)); + if (err) { + crypto_unregister_skciphers(otx2_cpt_skciphers, + ARRAY_SIZE(otx2_cpt_skciphers)); + return err; + } + + return 0; +} + +static inline void cpt_unregister_algs(void) +{ + crypto_unregister_skciphers(otx2_cpt_skciphers, + ARRAY_SIZE(otx2_cpt_skciphers)); + crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads)); +} + +static int compare_func(const void *lptr, const void *rptr) +{ + const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr; + const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr; + + if (ldesc->dev->devfn < rdesc->dev->devfn) + return -1; + if (ldesc->dev->devfn > rdesc->dev->devfn) + return 1; + return 0; +} + +static void swap_func(void *lptr, void *rptr, int size) +{ + struct cpt_device_desc *ldesc = lptr; + struct cpt_device_desc *rdesc = rptr; + struct cpt_device_desc desc; + + desc = *ldesc; + *ldesc = *rdesc; + *rdesc = desc; +} + +int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, + int num_queues, int num_devices) +{ + int ret = 0; + int count; + + mutex_lock(&mutex); + count = atomic_read(&se_devices.count); + if (count >= OTX2_CPT_MAX_LFS_NUM) { + dev_err(&pdev->dev, "No space to add a new device\n"); + ret = -ENOSPC; + goto unlock; + } + se_devices.desc[count].num_queues = num_queues; + se_devices.desc[count++].dev = pdev; + atomic_inc(&se_devices.count); + + if (atomic_read(&se_devices.count) == num_devices && + is_crypto_registered == false) { + if (cpt_register_algs()) { + dev_err(&pdev->dev, + "Error in registering crypto algorithms\n"); + ret = -EINVAL; + goto unlock; + } + try_module_get(mod); + is_crypto_registered = true; + } + sort(se_devices.desc, count, sizeof(struct cpt_device_desc), + compare_func, swap_func); + +unlock: + mutex_unlock(&mutex); + return ret; +} + +void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod) +{ + struct cpt_device_table *dev_tbl; + bool dev_found = false; + int i, j, count; + + mutex_lock(&mutex); + + dev_tbl = &se_devices; + count = atomic_read(&dev_tbl->count); + for (i = 0; i < count; i++) { + if (pdev == dev_tbl->desc[i].dev) { + for (j = i; j < count-1; j++) + dev_tbl->desc[j] = dev_tbl->desc[j+1]; + dev_found = true; + break; + } + } + + if (!dev_found) { + dev_err(&pdev->dev, "%s device not found\n", __func__); + goto unlock; + } + if (atomic_dec_and_test(&se_devices.count)) { + cpt_unregister_algs(); + module_put(mod); + is_crypto_registered = false; + } + +unlock: + mutex_unlock(&mutex); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h new file mode 100644 index 000000000000..f04184bd1744 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_ALGS_H +#define __OTX2_CPT_ALGS_H + +#include +#include +#include +#include "otx2_cpt_common.h" + +#define OTX2_CPT_MAX_ENC_KEY_SIZE 32 +#define OTX2_CPT_MAX_HASH_KEY_SIZE 64 +#define OTX2_CPT_MAX_KEY_SIZE (OTX2_CPT_MAX_ENC_KEY_SIZE + \ + OTX2_CPT_MAX_HASH_KEY_SIZE) +enum otx2_cpt_request_type { + OTX2_CPT_ENC_DEC_REQ = 0x1, + OTX2_CPT_AEAD_ENC_DEC_REQ = 0x2, + OTX2_CPT_AEAD_ENC_DEC_NULL_REQ = 0x3, + OTX2_CPT_PASSTHROUGH_REQ = 0x4 +}; + +enum otx2_cpt_major_opcodes { + OTX2_CPT_MAJOR_OP_MISC = 0x01, + OTX2_CPT_MAJOR_OP_FC = 0x33, + OTX2_CPT_MAJOR_OP_HMAC = 0x35, +}; + +enum otx2_cpt_cipher_type { + OTX2_CPT_CIPHER_NULL = 0x0, + OTX2_CPT_DES3_CBC = 0x1, + OTX2_CPT_DES3_ECB = 0x2, + OTX2_CPT_AES_CBC = 0x3, + OTX2_CPT_AES_ECB = 0x4, + OTX2_CPT_AES_CFB = 0x5, + OTX2_CPT_AES_CTR = 0x6, + OTX2_CPT_AES_GCM = 0x7, + OTX2_CPT_AES_XTS = 0x8 +}; + +enum otx2_cpt_mac_type { + OTX2_CPT_MAC_NULL = 0x0, + OTX2_CPT_MD5 = 0x1, + OTX2_CPT_SHA1 = 0x2, + OTX2_CPT_SHA224 = 0x3, + OTX2_CPT_SHA256 = 0x4, + OTX2_CPT_SHA384 = 0x5, + OTX2_CPT_SHA512 = 0x6, + OTX2_CPT_GMAC = 0x7 +}; + +enum otx2_cpt_aes_key_len { + OTX2_CPT_AES_128_BIT = 0x1, + OTX2_CPT_AES_192_BIT = 0x2, + OTX2_CPT_AES_256_BIT = 0x3 +}; + +union otx2_cpt_encr_ctrl { + u64 u; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 enc_cipher:4; + u64 reserved_59:1; + u64 aes_key:2; + u64 iv_source:1; + u64 mac_type:4; + u64 reserved_49_51:3; + u64 auth_input_type:1; + u64 mac_len:8; + u64 reserved_32_39:8; + u64 encr_offset:16; + u64 iv_offset:8; + u64 auth_offset:8; +#else + u64 auth_offset:8; + u64 iv_offset:8; + u64 encr_offset:16; + u64 reserved_32_39:8; + u64 mac_len:8; + u64 auth_input_type:1; + u64 reserved_49_51:3; + u64 mac_type:4; + u64 iv_source:1; + u64 aes_key:2; + u64 reserved_59:1; + u64 enc_cipher:4; +#endif + } e; +}; + +struct otx2_cpt_cipher { + const char *name; + u8 value; +}; + +struct otx2_cpt_fc_enc_ctx { + union otx2_cpt_encr_ctrl enc_ctrl; + u8 encr_key[32]; + u8 encr_iv[16]; +}; + +union otx2_cpt_fc_hmac_ctx { + struct { + u8 ipad[64]; + u8 opad[64]; + } e; + struct { + u8 hmac_calc[64]; /* HMAC calculated */ + u8 hmac_recv[64]; /* HMAC received */ + } s; +}; + +struct otx2_cpt_fc_ctx { + struct otx2_cpt_fc_enc_ctx enc; + union otx2_cpt_fc_hmac_ctx hmac; +}; + +struct otx2_cpt_enc_ctx { + u32 key_len; + u8 enc_key[OTX2_CPT_MAX_KEY_SIZE]; + u8 cipher_type; + u8 key_type; + u8 enc_align_len; + struct crypto_skcipher *fbk_cipher; +}; + +union otx2_cpt_offset_ctrl { + u64 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved:32; + u64 enc_data_offset:16; + u64 iv_offset:8; + u64 auth_offset:8; +#else + u64 auth_offset:8; + u64 iv_offset:8; + u64 enc_data_offset:16; + u64 reserved:32; +#endif + } e; +}; + +struct otx2_cpt_req_ctx { + struct otx2_cpt_req_info cpt_req; + union otx2_cpt_offset_ctrl ctrl_word; + struct otx2_cpt_fc_ctx fctx; + union { + struct skcipher_request sk_fbk_req; + struct aead_request fbk_req; + }; +}; + +struct otx2_cpt_sdesc { + struct shash_desc shash; +}; + +struct otx2_cpt_aead_ctx { + u8 key[OTX2_CPT_MAX_KEY_SIZE]; + struct crypto_shash *hashalg; + struct otx2_cpt_sdesc *sdesc; + struct crypto_aead *fbk_cipher; + u8 *ipad; + u8 *opad; + u32 enc_key_len; + u32 auth_key_len; + u8 cipher_type; + u8 mac_type; + u8 key_type; + u8 is_trunc_hmac; + u8 enc_align_len; +}; +int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, + int num_queues, int num_devices); +void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod); + +#endif /* __OTX2_CPT_ALGS_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c new file mode 100644 index 000000000000..47f378731024 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptvf.h" +#include "otx2_cptlf.h" +#include "otx2_cptvf_algs.h" +#include + +#define OTX2_CPTVF_DRV_NAME "octeontx2-cptvf" + +static void cptvf_enable_pfvf_mbox_intrs(struct otx2_cptvf_dev *cptvf) +{ + /* Clear interrupt if any */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, OTX2_RVU_VF_INT, + 0x1ULL); + + /* Enable PF-VF interrupt */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT_ENA_W1S, 0x1ULL); +} + +static void cptvf_disable_pfvf_mbox_intrs(struct otx2_cptvf_dev *cptvf) +{ + /* Disable PF-VF interrupt */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT_ENA_W1C, 0x1ULL); + + /* Clear interrupt if any */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, OTX2_RVU_VF_INT, + 0x1ULL); +} + +static int cptvf_register_interrupts(struct otx2_cptvf_dev *cptvf) +{ + int ret, irq; + int num_vec; + + num_vec = pci_msix_vec_count(cptvf->pdev); + if (num_vec <= 0) + return -EINVAL; + + /* Enable MSI-X */ + ret = pci_alloc_irq_vectors(cptvf->pdev, num_vec, num_vec, + PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(&cptvf->pdev->dev, + "Request for %d msix vectors failed\n", num_vec); + return ret; + } + irq = pci_irq_vector(cptvf->pdev, OTX2_CPT_VF_INT_VEC_E_MBOX); + /* Register VF<=>PF mailbox interrupt handler */ + ret = devm_request_irq(&cptvf->pdev->dev, irq, + otx2_cptvf_pfvf_mbox_intr, 0, + "CPTPFVF Mbox", cptvf); + if (ret) + return ret; + /* Enable PF-VF mailbox interrupts */ + cptvf_enable_pfvf_mbox_intrs(cptvf); + + ret = otx2_cpt_send_ready_msg(&cptvf->pfvf_mbox, cptvf->pdev); + if (ret) { + dev_warn(&cptvf->pdev->dev, + "PF not responding to mailbox, deferring probe\n"); + cptvf_disable_pfvf_mbox_intrs(cptvf); + return -EPROBE_DEFER; + } + return 0; +} + +static int cptvf_pfvf_mbox_init(struct otx2_cptvf_dev *cptvf) +{ + int ret; + + cptvf->pfvf_mbox_wq = alloc_workqueue("cpt_pfvf_mailbox", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, 1); + if (!cptvf->pfvf_mbox_wq) + return -ENOMEM; + + ret = otx2_mbox_init(&cptvf->pfvf_mbox, cptvf->pfvf_mbox_base, + cptvf->pdev, cptvf->reg_base, MBOX_DIR_VFPF, 1); + if (ret) + goto free_wqe; + + INIT_WORK(&cptvf->pfvf_mbox_work, otx2_cptvf_pfvf_mbox_handler); + return 0; + +free_wqe: + destroy_workqueue(cptvf->pfvf_mbox_wq); + return ret; +} + +static void cptvf_pfvf_mbox_destroy(struct otx2_cptvf_dev *cptvf) +{ + destroy_workqueue(cptvf->pfvf_mbox_wq); + otx2_mbox_destroy(&cptvf->pfvf_mbox); +} + +static void cptlf_work_handler(unsigned long data) +{ + otx2_cpt_post_process((struct otx2_cptlf_wqe *) data); +} + +static void cleanup_tasklet_work(struct otx2_cptlfs_info *lfs) +{ + int i; + + for (i = 0; i < lfs->lfs_num; i++) { + if (!lfs->lf[i].wqe) + continue; + + tasklet_kill(&lfs->lf[i].wqe->work); + kfree(lfs->lf[i].wqe); + lfs->lf[i].wqe = NULL; + } +} + +static int init_tasklet_work(struct otx2_cptlfs_info *lfs) +{ + struct otx2_cptlf_wqe *wqe; + int i, ret = 0; + + for (i = 0; i < lfs->lfs_num; i++) { + wqe = kzalloc(sizeof(struct otx2_cptlf_wqe), GFP_KERNEL); + if (!wqe) { + ret = -ENOMEM; + goto cleanup_tasklet; + } + + tasklet_init(&wqe->work, cptlf_work_handler, (u64) wqe); + wqe->lfs = lfs; + wqe->lf_num = i; + lfs->lf[i].wqe = wqe; + } + return 0; + +cleanup_tasklet: + cleanup_tasklet_work(lfs); + return ret; +} + +static void free_pending_queues(struct otx2_cptlfs_info *lfs) +{ + int i; + + for (i = 0; i < lfs->lfs_num; i++) { + kfree(lfs->lf[i].pqueue.head); + lfs->lf[i].pqueue.head = NULL; + } +} + +static int alloc_pending_queues(struct otx2_cptlfs_info *lfs) +{ + int size, ret, i; + + if (!lfs->lfs_num) + return -EINVAL; + + for (i = 0; i < lfs->lfs_num; i++) { + lfs->lf[i].pqueue.qlen = OTX2_CPT_INST_QLEN_MSGS; + size = lfs->lf[i].pqueue.qlen * + sizeof(struct otx2_cpt_pending_entry); + + lfs->lf[i].pqueue.head = kzalloc(size, GFP_KERNEL); + if (!lfs->lf[i].pqueue.head) { + ret = -ENOMEM; + goto error; + } + + /* Initialize spin lock */ + spin_lock_init(&lfs->lf[i].pqueue.lock); + } + return 0; + +error: + free_pending_queues(lfs); + return ret; +} + +static void lf_sw_cleanup(struct otx2_cptlfs_info *lfs) +{ + cleanup_tasklet_work(lfs); + free_pending_queues(lfs); +} + +static int lf_sw_init(struct otx2_cptlfs_info *lfs) +{ + int ret; + + ret = alloc_pending_queues(lfs); + if (ret) { + dev_err(&lfs->pdev->dev, + "Allocating pending queues failed\n"); + return ret; + } + ret = init_tasklet_work(lfs); + if (ret) { + dev_err(&lfs->pdev->dev, + "Tasklet work init failed\n"); + goto pending_queues_free; + } + return 0; + +pending_queues_free: + free_pending_queues(lfs); + return ret; +} + +static void cptvf_lf_shutdown(struct otx2_cptlfs_info *lfs) +{ + atomic_set(&lfs->state, OTX2_CPTLF_IN_RESET); + + /* Remove interrupts affinity */ + otx2_cptlf_free_irqs_affinity(lfs); + /* Disable instruction queue */ + otx2_cptlf_disable_iqueues(lfs); + /* Unregister crypto algorithms */ + otx2_cpt_crypto_exit(lfs->pdev, THIS_MODULE); + /* Unregister LFs interrupts */ + otx2_cptlf_unregister_interrupts(lfs); + /* Cleanup LFs software side */ + lf_sw_cleanup(lfs); + /* Send request to detach LFs */ + otx2_cpt_detach_rsrcs_msg(lfs); +} + +static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf) +{ + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + struct device *dev = &cptvf->pdev->dev; + int ret, lfs_num; + u8 eng_grp_msk; + + /* Get engine group number for symmetric crypto */ + cptvf->lfs.kcrypto_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP; + ret = otx2_cptvf_send_eng_grp_num_msg(cptvf, OTX2_CPT_SE_TYPES); + if (ret) + return ret; + + if (cptvf->lfs.kcrypto_eng_grp_num == OTX2_CPT_INVALID_CRYPTO_ENG_GRP) { + dev_err(dev, "Engine group for kernel crypto not available\n"); + ret = -ENOENT; + return ret; + } + eng_grp_msk = 1 << cptvf->lfs.kcrypto_eng_grp_num; + + ret = otx2_cptvf_send_kvf_limits_msg(cptvf); + if (ret) + return ret; + + lfs->reg_base = cptvf->reg_base; + lfs->pdev = cptvf->pdev; + lfs->mbox = &cptvf->pfvf_mbox; + + lfs_num = cptvf->lfs.kvf_limits ? cptvf->lfs.kvf_limits : + num_online_cpus(); + ret = otx2_cptlf_init(lfs, eng_grp_msk, OTX2_CPT_QUEUE_HI_PRIO, + lfs_num); + if (ret) + return ret; + + /* Get msix offsets for attached LFs */ + ret = otx2_cpt_msix_offset_msg(lfs); + if (ret) + goto cleanup_lf; + + /* Initialize LFs software side */ + ret = lf_sw_init(lfs); + if (ret) + goto cleanup_lf; + + /* Register LFs interrupts */ + ret = otx2_cptlf_register_interrupts(lfs); + if (ret) + goto cleanup_lf_sw; + + /* Set interrupts affinity */ + ret = otx2_cptlf_set_irqs_affinity(lfs); + if (ret) + goto unregister_intr; + + atomic_set(&lfs->state, OTX2_CPTLF_STARTED); + /* Register crypto algorithms */ + ret = otx2_cpt_crypto_init(lfs->pdev, THIS_MODULE, lfs_num, 1); + if (ret) { + dev_err(&lfs->pdev->dev, "algorithms registration failed\n"); + goto disable_irqs; + } + return 0; + +disable_irqs: + otx2_cptlf_free_irqs_affinity(lfs); +unregister_intr: + otx2_cptlf_unregister_interrupts(lfs); +cleanup_lf_sw: + lf_sw_cleanup(lfs); +cleanup_lf: + otx2_cptlf_shutdown(lfs); + + return ret; +} + +static int otx2_cptvf_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + resource_size_t offset, size; + struct otx2_cptvf_dev *cptvf; + int ret; + + cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); + if (!cptvf) + return -ENOMEM; + + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(dev, "Failed to enable PCI device\n"); + goto clear_drvdata; + } + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(dev, "Unable to get usable DMA configuration\n"); + goto clear_drvdata; + } + /* Map VF's configuration registers */ + ret = pcim_iomap_regions_request_all(pdev, 1 << PCI_PF_REG_BAR_NUM, + OTX2_CPTVF_DRV_NAME); + if (ret) { + dev_err(dev, "Couldn't get PCI resources 0x%x\n", ret); + goto clear_drvdata; + } + pci_set_master(pdev); + pci_set_drvdata(pdev, cptvf); + cptvf->pdev = pdev; + + cptvf->reg_base = pcim_iomap_table(pdev)[PCI_PF_REG_BAR_NUM]; + + offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); + size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); + /* Map PF-VF mailbox memory */ + cptvf->pfvf_mbox_base = devm_ioremap_wc(dev, offset, size); + if (!cptvf->pfvf_mbox_base) { + dev_err(&pdev->dev, "Unable to map BAR4\n"); + ret = -ENODEV; + goto clear_drvdata; + } + /* Initialize PF<=>VF mailbox */ + ret = cptvf_pfvf_mbox_init(cptvf); + if (ret) + goto clear_drvdata; + + /* Register interrupts */ + ret = cptvf_register_interrupts(cptvf); + if (ret) + goto destroy_pfvf_mbox; + + /* Initialize CPT LFs */ + ret = cptvf_lf_init(cptvf); + if (ret) + goto unregister_interrupts; + + return 0; + +unregister_interrupts: + cptvf_disable_pfvf_mbox_intrs(cptvf); +destroy_pfvf_mbox: + cptvf_pfvf_mbox_destroy(cptvf); +clear_drvdata: + pci_set_drvdata(pdev, NULL); + + return ret; +} + +static void otx2_cptvf_remove(struct pci_dev *pdev) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + + if (!cptvf) { + dev_err(&pdev->dev, "Invalid CPT VF device.\n"); + return; + } + cptvf_lf_shutdown(&cptvf->lfs); + /* Disable PF-VF mailbox interrupt */ + cptvf_disable_pfvf_mbox_intrs(cptvf); + /* Destroy PF-VF mbox */ + cptvf_pfvf_mbox_destroy(cptvf); + pci_set_drvdata(pdev, NULL); +} + +/* Supported devices */ +static const struct pci_device_id otx2_cptvf_id_table[] = { + {PCI_VDEVICE(CAVIUM, OTX2_CPT_PCI_VF_DEVICE_ID), 0}, + { 0, } /* end of table */ +}; + +static struct pci_driver otx2_cptvf_pci_driver = { + .name = OTX2_CPTVF_DRV_NAME, + .id_table = otx2_cptvf_id_table, + .probe = otx2_cptvf_probe, + .remove = otx2_cptvf_remove, +}; + +module_pci_driver(otx2_cptvf_pci_driver); + +MODULE_AUTHOR("Marvell"); +MODULE_DESCRIPTION("Marvell OcteonTX2 CPT Virtual Function Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_DEVICE_TABLE(pci, otx2_cptvf_id_table); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c new file mode 100644 index 000000000000..5d73b711cba6 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptvf.h" +#include + +irqreturn_t otx2_cptvf_pfvf_mbox_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptvf_dev *cptvf = arg; + u64 intr; + + /* Read the interrupt bits */ + intr = otx2_cpt_read64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT); + + if (intr & 0x1ULL) { + /* Schedule work queue function to process the MBOX request */ + queue_work(cptvf->pfvf_mbox_wq, &cptvf->pfvf_mbox_work); + /* Clear and ack the interrupt */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT, 0x1ULL); + } + return IRQ_HANDLED; +} + +static void process_pfvf_mbox_mbox_msg(struct otx2_cptvf_dev *cptvf, + struct mbox_msghdr *msg) +{ + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + struct otx2_cpt_kvf_limits_rsp *rsp_limits; + struct otx2_cpt_egrp_num_rsp *rsp_grp; + struct cpt_rd_wr_reg_msg *rsp_reg; + struct msix_offset_rsp *rsp_msix; + int i; + + if (msg->id >= MBOX_MSG_MAX) { + dev_err(&cptvf->pdev->dev, + "MBOX msg with unknown ID %d\n", msg->id); + return; + } + if (msg->sig != OTX2_MBOX_RSP_SIG) { + dev_err(&cptvf->pdev->dev, + "MBOX msg with wrong signature %x, ID %d\n", + msg->sig, msg->id); + return; + } + switch (msg->id) { + case MBOX_MSG_READY: + cptvf->vf_id = ((msg->pcifunc >> RVU_PFVF_FUNC_SHIFT) + & RVU_PFVF_FUNC_MASK) - 1; + break; + case MBOX_MSG_ATTACH_RESOURCES: + /* Check if resources were successfully attached */ + if (!msg->rc) + lfs->are_lfs_attached = 1; + break; + case MBOX_MSG_DETACH_RESOURCES: + /* Check if resources were successfully detached */ + if (!msg->rc) + lfs->are_lfs_attached = 0; + break; + case MBOX_MSG_MSIX_OFFSET: + rsp_msix = (struct msix_offset_rsp *) msg; + for (i = 0; i < rsp_msix->cptlfs; i++) + lfs->lf[i].msix_offset = rsp_msix->cptlf_msixoff[i]; + break; + case MBOX_MSG_CPT_RD_WR_REGISTER: + rsp_reg = (struct cpt_rd_wr_reg_msg *) msg; + if (msg->rc) { + dev_err(&cptvf->pdev->dev, + "Reg %llx rd/wr(%d) failed %d\n", + rsp_reg->reg_offset, rsp_reg->is_write, + msg->rc); + return; + } + if (!rsp_reg->is_write) + *rsp_reg->ret_val = rsp_reg->val; + break; + case MBOX_MSG_GET_ENG_GRP_NUM: + rsp_grp = (struct otx2_cpt_egrp_num_rsp *) msg; + cptvf->lfs.kcrypto_eng_grp_num = rsp_grp->eng_grp_num; + break; + case MBOX_MSG_GET_KVF_LIMITS: + rsp_limits = (struct otx2_cpt_kvf_limits_rsp *) msg; + cptvf->lfs.kvf_limits = rsp_limits->kvf_limits; + break; + default: + dev_err(&cptvf->pdev->dev, "Unsupported msg %d received.\n", + msg->id); + break; + } +} + +void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work) +{ + struct otx2_cptvf_dev *cptvf; + struct otx2_mbox *pfvf_mbox; + struct otx2_mbox_dev *mdev; + struct mbox_hdr *rsp_hdr; + struct mbox_msghdr *msg; + int offset, i; + + /* sync with mbox memory region */ + smp_rmb(); + + cptvf = container_of(work, struct otx2_cptvf_dev, pfvf_mbox_work); + pfvf_mbox = &cptvf->pfvf_mbox; + mdev = &pfvf_mbox->dev[0]; + rsp_hdr = (struct mbox_hdr *)(mdev->mbase + pfvf_mbox->rx_start); + if (rsp_hdr->num_msgs == 0) + return; + offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN); + + for (i = 0; i < rsp_hdr->num_msgs; i++) { + msg = (struct mbox_msghdr *)(mdev->mbase + pfvf_mbox->rx_start + + offset); + process_pfvf_mbox_mbox_msg(cptvf, msg); + offset = msg->next_msgoff; + mdev->msgs_acked++; + } + otx2_mbox_reset(pfvf_mbox, 0); +} + +int otx2_cptvf_send_eng_grp_num_msg(struct otx2_cptvf_dev *cptvf, int eng_type) +{ + struct otx2_mbox *mbox = &cptvf->pfvf_mbox; + struct pci_dev *pdev = cptvf->pdev; + struct otx2_cpt_egrp_num_msg *req; + + req = (struct otx2_cpt_egrp_num_msg *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct otx2_cpt_egrp_num_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->hdr.id = MBOX_MSG_GET_ENG_GRP_NUM; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->vf_id, 0); + req->eng_type = eng_type; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf) +{ + struct otx2_mbox *mbox = &cptvf->pfvf_mbox; + struct pci_dev *pdev = cptvf->pdev; + struct mbox_msghdr *req; + int ret; + + req = (struct mbox_msghdr *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct otx2_cpt_kvf_limits_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->id = MBOX_MSG_GET_KVF_LIMITS; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->vf_id, 0); + + ret = otx2_cpt_send_mbox_msg(mbox, pdev); + + return ret; +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c new file mode 100644 index 000000000000..d5c1c1b7c7e4 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cptvf.h" +#include "otx2_cpt_common.h" + +/* SG list header size in bytes */ +#define SG_LIST_HDR_SIZE 8 + +/* Default timeout when waiting for free pending entry in us */ +#define CPT_PENTRY_TIMEOUT 1000 +#define CPT_PENTRY_STEP 50 + +/* Default threshold for stopping and resuming sender requests */ +#define CPT_IQ_STOP_MARGIN 128 +#define CPT_IQ_RESUME_MARGIN 512 + +/* Default command timeout in seconds */ +#define CPT_COMMAND_TIMEOUT 4 +#define CPT_TIME_IN_RESET_COUNT 5 + +static void otx2_cpt_dump_sg_list(struct pci_dev *pdev, + struct otx2_cpt_req_info *req) +{ + int i; + + pr_debug("Gather list size %d\n", req->in_cnt); + for (i = 0; i < req->in_cnt; i++) { + pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, + req->in[i].size, req->in[i].vptr, + (void *) req->in[i].dma_addr); + pr_debug("Buffer hexdump (%d bytes)\n", + req->in[i].size); + print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, + req->in[i].vptr, req->in[i].size, false); + } + pr_debug("Scatter list size %d\n", req->out_cnt); + for (i = 0; i < req->out_cnt; i++) { + pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, + req->out[i].size, req->out[i].vptr, + (void *) req->out[i].dma_addr); + pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size); + print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, + req->out[i].vptr, req->out[i].size, false); + } +} + +static inline struct otx2_cpt_pending_entry *get_free_pending_entry( + struct otx2_cpt_pending_queue *q, + int qlen) +{ + struct otx2_cpt_pending_entry *ent = NULL; + + ent = &q->head[q->rear]; + if (unlikely(ent->busy)) + return NULL; + + q->rear++; + if (unlikely(q->rear == qlen)) + q->rear = 0; + + return ent; +} + +static inline u32 modulo_inc(u32 index, u32 length, u32 inc) +{ + if (WARN_ON(inc > length)) + inc = length; + + index += inc; + if (unlikely(index >= length)) + index -= length; + + return index; +} + +static inline void free_pentry(struct otx2_cpt_pending_entry *pentry) +{ + pentry->completion_addr = NULL; + pentry->info = NULL; + pentry->callback = NULL; + pentry->areq = NULL; + pentry->resume_sender = false; + pentry->busy = false; +} + +static inline int setup_sgio_components(struct pci_dev *pdev, + struct otx2_cpt_buf_ptr *list, + int buf_count, u8 *buffer) +{ + struct otx2_cpt_sglist_component *sg_ptr = NULL; + int ret = 0, i, j; + int components; + + if (unlikely(!list)) { + dev_err(&pdev->dev, "Input list pointer is NULL\n"); + return -EFAULT; + } + + for (i = 0; i < buf_count; i++) { + if (unlikely(!list[i].vptr)) + continue; + list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, + list[i].size, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { + dev_err(&pdev->dev, "Dma mapping failed\n"); + ret = -EIO; + goto sg_cleanup; + } + } + components = buf_count / 4; + sg_ptr = (struct otx2_cpt_sglist_component *)buffer; + for (i = 0; i < components; i++) { + sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size); + sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size); + sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size); + sg_ptr->len3 = cpu_to_be16(list[i * 4 + 3].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); + sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); + sg_ptr++; + } + components = buf_count % 4; + + switch (components) { + case 3: + sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size); + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); + fallthrough; + case 2: + sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size); + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); + fallthrough; + case 1: + sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); + break; + default: + break; + } + return ret; + +sg_cleanup: + for (j = 0; j < i; j++) { + if (list[j].dma_addr) { + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); + } + + list[j].dma_addr = 0; + } + return ret; +} + +static inline struct otx2_cpt_inst_info *info_create(struct pci_dev *pdev, + struct otx2_cpt_req_info *req, + gfp_t gfp) +{ + int align = OTX2_CPT_DMA_MINALIGN; + struct otx2_cpt_inst_info *info; + u32 dlen, align_dlen, info_len; + u16 g_sz_bytes, s_sz_bytes; + u32 total_mem_len; + + if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT || + req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) { + dev_err(&pdev->dev, "Error too many sg components\n"); + return NULL; + } + + g_sz_bytes = ((req->in_cnt + 3) / 4) * + sizeof(struct otx2_cpt_sglist_component); + s_sz_bytes = ((req->out_cnt + 3) / 4) * + sizeof(struct otx2_cpt_sglist_component); + + dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; + align_dlen = ALIGN(dlen, align); + info_len = ALIGN(sizeof(*info), align); + total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s); + + info = kzalloc(total_mem_len, gfp); + if (unlikely(!info)) + return NULL; + + info->dlen = dlen; + info->in_buffer = (u8 *)info + info_len; + + ((u16 *)info->in_buffer)[0] = req->out_cnt; + ((u16 *)info->in_buffer)[1] = req->in_cnt; + ((u16 *)info->in_buffer)[2] = 0; + ((u16 *)info->in_buffer)[3] = 0; + cpu_to_be64s((u64 *)info->in_buffer); + + /* Setup gather (input) components */ + if (setup_sgio_components(pdev, req->in, req->in_cnt, + &info->in_buffer[8])) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + goto destroy_info; + } + + if (setup_sgio_components(pdev, req->out, req->out_cnt, + &info->in_buffer[8 + g_sz_bytes])) { + dev_err(&pdev->dev, "Failed to setup scatter list\n"); + goto destroy_info; + } + + info->dma_len = total_mem_len - info_len; + info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, + info->dma_len, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { + dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); + goto destroy_info; + } + /* + * Get buffer for union otx2_cpt_res_s response + * structure and its physical address + */ + info->completion_addr = info->in_buffer + align_dlen; + info->comp_baddr = info->dptr_baddr + align_dlen; + + return info; + +destroy_info: + otx2_cpt_info_destroy(pdev, info); + return NULL; +} + +static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + struct otx2_cpt_pending_queue *pqueue, + struct otx2_cptlf_info *lf) +{ + struct otx2_cptvf_request *cpt_req = &req->req; + struct otx2_cpt_pending_entry *pentry = NULL; + union otx2_cpt_ctrl_info *ctrl = &req->ctrl; + struct otx2_cpt_inst_info *info = NULL; + union otx2_cpt_res_s *result = NULL; + struct otx2_cpt_iq_command iq_cmd; + union otx2_cpt_inst_s cptinst; + int retry, ret = 0; + u8 resume_sender; + gfp_t gfp; + + gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : + GFP_ATOMIC; + if (unlikely(!otx2_cptlf_started(lf->lfs))) + return -ENODEV; + + info = info_create(pdev, req, gfp); + if (unlikely(!info)) { + dev_err(&pdev->dev, "Setting up cpt inst info failed"); + return -ENOMEM; + } + cpt_req->dlen = info->dlen; + + result = info->completion_addr; + result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; + + spin_lock_bh(&pqueue->lock); + pentry = get_free_pending_entry(pqueue, pqueue->qlen); + retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP; + while (unlikely(!pentry) && retry--) { + spin_unlock_bh(&pqueue->lock); + udelay(CPT_PENTRY_STEP); + spin_lock_bh(&pqueue->lock); + pentry = get_free_pending_entry(pqueue, pqueue->qlen); + } + + if (unlikely(!pentry)) { + ret = -ENOSPC; + goto destroy_info; + } + + /* + * Check if we are close to filling in entire pending queue, + * if so then tell the sender to stop/sleep by returning -EBUSY + * We do it only for context which can sleep (GFP_KERNEL) + */ + if (gfp == GFP_KERNEL && + pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) { + pentry->resume_sender = true; + } else + pentry->resume_sender = false; + resume_sender = pentry->resume_sender; + pqueue->pending_count++; + + pentry->completion_addr = info->completion_addr; + pentry->info = info; + pentry->callback = req->callback; + pentry->areq = req->areq; + pentry->busy = true; + info->pentry = pentry; + info->time_in = jiffies; + info->req = req; + + /* Fill in the command */ + iq_cmd.cmd.u = 0; + iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); + iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); + iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); + iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); + + /* 64-bit swap for microcode data reads, not needed for addresses*/ + cpu_to_be64s(&iq_cmd.cmd.u); + iq_cmd.dptr = info->dptr_baddr; + iq_cmd.rptr = 0; + iq_cmd.cptr.u = 0; + iq_cmd.cptr.s.grp = ctrl->s.grp; + + /* Fill in the CPT_INST_S type command for HW interpretation */ + otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr); + + /* Print debug info if enabled */ + otx2_cpt_dump_sg_list(pdev, req); + pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE); + print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false); + pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen); + print_hex_dump_debug("", 0, 16, 1, info->in_buffer, + cpt_req->dlen, false); + + /* Send CPT command */ + otx2_cpt_send_cmd(&cptinst, 1, lf); + + /* + * We allocate and prepare pending queue entry in critical section + * together with submitting CPT instruction to CPT instruction queue + * to make sure that order of CPT requests is the same in both + * pending and instruction queues + */ + spin_unlock_bh(&pqueue->lock); + + ret = resume_sender ? -EBUSY : -EINPROGRESS; + return ret; + +destroy_info: + spin_unlock_bh(&pqueue->lock); + otx2_cpt_info_destroy(pdev, info); + return ret; +} + +int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + int cpu_num) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + + return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue, + &lfs->lf[cpu_num]); +} + +static int cpt_process_ccode(struct pci_dev *pdev, + union otx2_cpt_res_s *cpt_status, + struct otx2_cpt_inst_info *info, + u32 *res_code) +{ + u8 uc_ccode = cpt_status->s.uc_compcode; + u8 ccode = cpt_status->s.compcode; + + switch (ccode) { + case OTX2_CPT_COMP_E_FAULT: + dev_err(&pdev->dev, + "Request failed with DMA fault\n"); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + + case OTX2_CPT_COMP_E_HWERR: + dev_err(&pdev->dev, + "Request failed with hardware error\n"); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + + case OTX2_CPT_COMP_E_INSTERR: + dev_err(&pdev->dev, + "Request failed with instruction error\n"); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + + case OTX2_CPT_COMP_E_NOTDONE: + /* check for timeout */ + if (time_after_eq(jiffies, info->time_in + + CPT_COMMAND_TIMEOUT * HZ)) + dev_warn(&pdev->dev, + "Request timed out 0x%p", info->req); + else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) { + info->time_in = jiffies; + info->extra_time++; + } + return 1; + + case OTX2_CPT_COMP_E_GOOD: + /* + * Check microcode completion code, it is only valid + * when completion code is CPT_COMP_E::GOOD + */ + if (uc_ccode != OTX2_CPT_UCC_SUCCESS) { + /* + * If requested hmac is truncated and ucode returns + * s/g write length error then we report success + * because ucode writes as many bytes of calculated + * hmac as available in gather buffer and reports + * s/g write length error if number of bytes in gather + * buffer is less than full hmac size. + */ + if (info->req->is_trunc_hmac && + uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) { + *res_code = 0; + break; + } + + dev_err(&pdev->dev, + "Request failed with software error code 0x%x\n", + cpt_status->s.uc_compcode); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + } + /* Request has been processed with success */ + *res_code = 0; + break; + + default: + dev_err(&pdev->dev, + "Request returned invalid status %d\n", ccode); + break; + } + return 0; +} + +static inline void process_pending_queue(struct pci_dev *pdev, + struct otx2_cpt_pending_queue *pqueue) +{ + struct otx2_cpt_pending_entry *resume_pentry = NULL; + void (*callback)(int status, void *arg, void *req); + struct otx2_cpt_pending_entry *pentry = NULL; + union otx2_cpt_res_s *cpt_status = NULL; + struct otx2_cpt_inst_info *info = NULL; + struct otx2_cpt_req_info *req = NULL; + struct crypto_async_request *areq; + u32 res_code, resume_index; + + while (1) { + spin_lock_bh(&pqueue->lock); + pentry = &pqueue->head[pqueue->front]; + + if (WARN_ON(!pentry)) { + spin_unlock_bh(&pqueue->lock); + break; + } + + res_code = -EINVAL; + if (unlikely(!pentry->busy)) { + spin_unlock_bh(&pqueue->lock); + break; + } + + if (unlikely(!pentry->callback)) { + dev_err(&pdev->dev, "Callback NULL\n"); + goto process_pentry; + } + + info = pentry->info; + if (unlikely(!info)) { + dev_err(&pdev->dev, "Pending entry post arg NULL\n"); + goto process_pentry; + } + + req = info->req; + if (unlikely(!req)) { + dev_err(&pdev->dev, "Request NULL\n"); + goto process_pentry; + } + + cpt_status = pentry->completion_addr; + if (unlikely(!cpt_status)) { + dev_err(&pdev->dev, "Completion address NULL\n"); + goto process_pentry; + } + + if (cpt_process_ccode(pdev, cpt_status, info, &res_code)) { + spin_unlock_bh(&pqueue->lock); + return; + } + info->pdev = pdev; + +process_pentry: + /* + * Check if we should inform sending side to resume + * We do it CPT_IQ_RESUME_MARGIN elements in advance before + * pending queue becomes empty + */ + resume_index = modulo_inc(pqueue->front, pqueue->qlen, + CPT_IQ_RESUME_MARGIN); + resume_pentry = &pqueue->head[resume_index]; + if (resume_pentry && + resume_pentry->resume_sender) { + resume_pentry->resume_sender = false; + callback = resume_pentry->callback; + areq = resume_pentry->areq; + + if (callback) { + spin_unlock_bh(&pqueue->lock); + + /* + * EINPROGRESS is an indication for sending + * side that it can resume sending requests + */ + callback(-EINPROGRESS, areq, info); + spin_lock_bh(&pqueue->lock); + } + } + + callback = pentry->callback; + areq = pentry->areq; + free_pentry(pentry); + + pqueue->pending_count--; + pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1); + spin_unlock_bh(&pqueue->lock); + + /* + * Call callback after current pending entry has been + * processed, we don't do it if the callback pointer is + * invalid. + */ + if (callback) + callback(res_code, areq, info); + } +} + +void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe) +{ + process_pending_queue(wqe->lfs->pdev, + &wqe->lfs->lf[wqe->lf_num].pqueue); +} + +int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + + return cptvf->lfs.kcrypto_eng_grp_num; +} diff --git a/drivers/crypto/mediatek/Makefile b/drivers/crypto/mediatek/Makefile deleted file mode 100644 index 196a4653974e..000000000000 --- a/drivers/crypto/mediatek/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_CRYPTO_DEV_MEDIATEK) += mtk-crypto.o -mtk-crypto-objs:= mtk-platform.o mtk-aes.o mtk-sha.o diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c deleted file mode 100644 index 7323066724c3..000000000000 --- a/drivers/crypto/mediatek/mtk-aes.c +++ /dev/null @@ -1,1271 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Cryptographic API. - * - * Driver for EIP97 AES acceleration. - * - * Copyright (c) 2016 Ryder Lee - * - * Some ideas are from atmel-aes.c drivers. - */ - -#include -#include -#include -#include "mtk-platform.h" - -#define AES_QUEUE_SIZE 512 -#define AES_BUF_ORDER 2 -#define AES_BUF_SIZE ((PAGE_SIZE << AES_BUF_ORDER) \ - & ~(AES_BLOCK_SIZE - 1)) -#define AES_MAX_STATE_BUF_SIZE SIZE_IN_WORDS(AES_KEYSIZE_256 + \ - AES_BLOCK_SIZE * 2) -#define AES_MAX_CT_SIZE 6 - -#define AES_CT_CTRL_HDR cpu_to_le32(0x00220000) - -/* AES-CBC/ECB/CTR/OFB/CFB command token */ -#define AES_CMD0 cpu_to_le32(0x05000000) -#define AES_CMD1 cpu_to_le32(0x2d060000) -#define AES_CMD2 cpu_to_le32(0xe4a63806) -/* AES-GCM command token */ -#define AES_GCM_CMD0 cpu_to_le32(0x0b000000) -#define AES_GCM_CMD1 cpu_to_le32(0xa0800000) -#define AES_GCM_CMD2 cpu_to_le32(0x25000010) -#define AES_GCM_CMD3 cpu_to_le32(0x0f020000) -#define AES_GCM_CMD4 cpu_to_le32(0x21e60000) -#define AES_GCM_CMD5 cpu_to_le32(0x40e60000) -#define AES_GCM_CMD6 cpu_to_le32(0xd0070000) - -/* AES transform information word 0 fields */ -#define AES_TFM_BASIC_OUT cpu_to_le32(0x4 << 0) -#define AES_TFM_BASIC_IN cpu_to_le32(0x5 << 0) -#define AES_TFM_GCM_OUT cpu_to_le32(0x6 << 0) -#define AES_TFM_GCM_IN cpu_to_le32(0xf << 0) -#define AES_TFM_SIZE(x) cpu_to_le32((x) << 8) -#define AES_TFM_128BITS cpu_to_le32(0xb << 16) -#define AES_TFM_192BITS cpu_to_le32(0xd << 16) -#define AES_TFM_256BITS cpu_to_le32(0xf << 16) -#define AES_TFM_GHASH_DIGEST cpu_to_le32(0x2 << 21) -#define AES_TFM_GHASH cpu_to_le32(0x4 << 23) -/* AES transform information word 1 fields */ -#define AES_TFM_ECB cpu_to_le32(0x0 << 0) -#define AES_TFM_CBC cpu_to_le32(0x1 << 0) -#define AES_TFM_OFB cpu_to_le32(0x4 << 0) -#define AES_TFM_CFB128 cpu_to_le32(0x5 << 0) -#define AES_TFM_CTR_INIT cpu_to_le32(0x2 << 0) /* init counter to 1 */ -#define AES_TFM_CTR_LOAD cpu_to_le32(0x6 << 0) /* load/reuse counter */ -#define AES_TFM_3IV cpu_to_le32(0x7 << 5) /* using IV 0-2 */ -#define AES_TFM_FULL_IV cpu_to_le32(0xf << 5) /* using IV 0-3 */ -#define AES_TFM_IV_CTR_MODE cpu_to_le32(0x1 << 10) -#define AES_TFM_ENC_HASH cpu_to_le32(0x1 << 17) - -/* AES flags */ -#define AES_FLAGS_CIPHER_MSK GENMASK(4, 0) -#define AES_FLAGS_ECB BIT(0) -#define AES_FLAGS_CBC BIT(1) -#define AES_FLAGS_CTR BIT(2) -#define AES_FLAGS_OFB BIT(3) -#define AES_FLAGS_CFB128 BIT(4) -#define AES_FLAGS_GCM BIT(5) -#define AES_FLAGS_ENCRYPT BIT(6) -#define AES_FLAGS_BUSY BIT(7) - -#define AES_AUTH_TAG_ERR cpu_to_le32(BIT(26)) - -/** - * mtk_aes_info - hardware information of AES - * @cmd: command token, hardware instruction - * @tfm: transform state of cipher algorithm. - * @state: contains keys and initial vectors. - * - * Memory layout of GCM buffer: - * /-----------\ - * | AES KEY | 128/196/256 bits - * |-----------| - * | HASH KEY | a string 128 zero bits encrypted using the block cipher - * |-----------| - * | IVs | 4 * 4 bytes - * \-----------/ - * - * The engine requires all these info to do: - * - Commands decoding and control of the engine's data path. - * - Coordinating hardware data fetch and store operations. - * - Result token construction and output. - */ -struct mtk_aes_info { - __le32 cmd[AES_MAX_CT_SIZE]; - __le32 tfm[2]; - __le32 state[AES_MAX_STATE_BUF_SIZE]; -}; - -struct mtk_aes_reqctx { - u64 mode; -}; - -struct mtk_aes_base_ctx { - struct mtk_cryp *cryp; - u32 keylen; - __le32 key[12]; - __le32 keymode; - - mtk_aes_fn start; - - struct mtk_aes_info info; - dma_addr_t ct_dma; - dma_addr_t tfm_dma; - - __le32 ct_hdr; - u32 ct_size; -}; - -struct mtk_aes_ctx { - struct mtk_aes_base_ctx base; -}; - -struct mtk_aes_ctr_ctx { - struct mtk_aes_base_ctx base; - - __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; - size_t offset; - struct scatterlist src[2]; - struct scatterlist dst[2]; -}; - -struct mtk_aes_gcm_ctx { - struct mtk_aes_base_ctx base; - - u32 authsize; - size_t textlen; -}; - -struct mtk_aes_drv { - struct list_head dev_list; - /* Device list lock */ - spinlock_t lock; -}; - -static struct mtk_aes_drv mtk_aes = { - .dev_list = LIST_HEAD_INIT(mtk_aes.dev_list), - .lock = __SPIN_LOCK_UNLOCKED(mtk_aes.lock), -}; - -static inline u32 mtk_aes_read(struct mtk_cryp *cryp, u32 offset) -{ - return readl_relaxed(cryp->base + offset); -} - -static inline void mtk_aes_write(struct mtk_cryp *cryp, - u32 offset, u32 value) -{ - writel_relaxed(value, cryp->base + offset); -} - -static struct mtk_cryp *mtk_aes_find_dev(struct mtk_aes_base_ctx *ctx) -{ - struct mtk_cryp *cryp = NULL; - struct mtk_cryp *tmp; - - spin_lock_bh(&mtk_aes.lock); - if (!ctx->cryp) { - list_for_each_entry(tmp, &mtk_aes.dev_list, aes_list) { - cryp = tmp; - break; - } - ctx->cryp = cryp; - } else { - cryp = ctx->cryp; - } - spin_unlock_bh(&mtk_aes.lock); - - return cryp; -} - -static inline size_t mtk_aes_padlen(size_t len) -{ - len &= AES_BLOCK_SIZE - 1; - return len ? AES_BLOCK_SIZE - len : 0; -} - -static bool mtk_aes_check_aligned(struct scatterlist *sg, size_t len, - struct mtk_aes_dma *dma) -{ - int nents; - - if (!IS_ALIGNED(len, AES_BLOCK_SIZE)) - return false; - - for (nents = 0; sg; sg = sg_next(sg), ++nents) { - if (!IS_ALIGNED(sg->offset, sizeof(u32))) - return false; - - if (len <= sg->length) { - if (!IS_ALIGNED(len, AES_BLOCK_SIZE)) - return false; - - dma->nents = nents + 1; - dma->remainder = sg->length - len; - sg->length = len; - return true; - } - - if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE)) - return false; - - len -= sg->length; - } - - return false; -} - -static inline void mtk_aes_set_mode(struct mtk_aes_rec *aes, - const struct mtk_aes_reqctx *rctx) -{ - /* Clear all but persistent flags and set request flags. */ - aes->flags = (aes->flags & AES_FLAGS_BUSY) | rctx->mode; -} - -static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma) -{ - struct scatterlist *sg = dma->sg; - int nents = dma->nents; - - if (!dma->remainder) - return; - - while (--nents > 0 && sg) - sg = sg_next(sg); - - if (!sg) - return; - - sg->length += dma->remainder; -} - -static inline int mtk_aes_complete(struct mtk_cryp *cryp, - struct mtk_aes_rec *aes, - int err) -{ - aes->flags &= ~AES_FLAGS_BUSY; - aes->areq->complete(aes->areq, err); - /* Handle new request */ - tasklet_schedule(&aes->queue_task); - return err; -} - -/* - * Write descriptors for processing. This will configure the engine, load - * the transform information and then start the packet processing. - */ -static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_ring *ring = cryp->ring[aes->id]; - struct mtk_desc *cmd = NULL, *res = NULL; - struct scatterlist *ssg = aes->src.sg, *dsg = aes->dst.sg; - u32 slen = aes->src.sg_len, dlen = aes->dst.sg_len; - int nents; - - /* Write command descriptors */ - for (nents = 0; nents < slen; ++nents, ssg = sg_next(ssg)) { - cmd = ring->cmd_next; - cmd->hdr = MTK_DESC_BUF_LEN(ssg->length); - cmd->buf = cpu_to_le32(sg_dma_address(ssg)); - - if (nents == 0) { - cmd->hdr |= MTK_DESC_FIRST | - MTK_DESC_CT_LEN(aes->ctx->ct_size); - cmd->ct = cpu_to_le32(aes->ctx->ct_dma); - cmd->ct_hdr = aes->ctx->ct_hdr; - cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma); - } - - /* Shift ring buffer and check boundary */ - if (++ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) - ring->cmd_next = ring->cmd_base; - } - cmd->hdr |= MTK_DESC_LAST; - - /* Prepare result descriptors */ - for (nents = 0; nents < dlen; ++nents, dsg = sg_next(dsg)) { - res = ring->res_next; - res->hdr = MTK_DESC_BUF_LEN(dsg->length); - res->buf = cpu_to_le32(sg_dma_address(dsg)); - - if (nents == 0) - res->hdr |= MTK_DESC_FIRST; - - /* Shift ring buffer and check boundary */ - if (++ring->res_next == ring->res_base + MTK_DESC_NUM) - ring->res_next = ring->res_base; - } - res->hdr |= MTK_DESC_LAST; - - /* Pointer to current result descriptor */ - ring->res_prev = res; - - /* Prepare enough space for authenticated tag */ - if (aes->flags & AES_FLAGS_GCM) - le32_add_cpu(&res->hdr, AES_BLOCK_SIZE); - - /* - * Make sure that all changes to the DMA ring are done before we - * start engine. - */ - wmb(); - /* Start DMA transfer */ - mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen)); - mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen)); - - return -EINPROGRESS; -} - -static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_aes_base_ctx *ctx = aes->ctx; - - dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info), - DMA_TO_DEVICE); - - if (aes->src.sg == aes->dst.sg) { - dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents, - DMA_BIDIRECTIONAL); - - if (aes->src.sg != &aes->aligned_sg) - mtk_aes_restore_sg(&aes->src); - } else { - dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents, - DMA_FROM_DEVICE); - - if (aes->dst.sg != &aes->aligned_sg) - mtk_aes_restore_sg(&aes->dst); - - dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents, - DMA_TO_DEVICE); - - if (aes->src.sg != &aes->aligned_sg) - mtk_aes_restore_sg(&aes->src); - } - - if (aes->dst.sg == &aes->aligned_sg) - sg_copy_from_buffer(aes->real_dst, sg_nents(aes->real_dst), - aes->buf, aes->total); -} - -static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_aes_base_ctx *ctx = aes->ctx; - struct mtk_aes_info *info = &ctx->info; - - ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info), - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) - goto exit; - - ctx->tfm_dma = ctx->ct_dma + sizeof(info->cmd); - - if (aes->src.sg == aes->dst.sg) { - aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg, - aes->src.nents, - DMA_BIDIRECTIONAL); - aes->dst.sg_len = aes->src.sg_len; - if (unlikely(!aes->src.sg_len)) - goto sg_map_err; - } else { - aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg, - aes->src.nents, DMA_TO_DEVICE); - if (unlikely(!aes->src.sg_len)) - goto sg_map_err; - - aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg, - aes->dst.nents, DMA_FROM_DEVICE); - if (unlikely(!aes->dst.sg_len)) { - dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents, - DMA_TO_DEVICE); - goto sg_map_err; - } - } - - return mtk_aes_xmit(cryp, aes); - -sg_map_err: - dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(*info), DMA_TO_DEVICE); -exit: - return mtk_aes_complete(cryp, aes, -EINVAL); -} - -/* Initialize transform information of CBC/ECB/CTR/OFB/CFB mode */ -static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, - size_t len) -{ - struct skcipher_request *req = skcipher_request_cast(aes->areq); - struct mtk_aes_base_ctx *ctx = aes->ctx; - struct mtk_aes_info *info = &ctx->info; - u32 cnt = 0; - - ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len); - info->cmd[cnt++] = AES_CMD0 | cpu_to_le32(len); - info->cmd[cnt++] = AES_CMD1; - - info->tfm[0] = AES_TFM_SIZE(ctx->keylen) | ctx->keymode; - if (aes->flags & AES_FLAGS_ENCRYPT) - info->tfm[0] |= AES_TFM_BASIC_OUT; - else - info->tfm[0] |= AES_TFM_BASIC_IN; - - switch (aes->flags & AES_FLAGS_CIPHER_MSK) { - case AES_FLAGS_CBC: - info->tfm[1] = AES_TFM_CBC; - break; - case AES_FLAGS_ECB: - info->tfm[1] = AES_TFM_ECB; - goto ecb; - case AES_FLAGS_CTR: - info->tfm[1] = AES_TFM_CTR_LOAD; - goto ctr; - case AES_FLAGS_OFB: - info->tfm[1] = AES_TFM_OFB; - break; - case AES_FLAGS_CFB128: - info->tfm[1] = AES_TFM_CFB128; - break; - default: - /* Should not happen... */ - return; - } - - memcpy(info->state + ctx->keylen, req->iv, AES_BLOCK_SIZE); -ctr: - le32_add_cpu(&info->tfm[0], - le32_to_cpu(AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE)))); - info->tfm[1] |= AES_TFM_FULL_IV; - info->cmd[cnt++] = AES_CMD2; -ecb: - ctx->ct_size = cnt; -} - -static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, - struct scatterlist *src, struct scatterlist *dst, - size_t len) -{ - size_t padlen = 0; - bool src_aligned, dst_aligned; - - aes->total = len; - aes->src.sg = src; - aes->dst.sg = dst; - aes->real_dst = dst; - - src_aligned = mtk_aes_check_aligned(src, len, &aes->src); - if (src == dst) - dst_aligned = src_aligned; - else - dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst); - - if (!src_aligned || !dst_aligned) { - padlen = mtk_aes_padlen(len); - - if (len + padlen > AES_BUF_SIZE) - return mtk_aes_complete(cryp, aes, -ENOMEM); - - if (!src_aligned) { - sg_copy_to_buffer(src, sg_nents(src), aes->buf, len); - aes->src.sg = &aes->aligned_sg; - aes->src.nents = 1; - aes->src.remainder = 0; - } - - if (!dst_aligned) { - aes->dst.sg = &aes->aligned_sg; - aes->dst.nents = 1; - aes->dst.remainder = 0; - } - - sg_init_table(&aes->aligned_sg, 1); - sg_set_buf(&aes->aligned_sg, aes->buf, len + padlen); - } - - mtk_aes_info_init(cryp, aes, len + padlen); - - return mtk_aes_map(cryp, aes); -} - -static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id, - struct crypto_async_request *new_areq) -{ - struct mtk_aes_rec *aes = cryp->aes[id]; - struct crypto_async_request *areq, *backlog; - struct mtk_aes_base_ctx *ctx; - unsigned long flags; - int ret = 0; - - spin_lock_irqsave(&aes->lock, flags); - if (new_areq) - ret = crypto_enqueue_request(&aes->queue, new_areq); - if (aes->flags & AES_FLAGS_BUSY) { - spin_unlock_irqrestore(&aes->lock, flags); - return ret; - } - backlog = crypto_get_backlog(&aes->queue); - areq = crypto_dequeue_request(&aes->queue); - if (areq) - aes->flags |= AES_FLAGS_BUSY; - spin_unlock_irqrestore(&aes->lock, flags); - - if (!areq) - return ret; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - ctx = crypto_tfm_ctx(areq->tfm); - /* Write key into state buffer */ - memcpy(ctx->info.state, ctx->key, sizeof(ctx->key)); - - aes->areq = areq; - aes->ctx = ctx; - - return ctx->start(cryp, aes); -} - -static int mtk_aes_transfer_complete(struct mtk_cryp *cryp, - struct mtk_aes_rec *aes) -{ - return mtk_aes_complete(cryp, aes, 0); -} - -static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct skcipher_request *req = skcipher_request_cast(aes->areq); - struct mtk_aes_reqctx *rctx = skcipher_request_ctx(req); - - mtk_aes_set_mode(aes, rctx); - aes->resume = mtk_aes_transfer_complete; - - return mtk_aes_dma(cryp, aes, req->src, req->dst, req->cryptlen); -} - -static inline struct mtk_aes_ctr_ctx * -mtk_aes_ctr_ctx_cast(struct mtk_aes_base_ctx *ctx) -{ - return container_of(ctx, struct mtk_aes_ctr_ctx, base); -} - -static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_aes_base_ctx *ctx = aes->ctx; - struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(ctx); - struct skcipher_request *req = skcipher_request_cast(aes->areq); - struct scatterlist *src, *dst; - u32 start, end, ctr, blocks; - size_t datalen; - bool fragmented = false; - - /* Check for transfer completion. */ - cctx->offset += aes->total; - if (cctx->offset >= req->cryptlen) - return mtk_aes_transfer_complete(cryp, aes); - - /* Compute data length. */ - datalen = req->cryptlen - cctx->offset; - blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE); - ctr = be32_to_cpu(cctx->iv[3]); - - /* Check 32bit counter overflow. */ - start = ctr; - end = start + blocks - 1; - if (end < start) { - ctr = 0xffffffff; - datalen = AES_BLOCK_SIZE * -start; - fragmented = true; - } - - /* Jump to offset. */ - src = scatterwalk_ffwd(cctx->src, req->src, cctx->offset); - dst = ((req->src == req->dst) ? src : - scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset)); - - /* Write IVs into transform state buffer. */ - memcpy(ctx->info.state + ctx->keylen, cctx->iv, AES_BLOCK_SIZE); - - if (unlikely(fragmented)) { - /* - * Increment the counter manually to cope with the hardware - * counter overflow. - */ - cctx->iv[3] = cpu_to_be32(ctr); - crypto_inc((u8 *)cctx->iv, AES_BLOCK_SIZE); - } - - return mtk_aes_dma(cryp, aes, src, dst, datalen); -} - -static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(aes->ctx); - struct skcipher_request *req = skcipher_request_cast(aes->areq); - struct mtk_aes_reqctx *rctx = skcipher_request_ctx(req); - - mtk_aes_set_mode(aes, rctx); - - memcpy(cctx->iv, req->iv, AES_BLOCK_SIZE); - cctx->offset = 0; - aes->total = 0; - aes->resume = mtk_aes_ctr_transfer; - - return mtk_aes_ctr_transfer(cryp, aes); -} - -/* Check and set the AES key to transform state buffer */ -static int mtk_aes_setkey(struct crypto_skcipher *tfm, - const u8 *key, u32 keylen) -{ - struct mtk_aes_base_ctx *ctx = crypto_skcipher_ctx(tfm); - - switch (keylen) { - case AES_KEYSIZE_128: - ctx->keymode = AES_TFM_128BITS; - break; - case AES_KEYSIZE_192: - ctx->keymode = AES_TFM_192BITS; - break; - case AES_KEYSIZE_256: - ctx->keymode = AES_TFM_256BITS; - break; - - default: - return -EINVAL; - } - - ctx->keylen = SIZE_IN_WORDS(keylen); - memcpy(ctx->key, key, keylen); - - return 0; -} - -static int mtk_aes_crypt(struct skcipher_request *req, u64 mode) -{ - struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); - struct mtk_aes_base_ctx *ctx = crypto_skcipher_ctx(skcipher); - struct mtk_aes_reqctx *rctx; - struct mtk_cryp *cryp; - - cryp = mtk_aes_find_dev(ctx); - if (!cryp) - return -ENODEV; - - rctx = skcipher_request_ctx(req); - rctx->mode = mode; - - return mtk_aes_handle_queue(cryp, !(mode & AES_FLAGS_ENCRYPT), - &req->base); -} - -static int mtk_aes_ecb_encrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_ECB); -} - -static int mtk_aes_ecb_decrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ECB); -} - -static int mtk_aes_cbc_encrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CBC); -} - -static int mtk_aes_cbc_decrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_CBC); -} - -static int mtk_aes_ctr_encrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CTR); -} - -static int mtk_aes_ctr_decrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_CTR); -} - -static int mtk_aes_ofb_encrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_OFB); -} - -static int mtk_aes_ofb_decrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_OFB); -} - -static int mtk_aes_cfb_encrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CFB128); -} - -static int mtk_aes_cfb_decrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_CFB128); -} - -static int mtk_aes_init_tfm(struct crypto_skcipher *tfm) -{ - struct mtk_aes_ctx *ctx = crypto_skcipher_ctx(tfm); - - crypto_skcipher_set_reqsize(tfm, sizeof(struct mtk_aes_reqctx)); - ctx->base.start = mtk_aes_start; - return 0; -} - -static int mtk_aes_ctr_init_tfm(struct crypto_skcipher *tfm) -{ - struct mtk_aes_ctx *ctx = crypto_skcipher_ctx(tfm); - - crypto_skcipher_set_reqsize(tfm, sizeof(struct mtk_aes_reqctx)); - ctx->base.start = mtk_aes_ctr_start; - return 0; -} - -static struct skcipher_alg aes_algs[] = { -{ - .base.cra_name = "cbc(aes)", - .base.cra_driver_name = "cbc-aes-mtk", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_ASYNC, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mtk_aes_ctx), - .base.cra_alignmask = 0xf, - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = mtk_aes_setkey, - .encrypt = mtk_aes_cbc_encrypt, - .decrypt = mtk_aes_cbc_decrypt, - .ivsize = AES_BLOCK_SIZE, - .init = mtk_aes_init_tfm, -}, -{ - .base.cra_name = "ecb(aes)", - .base.cra_driver_name = "ecb-aes-mtk", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_ASYNC, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mtk_aes_ctx), - .base.cra_alignmask = 0xf, - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = mtk_aes_setkey, - .encrypt = mtk_aes_ecb_encrypt, - .decrypt = mtk_aes_ecb_decrypt, - .init = mtk_aes_init_tfm, -}, -{ - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-mtk", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_ASYNC, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct mtk_aes_ctx), - .base.cra_alignmask = 0xf, - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = mtk_aes_setkey, - .encrypt = mtk_aes_ctr_encrypt, - .decrypt = mtk_aes_ctr_decrypt, - .init = mtk_aes_ctr_init_tfm, -}, -{ - .base.cra_name = "ofb(aes)", - .base.cra_driver_name = "ofb-aes-mtk", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_ASYNC, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mtk_aes_ctx), - .base.cra_alignmask = 0xf, - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = mtk_aes_setkey, - .encrypt = mtk_aes_ofb_encrypt, - .decrypt = mtk_aes_ofb_decrypt, -}, -{ - .base.cra_name = "cfb(aes)", - .base.cra_driver_name = "cfb-aes-mtk", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_ASYNC, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct mtk_aes_ctx), - .base.cra_alignmask = 0xf, - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = mtk_aes_setkey, - .encrypt = mtk_aes_cfb_encrypt, - .decrypt = mtk_aes_cfb_decrypt, -}, -}; - -static inline struct mtk_aes_gcm_ctx * -mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx) -{ - return container_of(ctx, struct mtk_aes_gcm_ctx, base); -} - -/* - * Engine will verify and compare tag automatically, so we just need - * to check returned status which stored in the result descriptor. - */ -static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp, - struct mtk_aes_rec *aes) -{ - __le32 status = cryp->ring[aes->id]->res_prev->ct; - - return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ? - -EBADMSG : 0); -} - -/* Initialize transform information of GCM mode */ -static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, - struct mtk_aes_rec *aes, - size_t len) -{ - struct aead_request *req = aead_request_cast(aes->areq); - struct mtk_aes_base_ctx *ctx = aes->ctx; - struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); - struct mtk_aes_info *info = &ctx->info; - u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); - u32 cnt = 0; - - ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len); - - info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen); - info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen); - info->cmd[cnt++] = AES_GCM_CMD2; - info->cmd[cnt++] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen); - - if (aes->flags & AES_FLAGS_ENCRYPT) { - info->cmd[cnt++] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize); - info->tfm[0] = AES_TFM_GCM_OUT; - } else { - info->cmd[cnt++] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize); - info->cmd[cnt++] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize); - info->tfm[0] = AES_TFM_GCM_IN; - } - ctx->ct_size = cnt; - - info->tfm[0] |= AES_TFM_GHASH_DIGEST | AES_TFM_GHASH | AES_TFM_SIZE( - ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE + ivsize)) | - ctx->keymode; - info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV | - AES_TFM_ENC_HASH; - - memcpy(info->state + ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE), - req->iv, ivsize); -} - -static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, - struct scatterlist *src, struct scatterlist *dst, - size_t len) -{ - bool src_aligned, dst_aligned; - - aes->src.sg = src; - aes->dst.sg = dst; - aes->real_dst = dst; - - src_aligned = mtk_aes_check_aligned(src, len, &aes->src); - if (src == dst) - dst_aligned = src_aligned; - else - dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst); - - if (!src_aligned || !dst_aligned) { - if (aes->total > AES_BUF_SIZE) - return mtk_aes_complete(cryp, aes, -ENOMEM); - - if (!src_aligned) { - sg_copy_to_buffer(src, sg_nents(src), aes->buf, len); - aes->src.sg = &aes->aligned_sg; - aes->src.nents = 1; - aes->src.remainder = 0; - } - - if (!dst_aligned) { - aes->dst.sg = &aes->aligned_sg; - aes->dst.nents = 1; - aes->dst.remainder = 0; - } - - sg_init_table(&aes->aligned_sg, 1); - sg_set_buf(&aes->aligned_sg, aes->buf, aes->total); - } - - mtk_aes_gcm_info_init(cryp, aes, len); - - return mtk_aes_map(cryp, aes); -} - -/* Todo: GMAC */ -static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(aes->ctx); - struct aead_request *req = aead_request_cast(aes->areq); - struct mtk_aes_reqctx *rctx = aead_request_ctx(req); - u32 len = req->assoclen + req->cryptlen; - - mtk_aes_set_mode(aes, rctx); - - if (aes->flags & AES_FLAGS_ENCRYPT) { - u32 tag[4]; - - aes->resume = mtk_aes_transfer_complete; - /* Compute total process length. */ - aes->total = len + gctx->authsize; - /* Hardware will append authenticated tag to output buffer */ - scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1); - } else { - aes->resume = mtk_aes_gcm_tag_verify; - aes->total = len; - } - - return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len); -} - -static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode) -{ - struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); - struct mtk_aes_reqctx *rctx = aead_request_ctx(req); - struct mtk_cryp *cryp; - bool enc = !!(mode & AES_FLAGS_ENCRYPT); - - cryp = mtk_aes_find_dev(ctx); - if (!cryp) - return -ENODEV; - - /* Compute text length. */ - gctx->textlen = req->cryptlen - (enc ? 0 : gctx->authsize); - - /* Empty messages are not supported yet */ - if (!gctx->textlen && !req->assoclen) - return -EINVAL; - - rctx->mode = AES_FLAGS_GCM | mode; - - return mtk_aes_handle_queue(cryp, enc, &req->base); -} - -/* - * Because of the hardware limitation, we need to pre-calculate key(H) - * for the GHASH operation. The result of the encryption operation - * need to be stored in the transform state buffer. - */ -static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, - u32 keylen) -{ - struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead); - union { - u32 x32[SIZE_IN_WORDS(AES_BLOCK_SIZE)]; - u8 x8[AES_BLOCK_SIZE]; - } hash = {}; - struct crypto_aes_ctx aes_ctx; - int err; - int i; - - switch (keylen) { - case AES_KEYSIZE_128: - ctx->keymode = AES_TFM_128BITS; - break; - case AES_KEYSIZE_192: - ctx->keymode = AES_TFM_192BITS; - break; - case AES_KEYSIZE_256: - ctx->keymode = AES_TFM_256BITS; - break; - - default: - return -EINVAL; - } - - ctx->keylen = SIZE_IN_WORDS(keylen); - - err = aes_expandkey(&aes_ctx, key, keylen); - if (err) - return err; - - aes_encrypt(&aes_ctx, hash.x8, hash.x8); - memzero_explicit(&aes_ctx, sizeof(aes_ctx)); - - memcpy(ctx->key, key, keylen); - - /* Why do we need to do this? */ - for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++) - hash.x32[i] = swab32(hash.x32[i]); - - memcpy(ctx->key + ctx->keylen, &hash, AES_BLOCK_SIZE); - - return 0; -} - -static int mtk_aes_gcm_setauthsize(struct crypto_aead *aead, - u32 authsize) -{ - struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead); - struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); - - /* Same as crypto_gcm_authsize() from crypto/gcm.c */ - switch (authsize) { - case 8: - case 12: - case 16: - break; - default: - return -EINVAL; - } - - gctx->authsize = authsize; - return 0; -} - -static int mtk_aes_gcm_encrypt(struct aead_request *req) -{ - return mtk_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT); -} - -static int mtk_aes_gcm_decrypt(struct aead_request *req) -{ - return mtk_aes_gcm_crypt(req, 0); -} - -static int mtk_aes_gcm_init(struct crypto_aead *aead) -{ - struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); - - crypto_aead_set_reqsize(aead, sizeof(struct mtk_aes_reqctx)); - ctx->base.start = mtk_aes_gcm_start; - return 0; -} - -static struct aead_alg aes_gcm_alg = { - .setkey = mtk_aes_gcm_setkey, - .setauthsize = mtk_aes_gcm_setauthsize, - .encrypt = mtk_aes_gcm_encrypt, - .decrypt = mtk_aes_gcm_decrypt, - .init = mtk_aes_gcm_init, - .ivsize = GCM_AES_IV_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - - .base = { - .cra_name = "gcm(aes)", - .cra_driver_name = "gcm-aes-mtk", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct mtk_aes_gcm_ctx), - .cra_alignmask = 0xf, - .cra_module = THIS_MODULE, - }, -}; - -static void mtk_aes_queue_task(unsigned long data) -{ - struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; - - mtk_aes_handle_queue(aes->cryp, aes->id, NULL); -} - -static void mtk_aes_done_task(unsigned long data) -{ - struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; - struct mtk_cryp *cryp = aes->cryp; - - mtk_aes_unmap(cryp, aes); - aes->resume(cryp, aes); -} - -static irqreturn_t mtk_aes_irq(int irq, void *dev_id) -{ - struct mtk_aes_rec *aes = (struct mtk_aes_rec *)dev_id; - struct mtk_cryp *cryp = aes->cryp; - u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id)); - - mtk_aes_write(cryp, RDR_STAT(aes->id), val); - - if (likely(AES_FLAGS_BUSY & aes->flags)) { - mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST); - mtk_aes_write(cryp, RDR_THRESH(aes->id), - MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - - tasklet_schedule(&aes->done_task); - } else { - dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); - } - return IRQ_HANDLED; -} - -/* - * The purpose of creating encryption and decryption records is - * to process outbound/inbound data in parallel, it can improve - * performance in most use cases, such as IPSec VPN, especially - * under heavy network traffic. - */ -static int mtk_aes_record_init(struct mtk_cryp *cryp) -{ - struct mtk_aes_rec **aes = cryp->aes; - int i, err = -ENOMEM; - - for (i = 0; i < MTK_REC_NUM; i++) { - aes[i] = kzalloc(sizeof(**aes), GFP_KERNEL); - if (!aes[i]) - goto err_cleanup; - - aes[i]->buf = (void *)__get_free_pages(GFP_KERNEL, - AES_BUF_ORDER); - if (!aes[i]->buf) - goto err_cleanup; - - aes[i]->cryp = cryp; - - spin_lock_init(&aes[i]->lock); - crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE); - - tasklet_init(&aes[i]->queue_task, mtk_aes_queue_task, - (unsigned long)aes[i]); - tasklet_init(&aes[i]->done_task, mtk_aes_done_task, - (unsigned long)aes[i]); - } - - /* Link to ring0 and ring1 respectively */ - aes[0]->id = MTK_RING0; - aes[1]->id = MTK_RING1; - - return 0; - -err_cleanup: - for (; i--; ) { - free_page((unsigned long)aes[i]->buf); - kfree(aes[i]); - } - - return err; -} - -static void mtk_aes_record_free(struct mtk_cryp *cryp) -{ - int i; - - for (i = 0; i < MTK_REC_NUM; i++) { - tasklet_kill(&cryp->aes[i]->done_task); - tasklet_kill(&cryp->aes[i]->queue_task); - - free_page((unsigned long)cryp->aes[i]->buf); - kfree(cryp->aes[i]); - } -} - -static void mtk_aes_unregister_algs(void) -{ - int i; - - crypto_unregister_aead(&aes_gcm_alg); - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) - crypto_unregister_skcipher(&aes_algs[i]); -} - -static int mtk_aes_register_algs(void) -{ - int err, i; - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - err = crypto_register_skcipher(&aes_algs[i]); - if (err) - goto err_aes_algs; - } - - err = crypto_register_aead(&aes_gcm_alg); - if (err) - goto err_aes_algs; - - return 0; - -err_aes_algs: - for (; i--; ) - crypto_unregister_skcipher(&aes_algs[i]); - - return err; -} - -int mtk_cipher_alg_register(struct mtk_cryp *cryp) -{ - int ret; - - INIT_LIST_HEAD(&cryp->aes_list); - - /* Initialize two cipher records */ - ret = mtk_aes_record_init(cryp); - if (ret) - goto err_record; - - ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING0], mtk_aes_irq, - 0, "mtk-aes", cryp->aes[0]); - if (ret) { - dev_err(cryp->dev, "unable to request AES irq.\n"); - goto err_res; - } - - ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING1], mtk_aes_irq, - 0, "mtk-aes", cryp->aes[1]); - if (ret) { - dev_err(cryp->dev, "unable to request AES irq.\n"); - goto err_res; - } - - /* Enable ring0 and ring1 interrupt */ - mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING0), MTK_IRQ_RDR0); - mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING1), MTK_IRQ_RDR1); - - spin_lock(&mtk_aes.lock); - list_add_tail(&cryp->aes_list, &mtk_aes.dev_list); - spin_unlock(&mtk_aes.lock); - - ret = mtk_aes_register_algs(); - if (ret) - goto err_algs; - - return 0; - -err_algs: - spin_lock(&mtk_aes.lock); - list_del(&cryp->aes_list); - spin_unlock(&mtk_aes.lock); -err_res: - mtk_aes_record_free(cryp); -err_record: - - dev_err(cryp->dev, "mtk-aes initialization failed.\n"); - return ret; -} - -void mtk_cipher_alg_release(struct mtk_cryp *cryp) -{ - spin_lock(&mtk_aes.lock); - list_del(&cryp->aes_list); - spin_unlock(&mtk_aes.lock); - - mtk_aes_unregister_algs(); - mtk_aes_record_free(cryp); -} diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c deleted file mode 100644 index 9d878620e5c9..000000000000 --- a/drivers/crypto/mediatek/mtk-platform.c +++ /dev/null @@ -1,586 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Driver for EIP97 cryptographic accelerator. - * - * Copyright (c) 2016 Ryder Lee - */ - -#include -#include -#include -#include -#include -#include -#include -#include "mtk-platform.h" - -#define MTK_BURST_SIZE_MSK GENMASK(7, 4) -#define MTK_BURST_SIZE(x) ((x) << 4) -#define MTK_DESC_SIZE(x) ((x) << 0) -#define MTK_DESC_OFFSET(x) ((x) << 16) -#define MTK_DESC_FETCH_SIZE(x) ((x) << 0) -#define MTK_DESC_FETCH_THRESH(x) ((x) << 16) -#define MTK_DESC_OVL_IRQ_EN BIT(25) -#define MTK_DESC_ATP_PRESENT BIT(30) - -#define MTK_DFSE_IDLE GENMASK(3, 0) -#define MTK_DFSE_THR_CTRL_EN BIT(30) -#define MTK_DFSE_THR_CTRL_RESET BIT(31) -#define MTK_DFSE_RING_ID(x) (((x) >> 12) & GENMASK(3, 0)) -#define MTK_DFSE_MIN_DATA(x) ((x) << 0) -#define MTK_DFSE_MAX_DATA(x) ((x) << 8) -#define MTK_DFE_MIN_CTRL(x) ((x) << 16) -#define MTK_DFE_MAX_CTRL(x) ((x) << 24) - -#define MTK_IN_BUF_MIN_THRESH(x) ((x) << 8) -#define MTK_IN_BUF_MAX_THRESH(x) ((x) << 12) -#define MTK_OUT_BUF_MIN_THRESH(x) ((x) << 0) -#define MTK_OUT_BUF_MAX_THRESH(x) ((x) << 4) -#define MTK_IN_TBUF_SIZE(x) (((x) >> 4) & GENMASK(3, 0)) -#define MTK_IN_DBUF_SIZE(x) (((x) >> 8) & GENMASK(3, 0)) -#define MTK_OUT_DBUF_SIZE(x) (((x) >> 16) & GENMASK(3, 0)) -#define MTK_CMD_FIFO_SIZE(x) (((x) >> 8) & GENMASK(3, 0)) -#define MTK_RES_FIFO_SIZE(x) (((x) >> 12) & GENMASK(3, 0)) - -#define MTK_PE_TK_LOC_AVL BIT(2) -#define MTK_PE_PROC_HELD BIT(14) -#define MTK_PE_TK_TIMEOUT_EN BIT(22) -#define MTK_PE_INPUT_DMA_ERR BIT(0) -#define MTK_PE_OUTPUT_DMA_ERR BIT(1) -#define MTK_PE_PKT_PORC_ERR BIT(2) -#define MTK_PE_PKT_TIMEOUT BIT(3) -#define MTK_PE_FATAL_ERR BIT(14) -#define MTK_PE_INPUT_DMA_ERR_EN BIT(16) -#define MTK_PE_OUTPUT_DMA_ERR_EN BIT(17) -#define MTK_PE_PKT_PORC_ERR_EN BIT(18) -#define MTK_PE_PKT_TIMEOUT_EN BIT(19) -#define MTK_PE_FATAL_ERR_EN BIT(30) -#define MTK_PE_INT_OUT_EN BIT(31) - -#define MTK_HIA_SIGNATURE ((u16)0x35ca) -#define MTK_HIA_DATA_WIDTH(x) (((x) >> 25) & GENMASK(1, 0)) -#define MTK_HIA_DMA_LENGTH(x) (((x) >> 20) & GENMASK(4, 0)) -#define MTK_CDR_STAT_CLR GENMASK(4, 0) -#define MTK_RDR_STAT_CLR GENMASK(7, 0) - -#define MTK_AIC_INT_MSK GENMASK(5, 0) -#define MTK_AIC_VER_MSK (GENMASK(15, 0) | GENMASK(27, 20)) -#define MTK_AIC_VER11 0x011036c9 -#define MTK_AIC_VER12 0x012036c9 -#define MTK_AIC_G_CLR GENMASK(30, 20) - -/** - * EIP97 is an integrated security subsystem to accelerate cryptographic - * functions and protocols to offload the host processor. - * Some important hardware modules are briefly introduced below: - * - * Host Interface Adapter(HIA) - the main interface between the host - * system and the hardware subsystem. It is responsible for attaching - * processing engine to the specific host bus interface and provides a - * standardized software view for off loading tasks to the engine. - * - * Command Descriptor Ring Manager(CDR Manager) - keeps track of how many - * CD the host has prepared in the CDR. It monitors the fill level of its - * CD-FIFO and if there's sufficient space for the next block of descriptors, - * then it fires off a DMA request to fetch a block of CDs. - * - * Data fetch engine(DFE) - It is responsible for parsing the CD and - * setting up the required control and packet data DMA transfers from - * system memory to the processing engine. - * - * Result Descriptor Ring Manager(RDR Manager) - same as CDR Manager, - * but target is result descriptors, Moreover, it also handles the RD - * updates under control of the DSE. For each packet data segment - * processed, the DSE triggers the RDR Manager to write the updated RD. - * If triggered to update, the RDR Manager sets up a DMA operation to - * copy the RD from the DSE to the correct location in the RDR. - * - * Data Store Engine(DSE) - It is responsible for parsing the prepared RD - * and setting up the required control and packet data DMA transfers from - * the processing engine to system memory. - * - * Advanced Interrupt Controllers(AICs) - receive interrupt request signals - * from various sources and combine them into one interrupt output. - * The AICs are used by: - * - One for the HIA global and processing engine interrupts. - * - The others for the descriptor ring interrupts. - */ - -/* Cryptographic engine capabilities */ -struct mtk_sys_cap { - /* host interface adapter */ - u32 hia_ver; - u32 hia_opt; - /* packet engine */ - u32 pkt_eng_opt; - /* global hardware */ - u32 hw_opt; -}; - -static void mtk_desc_ring_link(struct mtk_cryp *cryp, u32 mask) -{ - /* Assign rings to DFE/DSE thread and enable it */ - writel(MTK_DFSE_THR_CTRL_EN | mask, cryp->base + DFE_THR_CTRL); - writel(MTK_DFSE_THR_CTRL_EN | mask, cryp->base + DSE_THR_CTRL); -} - -static void mtk_dfe_dse_buf_setup(struct mtk_cryp *cryp, - struct mtk_sys_cap *cap) -{ - u32 width = MTK_HIA_DATA_WIDTH(cap->hia_opt) + 2; - u32 len = MTK_HIA_DMA_LENGTH(cap->hia_opt) - 1; - u32 ipbuf = min((u32)MTK_IN_DBUF_SIZE(cap->hw_opt) + width, len); - u32 opbuf = min((u32)MTK_OUT_DBUF_SIZE(cap->hw_opt) + width, len); - u32 itbuf = min((u32)MTK_IN_TBUF_SIZE(cap->hw_opt) + width, len); - - writel(MTK_DFSE_MIN_DATA(ipbuf - 1) | - MTK_DFSE_MAX_DATA(ipbuf) | - MTK_DFE_MIN_CTRL(itbuf - 1) | - MTK_DFE_MAX_CTRL(itbuf), - cryp->base + DFE_CFG); - - writel(MTK_DFSE_MIN_DATA(opbuf - 1) | - MTK_DFSE_MAX_DATA(opbuf), - cryp->base + DSE_CFG); - - writel(MTK_IN_BUF_MIN_THRESH(ipbuf - 1) | - MTK_IN_BUF_MAX_THRESH(ipbuf), - cryp->base + PE_IN_DBUF_THRESH); - - writel(MTK_IN_BUF_MIN_THRESH(itbuf - 1) | - MTK_IN_BUF_MAX_THRESH(itbuf), - cryp->base + PE_IN_TBUF_THRESH); - - writel(MTK_OUT_BUF_MIN_THRESH(opbuf - 1) | - MTK_OUT_BUF_MAX_THRESH(opbuf), - cryp->base + PE_OUT_DBUF_THRESH); - - writel(0, cryp->base + PE_OUT_TBUF_THRESH); - writel(0, cryp->base + PE_OUT_BUF_CTRL); -} - -static int mtk_dfe_dse_state_check(struct mtk_cryp *cryp) -{ - int ret = -EINVAL; - u32 val; - - /* Check for completion of all DMA transfers */ - val = readl(cryp->base + DFE_THR_STAT); - if (MTK_DFSE_RING_ID(val) == MTK_DFSE_IDLE) { - val = readl(cryp->base + DSE_THR_STAT); - if (MTK_DFSE_RING_ID(val) == MTK_DFSE_IDLE) - ret = 0; - } - - if (!ret) { - /* Take DFE/DSE thread out of reset */ - writel(0, cryp->base + DFE_THR_CTRL); - writel(0, cryp->base + DSE_THR_CTRL); - } else { - return -EBUSY; - } - - return 0; -} - -static int mtk_dfe_dse_reset(struct mtk_cryp *cryp) -{ - /* Reset DSE/DFE and correct system priorities for all rings. */ - writel(MTK_DFSE_THR_CTRL_RESET, cryp->base + DFE_THR_CTRL); - writel(0, cryp->base + DFE_PRIO_0); - writel(0, cryp->base + DFE_PRIO_1); - writel(0, cryp->base + DFE_PRIO_2); - writel(0, cryp->base + DFE_PRIO_3); - - writel(MTK_DFSE_THR_CTRL_RESET, cryp->base + DSE_THR_CTRL); - writel(0, cryp->base + DSE_PRIO_0); - writel(0, cryp->base + DSE_PRIO_1); - writel(0, cryp->base + DSE_PRIO_2); - writel(0, cryp->base + DSE_PRIO_3); - - return mtk_dfe_dse_state_check(cryp); -} - -static void mtk_cmd_desc_ring_setup(struct mtk_cryp *cryp, - int i, struct mtk_sys_cap *cap) -{ - /* Full descriptor that fits FIFO minus one */ - u32 count = - ((1 << MTK_CMD_FIFO_SIZE(cap->hia_opt)) / MTK_DESC_SZ) - 1; - - /* Temporarily disable external triggering */ - writel(0, cryp->base + CDR_CFG(i)); - - /* Clear CDR count */ - writel(MTK_CNT_RST, cryp->base + CDR_PREP_COUNT(i)); - writel(MTK_CNT_RST, cryp->base + CDR_PROC_COUNT(i)); - - writel(0, cryp->base + CDR_PREP_PNTR(i)); - writel(0, cryp->base + CDR_PROC_PNTR(i)); - writel(0, cryp->base + CDR_DMA_CFG(i)); - - /* Configure CDR host address space */ - writel(0, cryp->base + CDR_BASE_ADDR_HI(i)); - writel(cryp->ring[i]->cmd_dma, cryp->base + CDR_BASE_ADDR_LO(i)); - - writel(MTK_DESC_RING_SZ, cryp->base + CDR_RING_SIZE(i)); - - /* Clear and disable all CDR interrupts */ - writel(MTK_CDR_STAT_CLR, cryp->base + CDR_STAT(i)); - - /* - * Set command descriptor offset and enable additional - * token present in descriptor. - */ - writel(MTK_DESC_SIZE(MTK_DESC_SZ) | - MTK_DESC_OFFSET(MTK_DESC_OFF) | - MTK_DESC_ATP_PRESENT, - cryp->base + CDR_DESC_SIZE(i)); - - writel(MTK_DESC_FETCH_SIZE(count * MTK_DESC_OFF) | - MTK_DESC_FETCH_THRESH(count * MTK_DESC_SZ), - cryp->base + CDR_CFG(i)); -} - -static void mtk_res_desc_ring_setup(struct mtk_cryp *cryp, - int i, struct mtk_sys_cap *cap) -{ - u32 rndup = 2; - u32 count = ((1 << MTK_RES_FIFO_SIZE(cap->hia_opt)) / rndup) - 1; - - /* Temporarily disable external triggering */ - writel(0, cryp->base + RDR_CFG(i)); - - /* Clear RDR count */ - writel(MTK_CNT_RST, cryp->base + RDR_PREP_COUNT(i)); - writel(MTK_CNT_RST, cryp->base + RDR_PROC_COUNT(i)); - - writel(0, cryp->base + RDR_PREP_PNTR(i)); - writel(0, cryp->base + RDR_PROC_PNTR(i)); - writel(0, cryp->base + RDR_DMA_CFG(i)); - - /* Configure RDR host address space */ - writel(0, cryp->base + RDR_BASE_ADDR_HI(i)); - writel(cryp->ring[i]->res_dma, cryp->base + RDR_BASE_ADDR_LO(i)); - - writel(MTK_DESC_RING_SZ, cryp->base + RDR_RING_SIZE(i)); - writel(MTK_RDR_STAT_CLR, cryp->base + RDR_STAT(i)); - - /* - * RDR manager generates update interrupts on a per-completed-packet, - * and the rd_proc_thresh_irq interrupt is fired when proc_pkt_count - * for the RDR exceeds the number of packets. - */ - writel(MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE, - cryp->base + RDR_THRESH(i)); - - /* - * Configure a threshold and time-out value for the processed - * result descriptors (or complete packets) that are written to - * the RDR. - */ - writel(MTK_DESC_SIZE(MTK_DESC_SZ) | MTK_DESC_OFFSET(MTK_DESC_OFF), - cryp->base + RDR_DESC_SIZE(i)); - - /* - * Configure HIA fetch size and fetch threshold that are used to - * fetch blocks of multiple descriptors. - */ - writel(MTK_DESC_FETCH_SIZE(count * MTK_DESC_OFF) | - MTK_DESC_FETCH_THRESH(count * rndup) | - MTK_DESC_OVL_IRQ_EN, - cryp->base + RDR_CFG(i)); -} - -static int mtk_packet_engine_setup(struct mtk_cryp *cryp) -{ - struct mtk_sys_cap cap; - int i, err; - u32 val; - - cap.hia_ver = readl(cryp->base + HIA_VERSION); - cap.hia_opt = readl(cryp->base + HIA_OPTIONS); - cap.hw_opt = readl(cryp->base + EIP97_OPTIONS); - - if (!(((u16)cap.hia_ver) == MTK_HIA_SIGNATURE)) - return -EINVAL; - - /* Configure endianness conversion method for master (DMA) interface */ - writel(0, cryp->base + EIP97_MST_CTRL); - - /* Set HIA burst size */ - val = readl(cryp->base + HIA_MST_CTRL); - val &= ~MTK_BURST_SIZE_MSK; - val |= MTK_BURST_SIZE(5); - writel(val, cryp->base + HIA_MST_CTRL); - - err = mtk_dfe_dse_reset(cryp); - if (err) { - dev_err(cryp->dev, "Failed to reset DFE and DSE.\n"); - return err; - } - - mtk_dfe_dse_buf_setup(cryp, &cap); - - /* Enable the 4 rings for the packet engines. */ - mtk_desc_ring_link(cryp, 0xf); - - for (i = 0; i < MTK_RING_MAX; i++) { - mtk_cmd_desc_ring_setup(cryp, i, &cap); - mtk_res_desc_ring_setup(cryp, i, &cap); - } - - writel(MTK_PE_TK_LOC_AVL | MTK_PE_PROC_HELD | MTK_PE_TK_TIMEOUT_EN, - cryp->base + PE_TOKEN_CTRL_STAT); - - /* Clear all pending interrupts */ - writel(MTK_AIC_G_CLR, cryp->base + AIC_G_ACK); - writel(MTK_PE_INPUT_DMA_ERR | MTK_PE_OUTPUT_DMA_ERR | - MTK_PE_PKT_PORC_ERR | MTK_PE_PKT_TIMEOUT | - MTK_PE_FATAL_ERR | MTK_PE_INPUT_DMA_ERR_EN | - MTK_PE_OUTPUT_DMA_ERR_EN | MTK_PE_PKT_PORC_ERR_EN | - MTK_PE_PKT_TIMEOUT_EN | MTK_PE_FATAL_ERR_EN | - MTK_PE_INT_OUT_EN, - cryp->base + PE_INTERRUPT_CTRL_STAT); - - return 0; -} - -static int mtk_aic_cap_check(struct mtk_cryp *cryp, int hw) -{ - u32 val; - - if (hw == MTK_RING_MAX) - val = readl(cryp->base + AIC_G_VERSION); - else - val = readl(cryp->base + AIC_VERSION(hw)); - - val &= MTK_AIC_VER_MSK; - if (val != MTK_AIC_VER11 && val != MTK_AIC_VER12) - return -ENXIO; - - if (hw == MTK_RING_MAX) - val = readl(cryp->base + AIC_G_OPTIONS); - else - val = readl(cryp->base + AIC_OPTIONS(hw)); - - val &= MTK_AIC_INT_MSK; - if (!val || val > 32) - return -ENXIO; - - return 0; -} - -static int mtk_aic_init(struct mtk_cryp *cryp, int hw) -{ - int err; - - err = mtk_aic_cap_check(cryp, hw); - if (err) - return err; - - /* Disable all interrupts and set initial configuration */ - if (hw == MTK_RING_MAX) { - writel(0, cryp->base + AIC_G_ENABLE_CTRL); - writel(0, cryp->base + AIC_G_POL_CTRL); - writel(0, cryp->base + AIC_G_TYPE_CTRL); - writel(0, cryp->base + AIC_G_ENABLE_SET); - } else { - writel(0, cryp->base + AIC_ENABLE_CTRL(hw)); - writel(0, cryp->base + AIC_POL_CTRL(hw)); - writel(0, cryp->base + AIC_TYPE_CTRL(hw)); - writel(0, cryp->base + AIC_ENABLE_SET(hw)); - } - - return 0; -} - -static int mtk_accelerator_init(struct mtk_cryp *cryp) -{ - int i, err; - - /* Initialize advanced interrupt controller(AIC) */ - for (i = 0; i < MTK_IRQ_NUM; i++) { - err = mtk_aic_init(cryp, i); - if (err) { - dev_err(cryp->dev, "Failed to initialize AIC.\n"); - return err; - } - } - - /* Initialize packet engine */ - err = mtk_packet_engine_setup(cryp); - if (err) { - dev_err(cryp->dev, "Failed to configure packet engine.\n"); - return err; - } - - return 0; -} - -static void mtk_desc_dma_free(struct mtk_cryp *cryp) -{ - int i; - - for (i = 0; i < MTK_RING_MAX; i++) { - dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, - cryp->ring[i]->res_base, - cryp->ring[i]->res_dma); - dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, - cryp->ring[i]->cmd_base, - cryp->ring[i]->cmd_dma); - kfree(cryp->ring[i]); - } -} - -static int mtk_desc_ring_alloc(struct mtk_cryp *cryp) -{ - struct mtk_ring **ring = cryp->ring; - int i; - - for (i = 0; i < MTK_RING_MAX; i++) { - ring[i] = kzalloc(sizeof(**ring), GFP_KERNEL); - if (!ring[i]) - goto err_cleanup; - - ring[i]->cmd_base = dma_alloc_coherent(cryp->dev, - MTK_DESC_RING_SZ, - &ring[i]->cmd_dma, - GFP_KERNEL); - if (!ring[i]->cmd_base) - goto err_cleanup; - - ring[i]->res_base = dma_alloc_coherent(cryp->dev, - MTK_DESC_RING_SZ, - &ring[i]->res_dma, - GFP_KERNEL); - if (!ring[i]->res_base) - goto err_cleanup; - - ring[i]->cmd_next = ring[i]->cmd_base; - ring[i]->res_next = ring[i]->res_base; - } - return 0; - -err_cleanup: - do { - dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, - ring[i]->res_base, ring[i]->res_dma); - dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, - ring[i]->cmd_base, ring[i]->cmd_dma); - kfree(ring[i]); - } while (i--); - return -ENOMEM; -} - -static int mtk_crypto_probe(struct platform_device *pdev) -{ - struct mtk_cryp *cryp; - int i, err; - - cryp = devm_kzalloc(&pdev->dev, sizeof(*cryp), GFP_KERNEL); - if (!cryp) - return -ENOMEM; - - cryp->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(cryp->base)) - return PTR_ERR(cryp->base); - - for (i = 0; i < MTK_IRQ_NUM; i++) { - cryp->irq[i] = platform_get_irq(pdev, i); - if (cryp->irq[i] < 0) - return cryp->irq[i]; - } - - cryp->clk_cryp = devm_clk_get(&pdev->dev, "cryp"); - if (IS_ERR(cryp->clk_cryp)) - return -EPROBE_DEFER; - - cryp->dev = &pdev->dev; - pm_runtime_enable(cryp->dev); - pm_runtime_get_sync(cryp->dev); - - err = clk_prepare_enable(cryp->clk_cryp); - if (err) - goto err_clk_cryp; - - /* Allocate four command/result descriptor rings */ - err = mtk_desc_ring_alloc(cryp); - if (err) { - dev_err(cryp->dev, "Unable to allocate descriptor rings.\n"); - goto err_resource; - } - - /* Initialize hardware modules */ - err = mtk_accelerator_init(cryp); - if (err) { - dev_err(cryp->dev, "Failed to initialize cryptographic engine.\n"); - goto err_engine; - } - - err = mtk_cipher_alg_register(cryp); - if (err) { - dev_err(cryp->dev, "Unable to register cipher algorithm.\n"); - goto err_cipher; - } - - err = mtk_hash_alg_register(cryp); - if (err) { - dev_err(cryp->dev, "Unable to register hash algorithm.\n"); - goto err_hash; - } - - platform_set_drvdata(pdev, cryp); - return 0; - -err_hash: - mtk_cipher_alg_release(cryp); -err_cipher: - mtk_dfe_dse_reset(cryp); -err_engine: - mtk_desc_dma_free(cryp); -err_resource: - clk_disable_unprepare(cryp->clk_cryp); -err_clk_cryp: - pm_runtime_put_sync(cryp->dev); - pm_runtime_disable(cryp->dev); - - return err; -} - -static int mtk_crypto_remove(struct platform_device *pdev) -{ - struct mtk_cryp *cryp = platform_get_drvdata(pdev); - - mtk_hash_alg_release(cryp); - mtk_cipher_alg_release(cryp); - mtk_desc_dma_free(cryp); - - clk_disable_unprepare(cryp->clk_cryp); - - pm_runtime_put_sync(cryp->dev); - pm_runtime_disable(cryp->dev); - platform_set_drvdata(pdev, NULL); - - return 0; -} - -static const struct of_device_id of_crypto_id[] = { - { .compatible = "mediatek,eip97-crypto" }, - {}, -}; -MODULE_DEVICE_TABLE(of, of_crypto_id); - -static struct platform_driver mtk_crypto_driver = { - .probe = mtk_crypto_probe, - .remove = mtk_crypto_remove, - .driver = { - .name = "mtk-crypto", - .of_match_table = of_crypto_id, - }, -}; -module_platform_driver(mtk_crypto_driver); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Ryder Lee "); -MODULE_DESCRIPTION("Cryptographic accelerator driver for EIP97"); diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h deleted file mode 100644 index 47920c51abac..000000000000 --- a/drivers/crypto/mediatek/mtk-platform.h +++ /dev/null @@ -1,231 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Driver for EIP97 cryptographic accelerator. - * - * Copyright (c) 2016 Ryder Lee - */ - -#ifndef __MTK_PLATFORM_H_ -#define __MTK_PLATFORM_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "mtk-regs.h" - -#define MTK_RDR_PROC_THRESH BIT(0) -#define MTK_RDR_PROC_MODE BIT(23) -#define MTK_CNT_RST BIT(31) -#define MTK_IRQ_RDR0 BIT(1) -#define MTK_IRQ_RDR1 BIT(3) -#define MTK_IRQ_RDR2 BIT(5) -#define MTK_IRQ_RDR3 BIT(7) - -#define SIZE_IN_WORDS(x) ((x) >> 2) - -/** - * Ring 0/1 are used by AES encrypt and decrypt. - * Ring 2/3 are used by SHA. - */ -enum { - MTK_RING0, - MTK_RING1, - MTK_RING2, - MTK_RING3, - MTK_RING_MAX -}; - -#define MTK_REC_NUM (MTK_RING_MAX / 2) -#define MTK_IRQ_NUM 5 - -/** - * struct mtk_desc - DMA descriptor - * @hdr: the descriptor control header - * @buf: DMA address of input buffer segment - * @ct: DMA address of command token that control operation flow - * @ct_hdr: the command token control header - * @tag: the user-defined field - * @tfm: DMA address of transform state - * @bound: align descriptors offset boundary - * - * Structure passed to the crypto engine to describe where source - * data needs to be fetched and how it needs to be processed. - */ -struct mtk_desc { - __le32 hdr; - __le32 buf; - __le32 ct; - __le32 ct_hdr; - __le32 tag; - __le32 tfm; - __le32 bound[2]; -}; - -#define MTK_DESC_NUM 512 -#define MTK_DESC_OFF SIZE_IN_WORDS(sizeof(struct mtk_desc)) -#define MTK_DESC_SZ (MTK_DESC_OFF - 2) -#define MTK_DESC_RING_SZ ((sizeof(struct mtk_desc) * MTK_DESC_NUM)) -#define MTK_DESC_CNT(x) ((MTK_DESC_OFF * (x)) << 2) -#define MTK_DESC_LAST cpu_to_le32(BIT(22)) -#define MTK_DESC_FIRST cpu_to_le32(BIT(23)) -#define MTK_DESC_BUF_LEN(x) cpu_to_le32(x) -#define MTK_DESC_CT_LEN(x) cpu_to_le32((x) << 24) - -/** - * struct mtk_ring - Descriptor ring - * @cmd_base: pointer to command descriptor ring base - * @cmd_next: pointer to the next command descriptor - * @cmd_dma: DMA address of command descriptor ring - * @res_base: pointer to result descriptor ring base - * @res_next: pointer to the next result descriptor - * @res_prev: pointer to the previous result descriptor - * @res_dma: DMA address of result descriptor ring - * - * A descriptor ring is a circular buffer that is used to manage - * one or more descriptors. There are two type of descriptor rings; - * the command descriptor ring and result descriptor ring. - */ -struct mtk_ring { - struct mtk_desc *cmd_base; - struct mtk_desc *cmd_next; - dma_addr_t cmd_dma; - struct mtk_desc *res_base; - struct mtk_desc *res_next; - struct mtk_desc *res_prev; - dma_addr_t res_dma; -}; - -/** - * struct mtk_aes_dma - Structure that holds sg list info - * @sg: pointer to scatter-gather list - * @nents: number of entries in the sg list - * @remainder: remainder of sg list - * @sg_len: number of entries in the sg mapped list - */ -struct mtk_aes_dma { - struct scatterlist *sg; - int nents; - u32 remainder; - u32 sg_len; -}; - -struct mtk_aes_base_ctx; -struct mtk_aes_rec; -struct mtk_cryp; - -typedef int (*mtk_aes_fn)(struct mtk_cryp *cryp, struct mtk_aes_rec *aes); - -/** - * struct mtk_aes_rec - AES operation record - * @cryp: pointer to Cryptographic device - * @queue: crypto request queue - * @areq: pointer to async request - * @done_task: the tasklet is use in AES interrupt - * @queue_task: the tasklet is used to dequeue request - * @ctx: pointer to current context - * @src: the structure that holds source sg list info - * @dst: the structure that holds destination sg list info - * @aligned_sg: the scatter list is use to alignment - * @real_dst: pointer to the destination sg list - * @resume: pointer to resume function - * @total: request buffer length - * @buf: pointer to page buffer - * @id: the current use of ring - * @flags: it's describing AES operation state - * @lock: the async queue lock - * - * Structure used to record AES execution state. - */ -struct mtk_aes_rec { - struct mtk_cryp *cryp; - struct crypto_queue queue; - struct crypto_async_request *areq; - struct tasklet_struct done_task; - struct tasklet_struct queue_task; - struct mtk_aes_base_ctx *ctx; - struct mtk_aes_dma src; - struct mtk_aes_dma dst; - - struct scatterlist aligned_sg; - struct scatterlist *real_dst; - - mtk_aes_fn resume; - - size_t total; - void *buf; - - u8 id; - unsigned long flags; - /* queue lock */ - spinlock_t lock; -}; - -/** - * struct mtk_sha_rec - SHA operation record - * @cryp: pointer to Cryptographic device - * @queue: crypto request queue - * @req: pointer to ahash request - * @done_task: the tasklet is use in SHA interrupt - * @queue_task: the tasklet is used to dequeue request - * @id: the current use of ring - * @flags: it's describing SHA operation state - * @lock: the async queue lock - * - * Structure used to record SHA execution state. - */ -struct mtk_sha_rec { - struct mtk_cryp *cryp; - struct crypto_queue queue; - struct ahash_request *req; - struct tasklet_struct done_task; - struct tasklet_struct queue_task; - - u8 id; - unsigned long flags; - /* queue lock */ - spinlock_t lock; -}; - -/** - * struct mtk_cryp - Cryptographic device - * @base: pointer to mapped register I/O base - * @dev: pointer to device - * @clk_cryp: pointer to crypto clock - * @irq: global system and rings IRQ - * @ring: pointer to descriptor rings - * @aes: pointer to operation record of AES - * @sha: pointer to operation record of SHA - * @aes_list: device list of AES - * @sha_list: device list of SHA - * @rec: it's used to select SHA record for tfm - * - * Structure storing cryptographic device information. - */ -struct mtk_cryp { - void __iomem *base; - struct device *dev; - struct clk *clk_cryp; - int irq[MTK_IRQ_NUM]; - - struct mtk_ring *ring[MTK_RING_MAX]; - struct mtk_aes_rec *aes[MTK_REC_NUM]; - struct mtk_sha_rec *sha[MTK_REC_NUM]; - - struct list_head aes_list; - struct list_head sha_list; - - bool rec; -}; - -int mtk_cipher_alg_register(struct mtk_cryp *cryp); -void mtk_cipher_alg_release(struct mtk_cryp *cryp); -int mtk_hash_alg_register(struct mtk_cryp *cryp); -void mtk_hash_alg_release(struct mtk_cryp *cryp); - -#endif /* __MTK_PLATFORM_H_ */ diff --git a/drivers/crypto/mediatek/mtk-regs.h b/drivers/crypto/mediatek/mtk-regs.h deleted file mode 100644 index d3defda7a750..000000000000 --- a/drivers/crypto/mediatek/mtk-regs.h +++ /dev/null @@ -1,190 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Support for MediaTek cryptographic accelerator. - * - * Copyright (c) 2016 MediaTek Inc. - * Author: Ryder Lee - */ - -#ifndef __MTK_REGS_H__ -#define __MTK_REGS_H__ - -/* HIA, Command Descriptor Ring Manager */ -#define CDR_BASE_ADDR_LO(x) (0x0 + ((x) << 12)) -#define CDR_BASE_ADDR_HI(x) (0x4 + ((x) << 12)) -#define CDR_DATA_BASE_ADDR_LO(x) (0x8 + ((x) << 12)) -#define CDR_DATA_BASE_ADDR_HI(x) (0xC + ((x) << 12)) -#define CDR_ACD_BASE_ADDR_LO(x) (0x10 + ((x) << 12)) -#define CDR_ACD_BASE_ADDR_HI(x) (0x14 + ((x) << 12)) -#define CDR_RING_SIZE(x) (0x18 + ((x) << 12)) -#define CDR_DESC_SIZE(x) (0x1C + ((x) << 12)) -#define CDR_CFG(x) (0x20 + ((x) << 12)) -#define CDR_DMA_CFG(x) (0x24 + ((x) << 12)) -#define CDR_THRESH(x) (0x28 + ((x) << 12)) -#define CDR_PREP_COUNT(x) (0x2C + ((x) << 12)) -#define CDR_PROC_COUNT(x) (0x30 + ((x) << 12)) -#define CDR_PREP_PNTR(x) (0x34 + ((x) << 12)) -#define CDR_PROC_PNTR(x) (0x38 + ((x) << 12)) -#define CDR_STAT(x) (0x3C + ((x) << 12)) - -/* HIA, Result Descriptor Ring Manager */ -#define RDR_BASE_ADDR_LO(x) (0x800 + ((x) << 12)) -#define RDR_BASE_ADDR_HI(x) (0x804 + ((x) << 12)) -#define RDR_DATA_BASE_ADDR_LO(x) (0x808 + ((x) << 12)) -#define RDR_DATA_BASE_ADDR_HI(x) (0x80C + ((x) << 12)) -#define RDR_ACD_BASE_ADDR_LO(x) (0x810 + ((x) << 12)) -#define RDR_ACD_BASE_ADDR_HI(x) (0x814 + ((x) << 12)) -#define RDR_RING_SIZE(x) (0x818 + ((x) << 12)) -#define RDR_DESC_SIZE(x) (0x81C + ((x) << 12)) -#define RDR_CFG(x) (0x820 + ((x) << 12)) -#define RDR_DMA_CFG(x) (0x824 + ((x) << 12)) -#define RDR_THRESH(x) (0x828 + ((x) << 12)) -#define RDR_PREP_COUNT(x) (0x82C + ((x) << 12)) -#define RDR_PROC_COUNT(x) (0x830 + ((x) << 12)) -#define RDR_PREP_PNTR(x) (0x834 + ((x) << 12)) -#define RDR_PROC_PNTR(x) (0x838 + ((x) << 12)) -#define RDR_STAT(x) (0x83C + ((x) << 12)) - -/* HIA, Ring AIC */ -#define AIC_POL_CTRL(x) (0xE000 - ((x) << 12)) -#define AIC_TYPE_CTRL(x) (0xE004 - ((x) << 12)) -#define AIC_ENABLE_CTRL(x) (0xE008 - ((x) << 12)) -#define AIC_RAW_STAL(x) (0xE00C - ((x) << 12)) -#define AIC_ENABLE_SET(x) (0xE00C - ((x) << 12)) -#define AIC_ENABLED_STAT(x) (0xE010 - ((x) << 12)) -#define AIC_ACK(x) (0xE010 - ((x) << 12)) -#define AIC_ENABLE_CLR(x) (0xE014 - ((x) << 12)) -#define AIC_OPTIONS(x) (0xE018 - ((x) << 12)) -#define AIC_VERSION(x) (0xE01C - ((x) << 12)) - -/* HIA, Global AIC */ -#define AIC_G_POL_CTRL 0xF800 -#define AIC_G_TYPE_CTRL 0xF804 -#define AIC_G_ENABLE_CTRL 0xF808 -#define AIC_G_RAW_STAT 0xF80C -#define AIC_G_ENABLE_SET 0xF80C -#define AIC_G_ENABLED_STAT 0xF810 -#define AIC_G_ACK 0xF810 -#define AIC_G_ENABLE_CLR 0xF814 -#define AIC_G_OPTIONS 0xF818 -#define AIC_G_VERSION 0xF81C - -/* HIA, Data Fetch Engine */ -#define DFE_CFG 0xF000 -#define DFE_PRIO_0 0xF010 -#define DFE_PRIO_1 0xF014 -#define DFE_PRIO_2 0xF018 -#define DFE_PRIO_3 0xF01C - -/* HIA, Data Fetch Engine access monitoring for CDR */ -#define DFE_RING_REGION_LO(x) (0xF080 + ((x) << 3)) -#define DFE_RING_REGION_HI(x) (0xF084 + ((x) << 3)) - -/* HIA, Data Fetch Engine thread control and status for thread */ -#define DFE_THR_CTRL 0xF200 -#define DFE_THR_STAT 0xF204 -#define DFE_THR_DESC_CTRL 0xF208 -#define DFE_THR_DESC_DPTR_LO 0xF210 -#define DFE_THR_DESC_DPTR_HI 0xF214 -#define DFE_THR_DESC_ACDPTR_LO 0xF218 -#define DFE_THR_DESC_ACDPTR_HI 0xF21C - -/* HIA, Data Store Engine */ -#define DSE_CFG 0xF400 -#define DSE_PRIO_0 0xF410 -#define DSE_PRIO_1 0xF414 -#define DSE_PRIO_2 0xF418 -#define DSE_PRIO_3 0xF41C - -/* HIA, Data Store Engine access monitoring for RDR */ -#define DSE_RING_REGION_LO(x) (0xF480 + ((x) << 3)) -#define DSE_RING_REGION_HI(x) (0xF484 + ((x) << 3)) - -/* HIA, Data Store Engine thread control and status for thread */ -#define DSE_THR_CTRL 0xF600 -#define DSE_THR_STAT 0xF604 -#define DSE_THR_DESC_CTRL 0xF608 -#define DSE_THR_DESC_DPTR_LO 0xF610 -#define DSE_THR_DESC_DPTR_HI 0xF614 -#define DSE_THR_DESC_S_DPTR_LO 0xF618 -#define DSE_THR_DESC_S_DPTR_HI 0xF61C -#define DSE_THR_ERROR_STAT 0xF620 - -/* HIA Global */ -#define HIA_MST_CTRL 0xFFF4 -#define HIA_OPTIONS 0xFFF8 -#define HIA_VERSION 0xFFFC - -/* Processing Engine Input Side, Processing Engine */ -#define PE_IN_DBUF_THRESH 0x10000 -#define PE_IN_TBUF_THRESH 0x10100 - -/* Packet Engine Configuration / Status Registers */ -#define PE_TOKEN_CTRL_STAT 0x11000 -#define PE_FUNCTION_EN 0x11004 -#define PE_CONTEXT_CTRL 0x11008 -#define PE_INTERRUPT_CTRL_STAT 0x11010 -#define PE_CONTEXT_STAT 0x1100C -#define PE_OUT_TRANS_CTRL_STAT 0x11018 -#define PE_OUT_BUF_CTRL 0x1101C - -/* Packet Engine PRNG Registers */ -#define PE_PRNG_STAT 0x11040 -#define PE_PRNG_CTRL 0x11044 -#define PE_PRNG_SEED_L 0x11048 -#define PE_PRNG_SEED_H 0x1104C -#define PE_PRNG_KEY_0_L 0x11050 -#define PE_PRNG_KEY_0_H 0x11054 -#define PE_PRNG_KEY_1_L 0x11058 -#define PE_PRNG_KEY_1_H 0x1105C -#define PE_PRNG_RES_0 0x11060 -#define PE_PRNG_RES_1 0x11064 -#define PE_PRNG_RES_2 0x11068 -#define PE_PRNG_RES_3 0x1106C -#define PE_PRNG_LFSR_L 0x11070 -#define PE_PRNG_LFSR_H 0x11074 - -/* Packet Engine AIC */ -#define PE_EIP96_AIC_POL_CTRL 0x113C0 -#define PE_EIP96_AIC_TYPE_CTRL 0x113C4 -#define PE_EIP96_AIC_ENABLE_CTRL 0x113C8 -#define PE_EIP96_AIC_RAW_STAT 0x113CC -#define PE_EIP96_AIC_ENABLE_SET 0x113CC -#define PE_EIP96_AIC_ENABLED_STAT 0x113D0 -#define PE_EIP96_AIC_ACK 0x113D0 -#define PE_EIP96_AIC_ENABLE_CLR 0x113D4 -#define PE_EIP96_AIC_OPTIONS 0x113D8 -#define PE_EIP96_AIC_VERSION 0x113DC - -/* Packet Engine Options & Version Registers */ -#define PE_EIP96_OPTIONS 0x113F8 -#define PE_EIP96_VERSION 0x113FC - -/* Processing Engine Output Side */ -#define PE_OUT_DBUF_THRESH 0x11C00 -#define PE_OUT_TBUF_THRESH 0x11D00 - -/* Processing Engine Local AIC */ -#define PE_AIC_POL_CTRL 0x11F00 -#define PE_AIC_TYPE_CTRL 0x11F04 -#define PE_AIC_ENABLE_CTRL 0x11F08 -#define PE_AIC_RAW_STAT 0x11F0C -#define PE_AIC_ENABLE_SET 0x11F0C -#define PE_AIC_ENABLED_STAT 0x11F10 -#define PE_AIC_ENABLE_CLR 0x11F14 -#define PE_AIC_OPTIONS 0x11F18 -#define PE_AIC_VERSION 0x11F1C - -/* Processing Engine General Configuration and Version */ -#define PE_IN_FLIGHT 0x11FF0 -#define PE_OPTIONS 0x11FF8 -#define PE_VERSION 0x11FFC - -/* EIP-97 - Global */ -#define EIP97_CLOCK_STATE 0x1FFE4 -#define EIP97_FORCE_CLOCK_ON 0x1FFE8 -#define EIP97_FORCE_CLOCK_OFF 0x1FFEC -#define EIP97_MST_CTRL 0x1FFF4 -#define EIP97_OPTIONS 0x1FFF8 -#define EIP97_VERSION 0x1FFFC -#endif /* __MTK_REGS_H__ */ diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c deleted file mode 100644 index f55aacdafbef..000000000000 --- a/drivers/crypto/mediatek/mtk-sha.c +++ /dev/null @@ -1,1353 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Cryptographic API. - * - * Driver for EIP97 SHA1/SHA2(HMAC) acceleration. - * - * Copyright (c) 2016 Ryder Lee - * - * Some ideas are from atmel-sha.c and omap-sham.c drivers. - */ - -#include -#include -#include -#include "mtk-platform.h" - -#define SHA_ALIGN_MSK (sizeof(u32) - 1) -#define SHA_QUEUE_SIZE 512 -#define SHA_BUF_SIZE ((u32)PAGE_SIZE) - -#define SHA_OP_UPDATE 1 -#define SHA_OP_FINAL 2 - -#define SHA_DATA_LEN_MSK cpu_to_le32(GENMASK(16, 0)) -#define SHA_MAX_DIGEST_BUF_SIZE 32 - -/* SHA command token */ -#define SHA_CT_SIZE 5 -#define SHA_CT_CTRL_HDR cpu_to_le32(0x02220000) -#define SHA_CMD0 cpu_to_le32(0x03020000) -#define SHA_CMD1 cpu_to_le32(0x21060000) -#define SHA_CMD2 cpu_to_le32(0xe0e63802) - -/* SHA transform information */ -#define SHA_TFM_HASH cpu_to_le32(0x2 << 0) -#define SHA_TFM_SIZE(x) cpu_to_le32((x) << 8) -#define SHA_TFM_START cpu_to_le32(0x1 << 4) -#define SHA_TFM_CONTINUE cpu_to_le32(0x1 << 5) -#define SHA_TFM_HASH_STORE cpu_to_le32(0x1 << 19) -#define SHA_TFM_SHA1 cpu_to_le32(0x2 << 23) -#define SHA_TFM_SHA256 cpu_to_le32(0x3 << 23) -#define SHA_TFM_SHA224 cpu_to_le32(0x4 << 23) -#define SHA_TFM_SHA512 cpu_to_le32(0x5 << 23) -#define SHA_TFM_SHA384 cpu_to_le32(0x6 << 23) -#define SHA_TFM_DIGEST(x) cpu_to_le32(((x) & GENMASK(3, 0)) << 24) - -/* SHA flags */ -#define SHA_FLAGS_BUSY BIT(0) -#define SHA_FLAGS_FINAL BIT(1) -#define SHA_FLAGS_FINUP BIT(2) -#define SHA_FLAGS_SG BIT(3) -#define SHA_FLAGS_ALGO_MSK GENMASK(8, 4) -#define SHA_FLAGS_SHA1 BIT(4) -#define SHA_FLAGS_SHA224 BIT(5) -#define SHA_FLAGS_SHA256 BIT(6) -#define SHA_FLAGS_SHA384 BIT(7) -#define SHA_FLAGS_SHA512 BIT(8) -#define SHA_FLAGS_HMAC BIT(9) -#define SHA_FLAGS_PAD BIT(10) - -/** - * mtk_sha_info - hardware information of AES - * @cmd: command token, hardware instruction - * @tfm: transform state of cipher algorithm. - * @state: contains keys and initial vectors. - * - */ -struct mtk_sha_info { - __le32 ctrl[2]; - __le32 cmd[3]; - __le32 tfm[2]; - __le32 digest[SHA_MAX_DIGEST_BUF_SIZE]; -}; - -struct mtk_sha_reqctx { - struct mtk_sha_info info; - unsigned long flags; - unsigned long op; - - u64 digcnt; - size_t bufcnt; - dma_addr_t dma_addr; - - __le32 ct_hdr; - u32 ct_size; - dma_addr_t ct_dma; - dma_addr_t tfm_dma; - - /* Walk state */ - struct scatterlist *sg; - u32 offset; /* Offset in current sg */ - u32 total; /* Total request */ - size_t ds; - size_t bs; - - u8 *buffer; -}; - -struct mtk_sha_hmac_ctx { - struct crypto_shash *shash; - u8 ipad[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); - u8 opad[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); -}; - -struct mtk_sha_ctx { - struct mtk_cryp *cryp; - unsigned long flags; - u8 id; - u8 buf[SHA_BUF_SIZE] __aligned(sizeof(u32)); - - struct mtk_sha_hmac_ctx base[]; -}; - -struct mtk_sha_drv { - struct list_head dev_list; - /* Device list lock */ - spinlock_t lock; -}; - -static struct mtk_sha_drv mtk_sha = { - .dev_list = LIST_HEAD_INIT(mtk_sha.dev_list), - .lock = __SPIN_LOCK_UNLOCKED(mtk_sha.lock), -}; - -static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id, - struct ahash_request *req); - -static inline u32 mtk_sha_read(struct mtk_cryp *cryp, u32 offset) -{ - return readl_relaxed(cryp->base + offset); -} - -static inline void mtk_sha_write(struct mtk_cryp *cryp, - u32 offset, u32 value) -{ - writel_relaxed(value, cryp->base + offset); -} - -static inline void mtk_sha_ring_shift(struct mtk_ring *ring, - struct mtk_desc **cmd_curr, - struct mtk_desc **res_curr, - int *count) -{ - *cmd_curr = ring->cmd_next++; - *res_curr = ring->res_next++; - (*count)++; - - if (ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) { - ring->cmd_next = ring->cmd_base; - ring->res_next = ring->res_base; - } -} - -static struct mtk_cryp *mtk_sha_find_dev(struct mtk_sha_ctx *tctx) -{ - struct mtk_cryp *cryp = NULL; - struct mtk_cryp *tmp; - - spin_lock_bh(&mtk_sha.lock); - if (!tctx->cryp) { - list_for_each_entry(tmp, &mtk_sha.dev_list, sha_list) { - cryp = tmp; - break; - } - tctx->cryp = cryp; - } else { - cryp = tctx->cryp; - } - - /* - * Assign record id to tfm in round-robin fashion, and this - * will help tfm to bind to corresponding descriptor rings. - */ - tctx->id = cryp->rec; - cryp->rec = !cryp->rec; - - spin_unlock_bh(&mtk_sha.lock); - - return cryp; -} - -static int mtk_sha_append_sg(struct mtk_sha_reqctx *ctx) -{ - size_t count; - - while ((ctx->bufcnt < SHA_BUF_SIZE) && ctx->total) { - count = min(ctx->sg->length - ctx->offset, ctx->total); - count = min(count, SHA_BUF_SIZE - ctx->bufcnt); - - if (count <= 0) { - /* - * Check if count <= 0 because the buffer is full or - * because the sg length is 0. In the latest case, - * check if there is another sg in the list, a 0 length - * sg doesn't necessarily mean the end of the sg list. - */ - if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) { - ctx->sg = sg_next(ctx->sg); - continue; - } else { - break; - } - } - - scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg, - ctx->offset, count, 0); - - ctx->bufcnt += count; - ctx->offset += count; - ctx->total -= count; - - if (ctx->offset == ctx->sg->length) { - ctx->sg = sg_next(ctx->sg); - if (ctx->sg) - ctx->offset = 0; - else - ctx->total = 0; - } - } - - return 0; -} - -/* - * The purpose of this padding is to ensure that the padded message is a - * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). - * The bit "1" is appended at the end of the message followed by - * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or - * 128 bits block (SHA384/SHA512) equals to the message length in bits - * is appended. - * - * For SHA1/SHA224/SHA256, padlen is calculated as followed: - * - if message length < 56 bytes then padlen = 56 - message length - * - else padlen = 64 + 56 - message length - * - * For SHA384/SHA512, padlen is calculated as followed: - * - if message length < 112 bytes then padlen = 112 - message length - * - else padlen = 128 + 112 - message length - */ -static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) -{ - u32 index, padlen; - __be64 bits[2]; - u64 size = ctx->digcnt; - - size += ctx->bufcnt; - size += len; - - bits[1] = cpu_to_be64(size << 3); - bits[0] = cpu_to_be64(size >> 61); - - switch (ctx->flags & SHA_FLAGS_ALGO_MSK) { - case SHA_FLAGS_SHA384: - case SHA_FLAGS_SHA512: - index = ctx->bufcnt & 0x7f; - padlen = (index < 112) ? (112 - index) : ((128 + 112) - index); - *(ctx->buffer + ctx->bufcnt) = 0x80; - memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen - 1); - memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); - ctx->bufcnt += padlen + 16; - ctx->flags |= SHA_FLAGS_PAD; - break; - - default: - index = ctx->bufcnt & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); - *(ctx->buffer + ctx->bufcnt) = 0x80; - memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen - 1); - memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); - ctx->bufcnt += padlen + 8; - ctx->flags |= SHA_FLAGS_PAD; - break; - } -} - -/* Initialize basic transform information of SHA */ -static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx) -{ - struct mtk_sha_info *info = &ctx->info; - - ctx->ct_hdr = SHA_CT_CTRL_HDR; - ctx->ct_size = SHA_CT_SIZE; - - info->tfm[0] = SHA_TFM_HASH | SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds)); - - switch (ctx->flags & SHA_FLAGS_ALGO_MSK) { - case SHA_FLAGS_SHA1: - info->tfm[0] |= SHA_TFM_SHA1; - break; - case SHA_FLAGS_SHA224: - info->tfm[0] |= SHA_TFM_SHA224; - break; - case SHA_FLAGS_SHA256: - info->tfm[0] |= SHA_TFM_SHA256; - break; - case SHA_FLAGS_SHA384: - info->tfm[0] |= SHA_TFM_SHA384; - break; - case SHA_FLAGS_SHA512: - info->tfm[0] |= SHA_TFM_SHA512; - break; - - default: - /* Should not happen... */ - return; - } - - info->tfm[1] = SHA_TFM_HASH_STORE; - info->ctrl[0] = info->tfm[0] | SHA_TFM_CONTINUE | SHA_TFM_START; - info->ctrl[1] = info->tfm[1]; - - info->cmd[0] = SHA_CMD0; - info->cmd[1] = SHA_CMD1; - info->cmd[2] = SHA_CMD2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds)); -} - -/* - * Update input data length field of transform information and - * map it to DMA region. - */ -static int mtk_sha_info_update(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha, - size_t len1, size_t len2) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - struct mtk_sha_info *info = &ctx->info; - - ctx->ct_hdr &= ~SHA_DATA_LEN_MSK; - ctx->ct_hdr |= cpu_to_le32(len1 + len2); - info->cmd[0] &= ~SHA_DATA_LEN_MSK; - info->cmd[0] |= cpu_to_le32(len1 + len2); - - /* Setting SHA_TFM_START only for the first iteration */ - if (ctx->digcnt) - info->ctrl[0] &= ~SHA_TFM_START; - - ctx->digcnt += len1; - - ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info), - DMA_BIDIRECTIONAL); - if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) { - dev_err(cryp->dev, "dma %zu bytes error\n", sizeof(*info)); - return -EINVAL; - } - - ctx->tfm_dma = ctx->ct_dma + sizeof(info->ctrl) + sizeof(info->cmd); - - return 0; -} - -/* - * Because of hardware limitation, we must pre-calculate the inner - * and outer digest that need to be processed firstly by engine, then - * apply the result digest to the input message. These complex hashing - * procedures limits HMAC performance, so we use fallback SW encoding. - */ -static int mtk_sha_finish_hmac(struct ahash_request *req) -{ - struct mtk_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); - struct mtk_sha_hmac_ctx *bctx = tctx->base; - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - SHASH_DESC_ON_STACK(shash, bctx->shash); - - shash->tfm = bctx->shash; - - return crypto_shash_init(shash) ?: - crypto_shash_update(shash, bctx->opad, ctx->bs) ?: - crypto_shash_finup(shash, req->result, ctx->ds, req->result); -} - -/* Initialize request context */ -static int mtk_sha_init(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm); - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - ctx->flags = 0; - ctx->ds = crypto_ahash_digestsize(tfm); - - switch (ctx->ds) { - case SHA1_DIGEST_SIZE: - ctx->flags |= SHA_FLAGS_SHA1; - ctx->bs = SHA1_BLOCK_SIZE; - break; - case SHA224_DIGEST_SIZE: - ctx->flags |= SHA_FLAGS_SHA224; - ctx->bs = SHA224_BLOCK_SIZE; - break; - case SHA256_DIGEST_SIZE: - ctx->flags |= SHA_FLAGS_SHA256; - ctx->bs = SHA256_BLOCK_SIZE; - break; - case SHA384_DIGEST_SIZE: - ctx->flags |= SHA_FLAGS_SHA384; - ctx->bs = SHA384_BLOCK_SIZE; - break; - case SHA512_DIGEST_SIZE: - ctx->flags |= SHA_FLAGS_SHA512; - ctx->bs = SHA512_BLOCK_SIZE; - break; - default: - return -EINVAL; - } - - ctx->bufcnt = 0; - ctx->digcnt = 0; - ctx->buffer = tctx->buf; - - if (tctx->flags & SHA_FLAGS_HMAC) { - struct mtk_sha_hmac_ctx *bctx = tctx->base; - - memcpy(ctx->buffer, bctx->ipad, ctx->bs); - ctx->bufcnt = ctx->bs; - ctx->flags |= SHA_FLAGS_HMAC; - } - - return 0; -} - -static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, - dma_addr_t addr1, size_t len1, - dma_addr_t addr2, size_t len2) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - struct mtk_ring *ring = cryp->ring[sha->id]; - struct mtk_desc *cmd, *res; - int err, count = 0; - - err = mtk_sha_info_update(cryp, sha, len1, len2); - if (err) - return err; - - /* Fill in the command/result descriptors */ - mtk_sha_ring_shift(ring, &cmd, &res, &count); - - res->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1); - cmd->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1) | - MTK_DESC_CT_LEN(ctx->ct_size); - cmd->buf = cpu_to_le32(addr1); - cmd->ct = cpu_to_le32(ctx->ct_dma); - cmd->ct_hdr = ctx->ct_hdr; - cmd->tfm = cpu_to_le32(ctx->tfm_dma); - - if (len2) { - mtk_sha_ring_shift(ring, &cmd, &res, &count); - - res->hdr = MTK_DESC_BUF_LEN(len2); - cmd->hdr = MTK_DESC_BUF_LEN(len2); - cmd->buf = cpu_to_le32(addr2); - } - - cmd->hdr |= MTK_DESC_LAST; - res->hdr |= MTK_DESC_LAST; - - /* - * Make sure that all changes to the DMA ring are done before we - * start engine. - */ - wmb(); - /* Start DMA transfer */ - mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count)); - mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count)); - - return -EINPROGRESS; -} - -static int mtk_sha_dma_map(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha, - struct mtk_sha_reqctx *ctx, - size_t count) -{ - ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer, - SHA_BUF_SIZE, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) { - dev_err(cryp->dev, "dma map error\n"); - return -EINVAL; - } - - ctx->flags &= ~SHA_FLAGS_SG; - - return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count, 0, 0); -} - -static int mtk_sha_update_slow(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - size_t count; - u32 final; - - mtk_sha_append_sg(ctx); - - final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; - - dev_dbg(cryp->dev, "slow: bufcnt: %zu\n", ctx->bufcnt); - - if (final) { - sha->flags |= SHA_FLAGS_FINAL; - mtk_sha_fill_padding(ctx, 0); - } - - if (final || (ctx->bufcnt == SHA_BUF_SIZE && ctx->total)) { - count = ctx->bufcnt; - ctx->bufcnt = 0; - - return mtk_sha_dma_map(cryp, sha, ctx, count); - } - return 0; -} - -static int mtk_sha_update_start(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - u32 len, final, tail; - struct scatterlist *sg; - - if (!ctx->total) - return 0; - - if (ctx->bufcnt || ctx->offset) - return mtk_sha_update_slow(cryp, sha); - - sg = ctx->sg; - - if (!IS_ALIGNED(sg->offset, sizeof(u32))) - return mtk_sha_update_slow(cryp, sha); - - if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->bs)) - /* size is not ctx->bs aligned */ - return mtk_sha_update_slow(cryp, sha); - - len = min(ctx->total, sg->length); - - if (sg_is_last(sg)) { - if (!(ctx->flags & SHA_FLAGS_FINUP)) { - /* not last sg must be ctx->bs aligned */ - tail = len & (ctx->bs - 1); - len -= tail; - } - } - - ctx->total -= len; - ctx->offset = len; /* offset where to start slow */ - - final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; - - /* Add padding */ - if (final) { - size_t count; - - tail = len & (ctx->bs - 1); - len -= tail; - ctx->total += tail; - ctx->offset = len; /* offset where to start slow */ - - sg = ctx->sg; - mtk_sha_append_sg(ctx); - mtk_sha_fill_padding(ctx, len); - - ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer, - SHA_BUF_SIZE, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) { - dev_err(cryp->dev, "dma map bytes error\n"); - return -EINVAL; - } - - sha->flags |= SHA_FLAGS_FINAL; - count = ctx->bufcnt; - ctx->bufcnt = 0; - - if (len == 0) { - ctx->flags &= ~SHA_FLAGS_SG; - return mtk_sha_xmit(cryp, sha, ctx->dma_addr, - count, 0, 0); - - } else { - ctx->sg = sg; - if (!dma_map_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE)) { - dev_err(cryp->dev, "dma_map_sg error\n"); - return -EINVAL; - } - - ctx->flags |= SHA_FLAGS_SG; - return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), - len, ctx->dma_addr, count); - } - } - - if (!dma_map_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE)) { - dev_err(cryp->dev, "dma_map_sg error\n"); - return -EINVAL; - } - - ctx->flags |= SHA_FLAGS_SG; - - return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), - len, 0, 0); -} - -static int mtk_sha_final_req(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - size_t count; - - mtk_sha_fill_padding(ctx, 0); - - sha->flags |= SHA_FLAGS_FINAL; - count = ctx->bufcnt; - ctx->bufcnt = 0; - - return mtk_sha_dma_map(cryp, sha, ctx, count); -} - -/* Copy ready hash (+ finalize hmac) */ -static int mtk_sha_finish(struct ahash_request *req) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - __le32 *digest = ctx->info.digest; - u32 *result = (u32 *)req->result; - int i; - - /* Get the hash from the digest buffer */ - for (i = 0; i < SIZE_IN_WORDS(ctx->ds); i++) - result[i] = le32_to_cpu(digest[i]); - - if (ctx->flags & SHA_FLAGS_HMAC) - return mtk_sha_finish_hmac(req); - - return 0; -} - -static void mtk_sha_finish_req(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha, - int err) -{ - if (likely(!err && (SHA_FLAGS_FINAL & sha->flags))) - err = mtk_sha_finish(sha->req); - - sha->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL); - - sha->req->base.complete(&sha->req->base, err); - - /* Handle new request */ - tasklet_schedule(&sha->queue_task); -} - -static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id, - struct ahash_request *req) -{ - struct mtk_sha_rec *sha = cryp->sha[id]; - struct crypto_async_request *async_req, *backlog; - struct mtk_sha_reqctx *ctx; - unsigned long flags; - int err = 0, ret = 0; - - spin_lock_irqsave(&sha->lock, flags); - if (req) - ret = ahash_enqueue_request(&sha->queue, req); - - if (SHA_FLAGS_BUSY & sha->flags) { - spin_unlock_irqrestore(&sha->lock, flags); - return ret; - } - - backlog = crypto_get_backlog(&sha->queue); - async_req = crypto_dequeue_request(&sha->queue); - if (async_req) - sha->flags |= SHA_FLAGS_BUSY; - spin_unlock_irqrestore(&sha->lock, flags); - - if (!async_req) - return ret; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - req = ahash_request_cast(async_req); - ctx = ahash_request_ctx(req); - - sha->req = req; - - mtk_sha_info_init(ctx); - - if (ctx->op == SHA_OP_UPDATE) { - err = mtk_sha_update_start(cryp, sha); - if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP)) - /* No final() after finup() */ - err = mtk_sha_final_req(cryp, sha); - } else if (ctx->op == SHA_OP_FINAL) { - err = mtk_sha_final_req(cryp, sha); - } - - if (unlikely(err != -EINPROGRESS)) - /* Task will not finish it, so do it here */ - mtk_sha_finish_req(cryp, sha, err); - - return ret; -} - -static int mtk_sha_enqueue(struct ahash_request *req, u32 op) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - struct mtk_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); - - ctx->op = op; - - return mtk_sha_handle_queue(tctx->cryp, tctx->id, req); -} - -static void mtk_sha_unmap(struct mtk_cryp *cryp, struct mtk_sha_rec *sha) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - - dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info), - DMA_BIDIRECTIONAL); - - if (ctx->flags & SHA_FLAGS_SG) { - dma_unmap_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE); - if (ctx->sg->length == ctx->offset) { - ctx->sg = sg_next(ctx->sg); - if (ctx->sg) - ctx->offset = 0; - } - if (ctx->flags & SHA_FLAGS_PAD) { - dma_unmap_single(cryp->dev, ctx->dma_addr, - SHA_BUF_SIZE, DMA_TO_DEVICE); - } - } else - dma_unmap_single(cryp->dev, ctx->dma_addr, - SHA_BUF_SIZE, DMA_TO_DEVICE); -} - -static void mtk_sha_complete(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha) -{ - int err = 0; - - err = mtk_sha_update_start(cryp, sha); - if (err != -EINPROGRESS) - mtk_sha_finish_req(cryp, sha, err); -} - -static int mtk_sha_update(struct ahash_request *req) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - ctx->total = req->nbytes; - ctx->sg = req->src; - ctx->offset = 0; - - if ((ctx->bufcnt + ctx->total < SHA_BUF_SIZE) && - !(ctx->flags & SHA_FLAGS_FINUP)) - return mtk_sha_append_sg(ctx); - - return mtk_sha_enqueue(req, SHA_OP_UPDATE); -} - -static int mtk_sha_final(struct ahash_request *req) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - ctx->flags |= SHA_FLAGS_FINUP; - - if (ctx->flags & SHA_FLAGS_PAD) - return mtk_sha_finish(req); - - return mtk_sha_enqueue(req, SHA_OP_FINAL); -} - -static int mtk_sha_finup(struct ahash_request *req) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - int err1, err2; - - ctx->flags |= SHA_FLAGS_FINUP; - - err1 = mtk_sha_update(req); - if (err1 == -EINPROGRESS || - (err1 == -EBUSY && (ahash_request_flags(req) & - CRYPTO_TFM_REQ_MAY_BACKLOG))) - return err1; - /* - * final() has to be always called to cleanup resources - * even if update() failed - */ - err2 = mtk_sha_final(req); - - return err1 ?: err2; -} - -static int mtk_sha_digest(struct ahash_request *req) -{ - return mtk_sha_init(req) ?: mtk_sha_finup(req); -} - -static int mtk_sha_setkey(struct crypto_ahash *tfm, const u8 *key, - u32 keylen) -{ - struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm); - struct mtk_sha_hmac_ctx *bctx = tctx->base; - size_t bs = crypto_shash_blocksize(bctx->shash); - size_t ds = crypto_shash_digestsize(bctx->shash); - int err, i; - - if (keylen > bs) { - err = crypto_shash_tfm_digest(bctx->shash, key, keylen, - bctx->ipad); - if (err) - return err; - keylen = ds; - } else { - memcpy(bctx->ipad, key, keylen); - } - - memset(bctx->ipad + keylen, 0, bs - keylen); - memcpy(bctx->opad, bctx->ipad, bs); - - for (i = 0; i < bs; i++) { - bctx->ipad[i] ^= HMAC_IPAD_VALUE; - bctx->opad[i] ^= HMAC_OPAD_VALUE; - } - - return 0; -} - -static int mtk_sha_export(struct ahash_request *req, void *out) -{ - const struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - memcpy(out, ctx, sizeof(*ctx)); - return 0; -} - -static int mtk_sha_import(struct ahash_request *req, const void *in) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - memcpy(ctx, in, sizeof(*ctx)); - return 0; -} - -static int mtk_sha_cra_init_alg(struct crypto_tfm *tfm, - const char *alg_base) -{ - struct mtk_sha_ctx *tctx = crypto_tfm_ctx(tfm); - struct mtk_cryp *cryp = NULL; - - cryp = mtk_sha_find_dev(tctx); - if (!cryp) - return -ENODEV; - - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct mtk_sha_reqctx)); - - if (alg_base) { - struct mtk_sha_hmac_ctx *bctx = tctx->base; - - tctx->flags |= SHA_FLAGS_HMAC; - bctx->shash = crypto_alloc_shash(alg_base, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(bctx->shash)) { - pr_err("base driver %s could not be loaded.\n", - alg_base); - - return PTR_ERR(bctx->shash); - } - } - return 0; -} - -static int mtk_sha_cra_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, NULL); -} - -static int mtk_sha_cra_sha1_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, "sha1"); -} - -static int mtk_sha_cra_sha224_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, "sha224"); -} - -static int mtk_sha_cra_sha256_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, "sha256"); -} - -static int mtk_sha_cra_sha384_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, "sha384"); -} - -static int mtk_sha_cra_sha512_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, "sha512"); -} - -static void mtk_sha_cra_exit(struct crypto_tfm *tfm) -{ - struct mtk_sha_ctx *tctx = crypto_tfm_ctx(tfm); - - if (tctx->flags & SHA_FLAGS_HMAC) { - struct mtk_sha_hmac_ctx *bctx = tctx->base; - - crypto_free_shash(bctx->shash); - } -} - -static struct ahash_alg algs_sha1_sha224_sha256[] = { -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .halg.digestsize = SHA1_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "sha1", - .cra_driver_name = "mtk-sha1", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .halg.digestsize = SHA224_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "sha224", - .cra_driver_name = "mtk-sha224", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .halg.digestsize = SHA256_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "sha256", - .cra_driver_name = "mtk-sha256", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .setkey = mtk_sha_setkey, - .halg.digestsize = SHA1_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "hmac(sha1)", - .cra_driver_name = "mtk-hmac-sha1", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx) + - sizeof(struct mtk_sha_hmac_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_sha1_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .setkey = mtk_sha_setkey, - .halg.digestsize = SHA224_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "hmac(sha224)", - .cra_driver_name = "mtk-hmac-sha224", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx) + - sizeof(struct mtk_sha_hmac_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_sha224_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .setkey = mtk_sha_setkey, - .halg.digestsize = SHA256_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "hmac(sha256)", - .cra_driver_name = "mtk-hmac-sha256", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx) + - sizeof(struct mtk_sha_hmac_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_sha256_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -}; - -static struct ahash_alg algs_sha384_sha512[] = { -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .halg.digestsize = SHA384_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "sha384", - .cra_driver_name = "mtk-sha384", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .halg.digestsize = SHA512_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "sha512", - .cra_driver_name = "mtk-sha512", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .setkey = mtk_sha_setkey, - .halg.digestsize = SHA384_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "hmac(sha384)", - .cra_driver_name = "mtk-hmac-sha384", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx) + - sizeof(struct mtk_sha_hmac_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_sha384_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .setkey = mtk_sha_setkey, - .halg.digestsize = SHA512_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "hmac(sha512)", - .cra_driver_name = "mtk-hmac-sha512", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx) + - sizeof(struct mtk_sha_hmac_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_sha512_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -}; - -static void mtk_sha_queue_task(unsigned long data) -{ - struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; - - mtk_sha_handle_queue(sha->cryp, sha->id - MTK_RING2, NULL); -} - -static void mtk_sha_done_task(unsigned long data) -{ - struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; - struct mtk_cryp *cryp = sha->cryp; - - mtk_sha_unmap(cryp, sha); - mtk_sha_complete(cryp, sha); -} - -static irqreturn_t mtk_sha_irq(int irq, void *dev_id) -{ - struct mtk_sha_rec *sha = (struct mtk_sha_rec *)dev_id; - struct mtk_cryp *cryp = sha->cryp; - u32 val = mtk_sha_read(cryp, RDR_STAT(sha->id)); - - mtk_sha_write(cryp, RDR_STAT(sha->id), val); - - if (likely((SHA_FLAGS_BUSY & sha->flags))) { - mtk_sha_write(cryp, RDR_PROC_COUNT(sha->id), MTK_CNT_RST); - mtk_sha_write(cryp, RDR_THRESH(sha->id), - MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - - tasklet_schedule(&sha->done_task); - } else { - dev_warn(cryp->dev, "SHA interrupt when no active requests.\n"); - } - return IRQ_HANDLED; -} - -/* - * The purpose of two SHA records is used to get extra performance. - * It is similar to mtk_aes_record_init(). - */ -static int mtk_sha_record_init(struct mtk_cryp *cryp) -{ - struct mtk_sha_rec **sha = cryp->sha; - int i, err = -ENOMEM; - - for (i = 0; i < MTK_REC_NUM; i++) { - sha[i] = kzalloc(sizeof(**sha), GFP_KERNEL); - if (!sha[i]) - goto err_cleanup; - - sha[i]->cryp = cryp; - - spin_lock_init(&sha[i]->lock); - crypto_init_queue(&sha[i]->queue, SHA_QUEUE_SIZE); - - tasklet_init(&sha[i]->queue_task, mtk_sha_queue_task, - (unsigned long)sha[i]); - tasklet_init(&sha[i]->done_task, mtk_sha_done_task, - (unsigned long)sha[i]); - } - - /* Link to ring2 and ring3 respectively */ - sha[0]->id = MTK_RING2; - sha[1]->id = MTK_RING3; - - cryp->rec = 1; - - return 0; - -err_cleanup: - for (; i--; ) - kfree(sha[i]); - return err; -} - -static void mtk_sha_record_free(struct mtk_cryp *cryp) -{ - int i; - - for (i = 0; i < MTK_REC_NUM; i++) { - tasklet_kill(&cryp->sha[i]->done_task); - tasklet_kill(&cryp->sha[i]->queue_task); - - kfree(cryp->sha[i]); - } -} - -static void mtk_sha_unregister_algs(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(algs_sha1_sha224_sha256); i++) - crypto_unregister_ahash(&algs_sha1_sha224_sha256[i]); - - for (i = 0; i < ARRAY_SIZE(algs_sha384_sha512); i++) - crypto_unregister_ahash(&algs_sha384_sha512[i]); -} - -static int mtk_sha_register_algs(void) -{ - int err, i; - - for (i = 0; i < ARRAY_SIZE(algs_sha1_sha224_sha256); i++) { - err = crypto_register_ahash(&algs_sha1_sha224_sha256[i]); - if (err) - goto err_sha_224_256_algs; - } - - for (i = 0; i < ARRAY_SIZE(algs_sha384_sha512); i++) { - err = crypto_register_ahash(&algs_sha384_sha512[i]); - if (err) - goto err_sha_384_512_algs; - } - - return 0; - -err_sha_384_512_algs: - for (; i--; ) - crypto_unregister_ahash(&algs_sha384_sha512[i]); - i = ARRAY_SIZE(algs_sha1_sha224_sha256); -err_sha_224_256_algs: - for (; i--; ) - crypto_unregister_ahash(&algs_sha1_sha224_sha256[i]); - - return err; -} - -int mtk_hash_alg_register(struct mtk_cryp *cryp) -{ - int err; - - INIT_LIST_HEAD(&cryp->sha_list); - - /* Initialize two hash records */ - err = mtk_sha_record_init(cryp); - if (err) - goto err_record; - - err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING2], mtk_sha_irq, - 0, "mtk-sha", cryp->sha[0]); - if (err) { - dev_err(cryp->dev, "unable to request sha irq0.\n"); - goto err_res; - } - - err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING3], mtk_sha_irq, - 0, "mtk-sha", cryp->sha[1]); - if (err) { - dev_err(cryp->dev, "unable to request sha irq1.\n"); - goto err_res; - } - - /* Enable ring2 and ring3 interrupt for hash */ - mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING2), MTK_IRQ_RDR2); - mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING3), MTK_IRQ_RDR3); - - spin_lock(&mtk_sha.lock); - list_add_tail(&cryp->sha_list, &mtk_sha.dev_list); - spin_unlock(&mtk_sha.lock); - - err = mtk_sha_register_algs(); - if (err) - goto err_algs; - - return 0; - -err_algs: - spin_lock(&mtk_sha.lock); - list_del(&cryp->sha_list); - spin_unlock(&mtk_sha.lock); -err_res: - mtk_sha_record_free(cryp); -err_record: - - dev_err(cryp->dev, "mtk-sha initialization failed.\n"); - return err; -} - -void mtk_hash_alg_release(struct mtk_cryp *cryp) -{ - spin_lock(&mtk_sha.lock); - list_del(&cryp->sha_list); - spin_unlock(&mtk_sha.lock); - - mtk_sha_unregister_algs(); - mtk_sha_record_free(cryp); -} diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c deleted file mode 100644 index 84f9c16d984c..000000000000 --- a/drivers/crypto/picoxcell_crypto.c +++ /dev/null @@ -1,1807 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (c) 2010-2011 Picochip Ltd., Jamie Iles - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "picoxcell_crypto_regs.h" - -/* - * The threshold for the number of entries in the CMD FIFO available before - * the CMD0_CNT interrupt is raised. Increasing this value will reduce the - * number of interrupts raised to the CPU. - */ -#define CMD0_IRQ_THRESHOLD 1 - -/* - * The timeout period (in jiffies) for a PDU. When the the number of PDUs in - * flight is greater than the STAT_IRQ_THRESHOLD or 0 the timer is disabled. - * When there are packets in flight but lower than the threshold, we enable - * the timer and at expiry, attempt to remove any processed packets from the - * queue and if there are still packets left, schedule the timer again. - */ -#define PACKET_TIMEOUT 1 - -/* The priority to register each algorithm with. */ -#define SPACC_CRYPTO_ALG_PRIORITY 10000 - -#define SPACC_CRYPTO_KASUMI_F8_KEY_LEN 16 -#define SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ 64 -#define SPACC_CRYPTO_IPSEC_HASH_PG_SZ 64 -#define SPACC_CRYPTO_IPSEC_MAX_CTXS 32 -#define SPACC_CRYPTO_IPSEC_FIFO_SZ 32 -#define SPACC_CRYPTO_L2_CIPHER_PG_SZ 64 -#define SPACC_CRYPTO_L2_HASH_PG_SZ 64 -#define SPACC_CRYPTO_L2_MAX_CTXS 128 -#define SPACC_CRYPTO_L2_FIFO_SZ 128 - -#define MAX_DDT_LEN 16 - -/* DDT format. This must match the hardware DDT format exactly. */ -struct spacc_ddt { - dma_addr_t p; - u32 len; -}; - -/* - * Asynchronous crypto request structure. - * - * This structure defines a request that is either queued for processing or - * being processed. - */ -struct spacc_req { - struct list_head list; - struct spacc_engine *engine; - struct crypto_async_request *req; - int result; - bool is_encrypt; - unsigned ctx_id; - dma_addr_t src_addr, dst_addr; - struct spacc_ddt *src_ddt, *dst_ddt; - void (*complete)(struct spacc_req *req); - struct skcipher_request fallback_req; // keep at the end -}; - -struct spacc_aead { - unsigned long ctrl_default; - unsigned long type; - struct aead_alg alg; - struct spacc_engine *engine; - struct list_head entry; - int key_offs; - int iv_offs; -}; - -struct spacc_engine { - void __iomem *regs; - struct list_head pending; - int next_ctx; - spinlock_t hw_lock; - int in_flight; - struct list_head completed; - struct list_head in_progress; - struct tasklet_struct complete; - unsigned long fifo_sz; - void __iomem *cipher_ctx_base; - void __iomem *hash_key_base; - struct spacc_alg *algs; - unsigned num_algs; - struct list_head registered_algs; - struct spacc_aead *aeads; - unsigned num_aeads; - struct list_head registered_aeads; - size_t cipher_pg_sz; - size_t hash_pg_sz; - const char *name; - struct clk *clk; - struct device *dev; - unsigned max_ctxs; - struct timer_list packet_timeout; - unsigned stat_irq_thresh; - struct dma_pool *req_pool; -}; - -/* Algorithm type mask. */ -#define SPACC_CRYPTO_ALG_MASK 0x7 - -/* SPACC definition of a crypto algorithm. */ -struct spacc_alg { - unsigned long ctrl_default; - unsigned long type; - struct skcipher_alg alg; - struct spacc_engine *engine; - struct list_head entry; - int key_offs; - int iv_offs; -}; - -/* Generic context structure for any algorithm type. */ -struct spacc_generic_ctx { - struct spacc_engine *engine; - int flags; - int key_offs; - int iv_offs; -}; - -/* Block cipher context. */ -struct spacc_ablk_ctx { - struct spacc_generic_ctx generic; - u8 key[AES_MAX_KEY_SIZE]; - u8 key_len; - /* - * The fallback cipher. If the operation can't be done in hardware, - * fallback to a software version. - */ - struct crypto_skcipher *sw_cipher; -}; - -/* AEAD cipher context. */ -struct spacc_aead_ctx { - struct spacc_generic_ctx generic; - u8 cipher_key[AES_MAX_KEY_SIZE]; - u8 hash_ctx[SPACC_CRYPTO_IPSEC_HASH_PG_SZ]; - u8 cipher_key_len; - u8 hash_key_len; - struct crypto_aead *sw_cipher; -}; - -static int spacc_ablk_submit(struct spacc_req *req); - -static inline struct spacc_alg *to_spacc_skcipher(struct skcipher_alg *alg) -{ - return alg ? container_of(alg, struct spacc_alg, alg) : NULL; -} - -static inline struct spacc_aead *to_spacc_aead(struct aead_alg *alg) -{ - return container_of(alg, struct spacc_aead, alg); -} - -static inline int spacc_fifo_cmd_full(struct spacc_engine *engine) -{ - u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET); - - return fifo_stat & SPA_FIFO_CMD_FULL; -} - -/* - * Given a cipher context, and a context number, get the base address of the - * context page. - * - * Returns the address of the context page where the key/context may - * be written. - */ -static inline void __iomem *spacc_ctx_page_addr(struct spacc_generic_ctx *ctx, - unsigned indx, - bool is_cipher_ctx) -{ - return is_cipher_ctx ? ctx->engine->cipher_ctx_base + - (indx * ctx->engine->cipher_pg_sz) : - ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz); -} - -/* The context pages can only be written with 32-bit accesses. */ -static inline void memcpy_toio32(u32 __iomem *dst, const void *src, - unsigned count) -{ - const u32 *src32 = (const u32 *) src; - - while (count--) - writel(*src32++, dst++); -} - -static void spacc_cipher_write_ctx(struct spacc_generic_ctx *ctx, - void __iomem *page_addr, const u8 *key, - size_t key_len, const u8 *iv, size_t iv_len) -{ - void __iomem *key_ptr = page_addr + ctx->key_offs; - void __iomem *iv_ptr = page_addr + ctx->iv_offs; - - memcpy_toio32(key_ptr, key, key_len / 4); - memcpy_toio32(iv_ptr, iv, iv_len / 4); -} - -/* - * Load a context into the engines context memory. - * - * Returns the index of the context page where the context was loaded. - */ -static unsigned spacc_load_ctx(struct spacc_generic_ctx *ctx, - const u8 *ciph_key, size_t ciph_len, - const u8 *iv, size_t ivlen, const u8 *hash_key, - size_t hash_len) -{ - unsigned indx = ctx->engine->next_ctx++; - void __iomem *ciph_page_addr, *hash_page_addr; - - ciph_page_addr = spacc_ctx_page_addr(ctx, indx, 1); - hash_page_addr = spacc_ctx_page_addr(ctx, indx, 0); - - ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1; - spacc_cipher_write_ctx(ctx, ciph_page_addr, ciph_key, ciph_len, iv, - ivlen); - writel(ciph_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET) | - (1 << SPA_KEY_SZ_CIPHER_OFFSET), - ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); - - if (hash_key) { - memcpy_toio32(hash_page_addr, hash_key, hash_len / 4); - writel(hash_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET), - ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); - } - - return indx; -} - -static inline void ddt_set(struct spacc_ddt *ddt, dma_addr_t phys, size_t len) -{ - ddt->p = phys; - ddt->len = len; -} - -/* - * Take a crypto request and scatterlists for the data and turn them into DDTs - * for passing to the crypto engines. This also DMA maps the data so that the - * crypto engines can DMA to/from them. - */ -static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine, - struct scatterlist *payload, - unsigned nbytes, - enum dma_data_direction dir, - dma_addr_t *ddt_phys) -{ - unsigned mapped_ents; - struct scatterlist *cur; - struct spacc_ddt *ddt; - int i; - int nents; - - nents = sg_nents_for_len(payload, nbytes); - if (nents < 0) { - dev_err(engine->dev, "Invalid numbers of SG.\n"); - return NULL; - } - mapped_ents = dma_map_sg(engine->dev, payload, nents, dir); - - if (mapped_ents + 1 > MAX_DDT_LEN) - goto out; - - ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys); - if (!ddt) - goto out; - - for_each_sg(payload, cur, mapped_ents, i) - ddt_set(&ddt[i], sg_dma_address(cur), sg_dma_len(cur)); - ddt_set(&ddt[mapped_ents], 0, 0); - - return ddt; - -out: - dma_unmap_sg(engine->dev, payload, nents, dir); - return NULL; -} - -static int spacc_aead_make_ddts(struct aead_request *areq) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(areq); - struct spacc_req *req = aead_request_ctx(areq); - struct spacc_engine *engine = req->engine; - struct spacc_ddt *src_ddt, *dst_ddt; - unsigned total; - int src_nents, dst_nents; - struct scatterlist *cur; - int i, dst_ents, src_ents; - - total = areq->assoclen + areq->cryptlen; - if (req->is_encrypt) - total += crypto_aead_authsize(aead); - - src_nents = sg_nents_for_len(areq->src, total); - if (src_nents < 0) { - dev_err(engine->dev, "Invalid numbers of src SG.\n"); - return src_nents; - } - if (src_nents + 1 > MAX_DDT_LEN) - return -E2BIG; - - dst_nents = 0; - if (areq->src != areq->dst) { - dst_nents = sg_nents_for_len(areq->dst, total); - if (dst_nents < 0) { - dev_err(engine->dev, "Invalid numbers of dst SG.\n"); - return dst_nents; - } - if (src_nents + 1 > MAX_DDT_LEN) - return -E2BIG; - } - - src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr); - if (!src_ddt) - goto err; - - dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr); - if (!dst_ddt) - goto err_free_src; - - req->src_ddt = src_ddt; - req->dst_ddt = dst_ddt; - - if (dst_nents) { - src_ents = dma_map_sg(engine->dev, areq->src, src_nents, - DMA_TO_DEVICE); - if (!src_ents) - goto err_free_dst; - - dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents, - DMA_FROM_DEVICE); - - if (!dst_ents) { - dma_unmap_sg(engine->dev, areq->src, src_nents, - DMA_TO_DEVICE); - goto err_free_dst; - } - } else { - src_ents = dma_map_sg(engine->dev, areq->src, src_nents, - DMA_BIDIRECTIONAL); - if (!src_ents) - goto err_free_dst; - dst_ents = src_ents; - } - - /* - * Now map in the payload for the source and destination and terminate - * with the NULL pointers. - */ - for_each_sg(areq->src, cur, src_ents, i) - ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur)); - - /* For decryption we need to skip the associated data. */ - total = req->is_encrypt ? 0 : areq->assoclen; - for_each_sg(areq->dst, cur, dst_ents, i) { - unsigned len = sg_dma_len(cur); - - if (len <= total) { - total -= len; - continue; - } - - ddt_set(dst_ddt++, sg_dma_address(cur) + total, len - total); - } - - ddt_set(src_ddt, 0, 0); - ddt_set(dst_ddt, 0, 0); - - return 0; - -err_free_dst: - dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr); -err_free_src: - dma_pool_free(engine->req_pool, src_ddt, req->src_addr); -err: - return -ENOMEM; -} - -static void spacc_aead_free_ddts(struct spacc_req *req) -{ - struct aead_request *areq = container_of(req->req, struct aead_request, - base); - struct crypto_aead *aead = crypto_aead_reqtfm(areq); - unsigned total = areq->assoclen + areq->cryptlen + - (req->is_encrypt ? crypto_aead_authsize(aead) : 0); - struct spacc_aead_ctx *aead_ctx = crypto_aead_ctx(aead); - struct spacc_engine *engine = aead_ctx->generic.engine; - int nents = sg_nents_for_len(areq->src, total); - - /* sg_nents_for_len should not fail since it works when mapping sg */ - if (unlikely(nents < 0)) { - dev_err(engine->dev, "Invalid numbers of src SG.\n"); - return; - } - - if (areq->src != areq->dst) { - dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE); - nents = sg_nents_for_len(areq->dst, total); - if (unlikely(nents < 0)) { - dev_err(engine->dev, "Invalid numbers of dst SG.\n"); - return; - } - dma_unmap_sg(engine->dev, areq->dst, nents, DMA_FROM_DEVICE); - } else - dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL); - - dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr); - dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr); -} - -static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt, - dma_addr_t ddt_addr, struct scatterlist *payload, - unsigned nbytes, enum dma_data_direction dir) -{ - int nents = sg_nents_for_len(payload, nbytes); - - if (nents < 0) { - dev_err(req->engine->dev, "Invalid numbers of SG.\n"); - return; - } - - dma_unmap_sg(req->engine->dev, payload, nents, dir); - dma_pool_free(req->engine->req_pool, ddt, ddt_addr); -} - -static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key, - unsigned int keylen) -{ - struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - struct crypto_authenc_keys keys; - int err; - - crypto_aead_clear_flags(ctx->sw_cipher, CRYPTO_TFM_REQ_MASK); - crypto_aead_set_flags(ctx->sw_cipher, crypto_aead_get_flags(tfm) & - CRYPTO_TFM_REQ_MASK); - err = crypto_aead_setkey(ctx->sw_cipher, key, keylen); - if (err) - return err; - - if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) - goto badkey; - - if (keys.enckeylen > AES_MAX_KEY_SIZE) - goto badkey; - - if (keys.authkeylen > sizeof(ctx->hash_ctx)) - goto badkey; - - memcpy(ctx->cipher_key, keys.enckey, keys.enckeylen); - ctx->cipher_key_len = keys.enckeylen; - - memcpy(ctx->hash_ctx, keys.authkey, keys.authkeylen); - ctx->hash_key_len = keys.authkeylen; - - memzero_explicit(&keys, sizeof(keys)); - return 0; - -badkey: - memzero_explicit(&keys, sizeof(keys)); - return -EINVAL; -} - -static int spacc_aead_setauthsize(struct crypto_aead *tfm, - unsigned int authsize) -{ - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm)); - - return crypto_aead_setauthsize(ctx->sw_cipher, authsize); -} - -/* - * Check if an AEAD request requires a fallback operation. Some requests can't - * be completed in hardware because the hardware may not support certain key - * sizes. In these cases we need to complete the request in software. - */ -static int spacc_aead_need_fallback(struct aead_request *aead_req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); - struct aead_alg *alg = crypto_aead_alg(aead); - struct spacc_aead *spacc_alg = to_spacc_aead(alg); - struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead); - - /* - * If we have a non-supported key-length, then we need to do a - * software fallback. - */ - if ((spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) == - SPA_CTRL_CIPH_ALG_AES && - ctx->cipher_key_len != AES_KEYSIZE_128 && - ctx->cipher_key_len != AES_KEYSIZE_256) - return 1; - - return 0; -} - -static int spacc_aead_do_fallback(struct aead_request *req, unsigned alg_type, - bool is_encrypt) -{ - struct crypto_tfm *old_tfm = crypto_aead_tfm(crypto_aead_reqtfm(req)); - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(old_tfm); - struct aead_request *subreq = aead_request_ctx(req); - - aead_request_set_tfm(subreq, ctx->sw_cipher); - aead_request_set_callback(subreq, req->base.flags, - req->base.complete, req->base.data); - aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, - req->iv); - aead_request_set_ad(subreq, req->assoclen); - - return is_encrypt ? crypto_aead_encrypt(subreq) : - crypto_aead_decrypt(subreq); -} - -static void spacc_aead_complete(struct spacc_req *req) -{ - spacc_aead_free_ddts(req); - req->req->complete(req->req, req->result); -} - -static int spacc_aead_submit(struct spacc_req *req) -{ - struct aead_request *aead_req = - container_of(req->req, struct aead_request, base); - struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); - unsigned int authsize = crypto_aead_authsize(aead); - struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead); - struct aead_alg *alg = crypto_aead_alg(aead); - struct spacc_aead *spacc_alg = to_spacc_aead(alg); - struct spacc_engine *engine = ctx->generic.engine; - u32 ctrl, proc_len, assoc_len; - - req->result = -EINPROGRESS; - req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key, - ctx->cipher_key_len, aead_req->iv, crypto_aead_ivsize(aead), - ctx->hash_ctx, ctx->hash_key_len); - - /* Set the source and destination DDT pointers. */ - writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); - writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); - writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); - - assoc_len = aead_req->assoclen; - proc_len = aead_req->cryptlen + assoc_len; - - /* - * If we are decrypting, we need to take the length of the ICV out of - * the processing length. - */ - if (!req->is_encrypt) - proc_len -= authsize; - - writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET); - writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET); - writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET); - writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); - writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); - - ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) | - (1 << SPA_CTRL_ICV_APPEND); - if (req->is_encrypt) - ctrl |= (1 << SPA_CTRL_ENCRYPT_IDX) | (1 << SPA_CTRL_AAD_COPY); - else - ctrl |= (1 << SPA_CTRL_KEY_EXP); - - mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); - - writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); - - return -EINPROGRESS; -} - -static int spacc_req_submit(struct spacc_req *req); - -static void spacc_push(struct spacc_engine *engine) -{ - struct spacc_req *req; - - while (!list_empty(&engine->pending) && - engine->in_flight + 1 <= engine->fifo_sz) { - - ++engine->in_flight; - req = list_first_entry(&engine->pending, struct spacc_req, - list); - list_move_tail(&req->list, &engine->in_progress); - - req->result = spacc_req_submit(req); - } -} - -/* - * Setup an AEAD request for processing. This will configure the engine, load - * the context and then start the packet processing. - */ -static int spacc_aead_setup(struct aead_request *req, - unsigned alg_type, bool is_encrypt) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct aead_alg *alg = crypto_aead_alg(aead); - struct spacc_engine *engine = to_spacc_aead(alg)->engine; - struct spacc_req *dev_req = aead_request_ctx(req); - int err; - unsigned long flags; - - dev_req->req = &req->base; - dev_req->is_encrypt = is_encrypt; - dev_req->result = -EBUSY; - dev_req->engine = engine; - dev_req->complete = spacc_aead_complete; - - if (unlikely(spacc_aead_need_fallback(req) || - ((err = spacc_aead_make_ddts(req)) == -E2BIG))) - return spacc_aead_do_fallback(req, alg_type, is_encrypt); - - if (err) - goto out; - - err = -EINPROGRESS; - spin_lock_irqsave(&engine->hw_lock, flags); - if (unlikely(spacc_fifo_cmd_full(engine)) || - engine->in_flight + 1 > engine->fifo_sz) { - if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { - err = -EBUSY; - spin_unlock_irqrestore(&engine->hw_lock, flags); - goto out_free_ddts; - } - list_add_tail(&dev_req->list, &engine->pending); - } else { - list_add_tail(&dev_req->list, &engine->pending); - spacc_push(engine); - } - spin_unlock_irqrestore(&engine->hw_lock, flags); - - goto out; - -out_free_ddts: - spacc_aead_free_ddts(dev_req); -out: - return err; -} - -static int spacc_aead_encrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead)); - - return spacc_aead_setup(req, alg->type, 1); -} - -static int spacc_aead_decrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead)); - - return spacc_aead_setup(req, alg->type, 0); -} - -/* - * Initialise a new AEAD context. This is responsible for allocating the - * fallback cipher and initialising the context. - */ -static int spacc_aead_cra_init(struct crypto_aead *tfm) -{ - struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - struct aead_alg *alg = crypto_aead_alg(tfm); - struct spacc_aead *spacc_alg = to_spacc_aead(alg); - struct spacc_engine *engine = spacc_alg->engine; - - ctx->generic.flags = spacc_alg->type; - ctx->generic.engine = engine; - ctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->sw_cipher)) - return PTR_ERR(ctx->sw_cipher); - ctx->generic.key_offs = spacc_alg->key_offs; - ctx->generic.iv_offs = spacc_alg->iv_offs; - - crypto_aead_set_reqsize( - tfm, - max(sizeof(struct spacc_req), - sizeof(struct aead_request) + - crypto_aead_reqsize(ctx->sw_cipher))); - - return 0; -} - -/* - * Destructor for an AEAD context. This is called when the transform is freed - * and must free the fallback cipher. - */ -static void spacc_aead_cra_exit(struct crypto_aead *tfm) -{ - struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - - crypto_free_aead(ctx->sw_cipher); -} - -/* - * Set the DES key for a block cipher transform. This also performs weak key - * checking if the transform has requested it. - */ -static int spacc_des_setkey(struct crypto_skcipher *cipher, const u8 *key, - unsigned int len) -{ - struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(cipher); - int err; - - err = verify_skcipher_des_key(cipher, key); - if (err) - return err; - - memcpy(ctx->key, key, len); - ctx->key_len = len; - - return 0; -} - -/* - * Set the 3DES key for a block cipher transform. This also performs weak key - * checking if the transform has requested it. - */ -static int spacc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, - unsigned int len) -{ - struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(cipher); - int err; - - err = verify_skcipher_des3_key(cipher, key); - if (err) - return err; - - memcpy(ctx->key, key, len); - ctx->key_len = len; - - return 0; -} - -/* - * Set the key for an AES block cipher. Some key lengths are not supported in - * hardware so this must also check whether a fallback is needed. - */ -static int spacc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, - unsigned int len) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); - struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm); - int err = 0; - - if (len > AES_MAX_KEY_SIZE) - return -EINVAL; - - /* - * IPSec engine only supports 128 and 256 bit AES keys. If we get a - * request for any other size (192 bits) then we need to do a software - * fallback. - */ - if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256) { - if (!ctx->sw_cipher) - return -EINVAL; - - /* - * Set the fallback transform to use the same request flags as - * the hardware transform. - */ - crypto_skcipher_clear_flags(ctx->sw_cipher, - CRYPTO_TFM_REQ_MASK); - crypto_skcipher_set_flags(ctx->sw_cipher, - cipher->base.crt_flags & - CRYPTO_TFM_REQ_MASK); - - err = crypto_skcipher_setkey(ctx->sw_cipher, key, len); - if (err) - goto sw_setkey_failed; - } - - memcpy(ctx->key, key, len); - ctx->key_len = len; - -sw_setkey_failed: - return err; -} - -static int spacc_kasumi_f8_setkey(struct crypto_skcipher *cipher, - const u8 *key, unsigned int len) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); - struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm); - int err = 0; - - if (len > AES_MAX_KEY_SIZE) { - err = -EINVAL; - goto out; - } - - memcpy(ctx->key, key, len); - ctx->key_len = len; - -out: - return err; -} - -static int spacc_ablk_need_fallback(struct spacc_req *req) -{ - struct skcipher_request *ablk_req = skcipher_request_cast(req->req); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(ablk_req); - struct spacc_alg *spacc_alg = to_spacc_skcipher(crypto_skcipher_alg(tfm)); - struct spacc_ablk_ctx *ctx; - - ctx = crypto_skcipher_ctx(tfm); - - return (spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) == - SPA_CTRL_CIPH_ALG_AES && - ctx->key_len != AES_KEYSIZE_128 && - ctx->key_len != AES_KEYSIZE_256; -} - -static void spacc_ablk_complete(struct spacc_req *req) -{ - struct skcipher_request *ablk_req = skcipher_request_cast(req->req); - - if (ablk_req->src != ablk_req->dst) { - spacc_free_ddt(req, req->src_ddt, req->src_addr, ablk_req->src, - ablk_req->cryptlen, DMA_TO_DEVICE); - spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst, - ablk_req->cryptlen, DMA_FROM_DEVICE); - } else - spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst, - ablk_req->cryptlen, DMA_BIDIRECTIONAL); - - req->req->complete(req->req, req->result); -} - -static int spacc_ablk_submit(struct spacc_req *req) -{ - struct skcipher_request *ablk_req = skcipher_request_cast(req->req); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(ablk_req); - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - struct spacc_alg *spacc_alg = to_spacc_skcipher(alg); - struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(tfm); - struct spacc_engine *engine = ctx->generic.engine; - u32 ctrl; - - req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->key, - ctx->key_len, ablk_req->iv, alg->ivsize, - NULL, 0); - - writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); - writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); - writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); - - writel(ablk_req->cryptlen, engine->regs + SPA_PROC_LEN_REG_OFFSET); - writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); - writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); - writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET); - - ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) | - (req->is_encrypt ? (1 << SPA_CTRL_ENCRYPT_IDX) : - (1 << SPA_CTRL_KEY_EXP)); - - mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); - - writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); - - return -EINPROGRESS; -} - -static int spacc_ablk_do_fallback(struct skcipher_request *req, - unsigned alg_type, bool is_encrypt) -{ - struct crypto_tfm *old_tfm = - crypto_skcipher_tfm(crypto_skcipher_reqtfm(req)); - struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(old_tfm); - struct spacc_req *dev_req = skcipher_request_ctx(req); - int err; - - /* - * Change the request to use the software fallback transform, and once - * the ciphering has completed, put the old transform back into the - * request. - */ - skcipher_request_set_tfm(&dev_req->fallback_req, ctx->sw_cipher); - skcipher_request_set_callback(&dev_req->fallback_req, req->base.flags, - req->base.complete, req->base.data); - skcipher_request_set_crypt(&dev_req->fallback_req, req->src, req->dst, - req->cryptlen, req->iv); - err = is_encrypt ? crypto_skcipher_encrypt(&dev_req->fallback_req) : - crypto_skcipher_decrypt(&dev_req->fallback_req); - - return err; -} - -static int spacc_ablk_setup(struct skcipher_request *req, unsigned alg_type, - bool is_encrypt) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - struct spacc_engine *engine = to_spacc_skcipher(alg)->engine; - struct spacc_req *dev_req = skcipher_request_ctx(req); - unsigned long flags; - int err = -ENOMEM; - - dev_req->req = &req->base; - dev_req->is_encrypt = is_encrypt; - dev_req->engine = engine; - dev_req->complete = spacc_ablk_complete; - dev_req->result = -EINPROGRESS; - - if (unlikely(spacc_ablk_need_fallback(dev_req))) - return spacc_ablk_do_fallback(req, alg_type, is_encrypt); - - /* - * Create the DDT's for the engine. If we share the same source and - * destination then we can optimize by reusing the DDT's. - */ - if (req->src != req->dst) { - dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src, - req->cryptlen, DMA_TO_DEVICE, &dev_req->src_addr); - if (!dev_req->src_ddt) - goto out; - - dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, - req->cryptlen, DMA_FROM_DEVICE, &dev_req->dst_addr); - if (!dev_req->dst_ddt) - goto out_free_src; - } else { - dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, - req->cryptlen, DMA_BIDIRECTIONAL, &dev_req->dst_addr); - if (!dev_req->dst_ddt) - goto out; - - dev_req->src_ddt = NULL; - dev_req->src_addr = dev_req->dst_addr; - } - - err = -EINPROGRESS; - spin_lock_irqsave(&engine->hw_lock, flags); - /* - * Check if the engine will accept the operation now. If it won't then - * we either stick it on the end of a pending list if we can backlog, - * or bailout with an error if not. - */ - if (unlikely(spacc_fifo_cmd_full(engine)) || - engine->in_flight + 1 > engine->fifo_sz) { - if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { - err = -EBUSY; - spin_unlock_irqrestore(&engine->hw_lock, flags); - goto out_free_ddts; - } - list_add_tail(&dev_req->list, &engine->pending); - } else { - list_add_tail(&dev_req->list, &engine->pending); - spacc_push(engine); - } - spin_unlock_irqrestore(&engine->hw_lock, flags); - - goto out; - -out_free_ddts: - spacc_free_ddt(dev_req, dev_req->dst_ddt, dev_req->dst_addr, req->dst, - req->cryptlen, req->src == req->dst ? - DMA_BIDIRECTIONAL : DMA_FROM_DEVICE); -out_free_src: - if (req->src != req->dst) - spacc_free_ddt(dev_req, dev_req->src_ddt, dev_req->src_addr, - req->src, req->cryptlen, DMA_TO_DEVICE); -out: - return err; -} - -static int spacc_ablk_init_tfm(struct crypto_skcipher *tfm) -{ - struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - struct spacc_alg *spacc_alg = to_spacc_skcipher(alg); - struct spacc_engine *engine = spacc_alg->engine; - - ctx->generic.flags = spacc_alg->type; - ctx->generic.engine = engine; - if (alg->base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) { - ctx->sw_cipher = crypto_alloc_skcipher(alg->base.cra_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->sw_cipher)) { - dev_warn(engine->dev, "failed to allocate fallback for %s\n", - alg->base.cra_name); - return PTR_ERR(ctx->sw_cipher); - } - crypto_skcipher_set_reqsize(tfm, sizeof(struct spacc_req) + - crypto_skcipher_reqsize(ctx->sw_cipher)); - } else { - /* take the size without the fallback skcipher_request at the end */ - crypto_skcipher_set_reqsize(tfm, offsetof(struct spacc_req, - fallback_req)); - } - - ctx->generic.key_offs = spacc_alg->key_offs; - ctx->generic.iv_offs = spacc_alg->iv_offs; - - return 0; -} - -static void spacc_ablk_exit_tfm(struct crypto_skcipher *tfm) -{ - struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(tfm); - - crypto_free_skcipher(ctx->sw_cipher); -} - -static int spacc_ablk_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); - struct skcipher_alg *alg = crypto_skcipher_alg(cipher); - struct spacc_alg *spacc_alg = to_spacc_skcipher(alg); - - return spacc_ablk_setup(req, spacc_alg->type, 1); -} - -static int spacc_ablk_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); - struct skcipher_alg *alg = crypto_skcipher_alg(cipher); - struct spacc_alg *spacc_alg = to_spacc_skcipher(alg); - - return spacc_ablk_setup(req, spacc_alg->type, 0); -} - -static inline int spacc_fifo_stat_empty(struct spacc_engine *engine) -{ - return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) & - SPA_FIFO_STAT_EMPTY; -} - -static void spacc_process_done(struct spacc_engine *engine) -{ - struct spacc_req *req; - unsigned long flags; - - spin_lock_irqsave(&engine->hw_lock, flags); - - while (!spacc_fifo_stat_empty(engine)) { - req = list_first_entry(&engine->in_progress, struct spacc_req, - list); - list_move_tail(&req->list, &engine->completed); - --engine->in_flight; - - /* POP the status register. */ - writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET); - req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) & - SPA_STATUS_RES_CODE_MASK) >> SPA_STATUS_RES_CODE_OFFSET; - - /* - * Convert the SPAcc error status into the standard POSIX error - * codes. - */ - if (unlikely(req->result)) { - switch (req->result) { - case SPA_STATUS_ICV_FAIL: - req->result = -EBADMSG; - break; - - case SPA_STATUS_MEMORY_ERROR: - dev_warn(engine->dev, - "memory error triggered\n"); - req->result = -EFAULT; - break; - - case SPA_STATUS_BLOCK_ERROR: - dev_warn(engine->dev, - "block error triggered\n"); - req->result = -EIO; - break; - } - } - } - - tasklet_schedule(&engine->complete); - - spin_unlock_irqrestore(&engine->hw_lock, flags); -} - -static irqreturn_t spacc_spacc_irq(int irq, void *dev) -{ - struct spacc_engine *engine = (struct spacc_engine *)dev; - u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET); - - writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET); - spacc_process_done(engine); - - return IRQ_HANDLED; -} - -static void spacc_packet_timeout(struct timer_list *t) -{ - struct spacc_engine *engine = from_timer(engine, t, packet_timeout); - - spacc_process_done(engine); -} - -static int spacc_req_submit(struct spacc_req *req) -{ - struct crypto_alg *alg = req->req->tfm->__crt_alg; - - if (CRYPTO_ALG_TYPE_AEAD == (CRYPTO_ALG_TYPE_MASK & alg->cra_flags)) - return spacc_aead_submit(req); - else - return spacc_ablk_submit(req); -} - -static void spacc_spacc_complete(unsigned long data) -{ - struct spacc_engine *engine = (struct spacc_engine *)data; - struct spacc_req *req, *tmp; - unsigned long flags; - LIST_HEAD(completed); - - spin_lock_irqsave(&engine->hw_lock, flags); - - list_splice_init(&engine->completed, &completed); - spacc_push(engine); - if (engine->in_flight) - mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); - - spin_unlock_irqrestore(&engine->hw_lock, flags); - - list_for_each_entry_safe(req, tmp, &completed, list) { - list_del(&req->list); - req->complete(req); - } -} - -#ifdef CONFIG_PM -static int spacc_suspend(struct device *dev) -{ - struct spacc_engine *engine = dev_get_drvdata(dev); - - /* - * We only support standby mode. All we have to do is gate the clock to - * the spacc. The hardware will preserve state until we turn it back - * on again. - */ - clk_disable(engine->clk); - - return 0; -} - -static int spacc_resume(struct device *dev) -{ - struct spacc_engine *engine = dev_get_drvdata(dev); - - return clk_enable(engine->clk); -} - -static const struct dev_pm_ops spacc_pm_ops = { - .suspend = spacc_suspend, - .resume = spacc_resume, -}; -#endif /* CONFIG_PM */ - -static inline struct spacc_engine *spacc_dev_to_engine(struct device *dev) -{ - return dev ? dev_get_drvdata(dev) : NULL; -} - -static ssize_t spacc_stat_irq_thresh_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct spacc_engine *engine = spacc_dev_to_engine(dev); - - return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh); -} - -static ssize_t spacc_stat_irq_thresh_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t len) -{ - struct spacc_engine *engine = spacc_dev_to_engine(dev); - unsigned long thresh; - - if (kstrtoul(buf, 0, &thresh)) - return -EINVAL; - - thresh = clamp(thresh, 1UL, engine->fifo_sz - 1); - - engine->stat_irq_thresh = thresh; - writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, - engine->regs + SPA_IRQ_CTRL_REG_OFFSET); - - return len; -} -static DEVICE_ATTR(stat_irq_thresh, 0644, spacc_stat_irq_thresh_show, - spacc_stat_irq_thresh_store); - -static struct spacc_alg ipsec_engine_algs[] = { - { - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC, - .key_offs = 0, - .iv_offs = AES_MAX_KEY_SIZE, - .alg = { - .base.cra_name = "cbc(aes)", - .base.cra_driver_name = "cbc-aes-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_aes_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, - { - .key_offs = 0, - .iv_offs = AES_MAX_KEY_SIZE, - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_ECB, - .alg = { - .base.cra_name = "ecb(aes)", - .base.cra_driver_name = "ecb-aes-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_aes_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC, - .alg = { - .base.cra_name = "cbc(des)", - .base.cra_driver_name = "cbc-des-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY, - .base.cra_blocksize = DES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_des_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB, - .alg = { - .base.cra_name = "ecb(des)", - .base.cra_driver_name = "ecb-des-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY, - .base.cra_blocksize = DES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_des_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC, - .alg = { - .base.cra_name = "cbc(des3_ede)", - .base.cra_driver_name = "cbc-des3-ede-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_des3_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB, - .alg = { - .base.cra_name = "ecb(des3_ede)", - .base.cra_driver_name = "ecb-des3-ede-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_des3_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, -}; - -static struct spacc_aead ipsec_engine_aeads[] = { - { - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA | - SPA_CTRL_HASH_MODE_HMAC, - .key_offs = 0, - .iv_offs = AES_MAX_KEY_SIZE, - .alg = { - .base = { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha1-" - "cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, - { - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA256 | - SPA_CTRL_HASH_MODE_HMAC, - .key_offs = 0, - .iv_offs = AES_MAX_KEY_SIZE, - .alg = { - .base = { - .cra_name = "authenc(hmac(sha256),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha256-" - "cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, - { - .key_offs = 0, - .iv_offs = AES_MAX_KEY_SIZE, - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_MD5 | - SPA_CTRL_HASH_MODE_HMAC, - .alg = { - .base = { - .cra_name = "authenc(hmac(md5),cbc(aes))", - .cra_driver_name = "authenc-hmac-md5-" - "cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA | - SPA_CTRL_HASH_MODE_HMAC, - .alg = { - .base = { - .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha1-" - "cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA256 | - SPA_CTRL_HASH_MODE_HMAC, - .alg = { - .base = { - .cra_name = "authenc(hmac(sha256)," - "cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha256-" - "cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_MD5 | - SPA_CTRL_HASH_MODE_HMAC, - .alg = { - .base = { - .cra_name = "authenc(hmac(md5),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-md5-" - "cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, -}; - -static struct spacc_alg l2_engine_algs[] = { - { - .key_offs = 0, - .iv_offs = SPACC_CRYPTO_KASUMI_F8_KEY_LEN, - .ctrl_default = SPA_CTRL_CIPH_ALG_KASUMI | - SPA_CTRL_CIPH_MODE_F8, - .alg = { - .base.cra_name = "f8(kasumi)", - .base.cra_driver_name = "f8-kasumi-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = 8, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_kasumi_f8_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = 16, - .max_keysize = 16, - .ivsize = 8, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, -}; - -#ifdef CONFIG_OF -static const struct of_device_id spacc_of_id_table[] = { - { .compatible = "picochip,spacc-ipsec" }, - { .compatible = "picochip,spacc-l2" }, - {} -}; -MODULE_DEVICE_TABLE(of, spacc_of_id_table); -#endif /* CONFIG_OF */ - -static void spacc_tasklet_kill(void *data) -{ - tasklet_kill(data); -} - -static int spacc_probe(struct platform_device *pdev) -{ - int i, err, ret; - struct resource *irq; - struct device_node *np = pdev->dev.of_node; - struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine), - GFP_KERNEL); - if (!engine) - return -ENOMEM; - - if (of_device_is_compatible(np, "picochip,spacc-ipsec")) { - engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS; - engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ; - engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ; - engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ; - engine->algs = ipsec_engine_algs; - engine->num_algs = ARRAY_SIZE(ipsec_engine_algs); - engine->aeads = ipsec_engine_aeads; - engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads); - } else if (of_device_is_compatible(np, "picochip,spacc-l2")) { - engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS; - engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ; - engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ; - engine->fifo_sz = SPACC_CRYPTO_L2_FIFO_SZ; - engine->algs = l2_engine_algs; - engine->num_algs = ARRAY_SIZE(l2_engine_algs); - } else { - return -EINVAL; - } - - engine->name = dev_name(&pdev->dev); - - engine->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(engine->regs)) - return PTR_ERR(engine->regs); - - irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!irq) { - dev_err(&pdev->dev, "no memory/irq resource for engine\n"); - return -ENXIO; - } - - tasklet_init(&engine->complete, spacc_spacc_complete, - (unsigned long)engine); - - ret = devm_add_action(&pdev->dev, spacc_tasklet_kill, - &engine->complete); - if (ret) - return ret; - - if (devm_request_irq(&pdev->dev, irq->start, spacc_spacc_irq, 0, - engine->name, engine)) { - dev_err(engine->dev, "failed to request IRQ\n"); - return -EBUSY; - } - - engine->dev = &pdev->dev; - engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET; - engine->hash_key_base = engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET; - - engine->req_pool = dmam_pool_create(engine->name, engine->dev, - MAX_DDT_LEN * sizeof(struct spacc_ddt), 8, SZ_64K); - if (!engine->req_pool) - return -ENOMEM; - - spin_lock_init(&engine->hw_lock); - - engine->clk = clk_get(&pdev->dev, "ref"); - if (IS_ERR(engine->clk)) { - dev_info(&pdev->dev, "clk unavailable\n"); - return PTR_ERR(engine->clk); - } - - if (clk_prepare_enable(engine->clk)) { - dev_info(&pdev->dev, "unable to prepare/enable clk\n"); - ret = -EIO; - goto err_clk_put; - } - - /* - * Use an IRQ threshold of 50% as a default. This seems to be a - * reasonable trade off of latency against throughput but can be - * changed at runtime. - */ - engine->stat_irq_thresh = (engine->fifo_sz / 2); - - ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh); - if (ret) - goto err_clk_disable; - - /* - * Configure the interrupts. We only use the STAT_CNT interrupt as we - * only submit a new packet for processing when we complete another in - * the queue. This minimizes time spent in the interrupt handler. - */ - writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, - engine->regs + SPA_IRQ_CTRL_REG_OFFSET); - writel(SPA_IRQ_EN_STAT_EN | SPA_IRQ_EN_GLBL_EN, - engine->regs + SPA_IRQ_EN_REG_OFFSET); - - timer_setup(&engine->packet_timeout, spacc_packet_timeout, 0); - - INIT_LIST_HEAD(&engine->pending); - INIT_LIST_HEAD(&engine->completed); - INIT_LIST_HEAD(&engine->in_progress); - engine->in_flight = 0; - - platform_set_drvdata(pdev, engine); - - ret = -EINVAL; - INIT_LIST_HEAD(&engine->registered_algs); - for (i = 0; i < engine->num_algs; ++i) { - engine->algs[i].engine = engine; - err = crypto_register_skcipher(&engine->algs[i].alg); - if (!err) { - list_add_tail(&engine->algs[i].entry, - &engine->registered_algs); - ret = 0; - } - if (err) - dev_err(engine->dev, "failed to register alg \"%s\"\n", - engine->algs[i].alg.base.cra_name); - else - dev_dbg(engine->dev, "registered alg \"%s\"\n", - engine->algs[i].alg.base.cra_name); - } - - INIT_LIST_HEAD(&engine->registered_aeads); - for (i = 0; i < engine->num_aeads; ++i) { - engine->aeads[i].engine = engine; - err = crypto_register_aead(&engine->aeads[i].alg); - if (!err) { - list_add_tail(&engine->aeads[i].entry, - &engine->registered_aeads); - ret = 0; - } - if (err) - dev_err(engine->dev, "failed to register alg \"%s\"\n", - engine->aeads[i].alg.base.cra_name); - else - dev_dbg(engine->dev, "registered alg \"%s\"\n", - engine->aeads[i].alg.base.cra_name); - } - - if (!ret) - return 0; - - del_timer_sync(&engine->packet_timeout); - device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh); -err_clk_disable: - clk_disable_unprepare(engine->clk); -err_clk_put: - clk_put(engine->clk); - - return ret; -} - -static int spacc_remove(struct platform_device *pdev) -{ - struct spacc_aead *aead, *an; - struct spacc_alg *alg, *next; - struct spacc_engine *engine = platform_get_drvdata(pdev); - - del_timer_sync(&engine->packet_timeout); - device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh); - - list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) { - list_del(&aead->entry); - crypto_unregister_aead(&aead->alg); - } - - list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) { - list_del(&alg->entry); - crypto_unregister_skcipher(&alg->alg); - } - - clk_disable_unprepare(engine->clk); - clk_put(engine->clk); - - return 0; -} - -static struct platform_driver spacc_driver = { - .probe = spacc_probe, - .remove = spacc_remove, - .driver = { - .name = "picochip,spacc", -#ifdef CONFIG_PM - .pm = &spacc_pm_ops, -#endif /* CONFIG_PM */ - .of_match_table = of_match_ptr(spacc_of_id_table), - }, -}; - -module_platform_driver(spacc_driver); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Jamie Iles"); diff --git a/drivers/crypto/picoxcell_crypto_regs.h b/drivers/crypto/picoxcell_crypto_regs.h deleted file mode 100644 index b870a50238ba..000000000000 --- a/drivers/crypto/picoxcell_crypto_regs.h +++ /dev/null @@ -1,115 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (c) 2010 Picochip Ltd., Jamie Iles - */ -#ifndef __PICOXCELL_CRYPTO_REGS_H__ -#define __PICOXCELL_CRYPTO_REGS_H__ - -#define SPA_STATUS_OK 0 -#define SPA_STATUS_ICV_FAIL 1 -#define SPA_STATUS_MEMORY_ERROR 2 -#define SPA_STATUS_BLOCK_ERROR 3 - -#define SPA_IRQ_CTRL_STAT_CNT_OFFSET 16 -#define SPA_IRQ_STAT_STAT_MASK (1 << 4) -#define SPA_FIFO_STAT_STAT_OFFSET 16 -#define SPA_FIFO_STAT_STAT_CNT_MASK (0x3F << SPA_FIFO_STAT_STAT_OFFSET) -#define SPA_STATUS_RES_CODE_OFFSET 24 -#define SPA_STATUS_RES_CODE_MASK (0x3 << SPA_STATUS_RES_CODE_OFFSET) -#define SPA_KEY_SZ_CTX_INDEX_OFFSET 8 -#define SPA_KEY_SZ_CIPHER_OFFSET 31 - -#define SPA_IRQ_EN_REG_OFFSET 0x00000000 -#define SPA_IRQ_STAT_REG_OFFSET 0x00000004 -#define SPA_IRQ_CTRL_REG_OFFSET 0x00000008 -#define SPA_FIFO_STAT_REG_OFFSET 0x0000000C -#define SPA_SDMA_BRST_SZ_REG_OFFSET 0x00000010 -#define SPA_SRC_PTR_REG_OFFSET 0x00000020 -#define SPA_DST_PTR_REG_OFFSET 0x00000024 -#define SPA_OFFSET_REG_OFFSET 0x00000028 -#define SPA_AAD_LEN_REG_OFFSET 0x0000002C -#define SPA_PROC_LEN_REG_OFFSET 0x00000030 -#define SPA_ICV_LEN_REG_OFFSET 0x00000034 -#define SPA_ICV_OFFSET_REG_OFFSET 0x00000038 -#define SPA_SW_CTRL_REG_OFFSET 0x0000003C -#define SPA_CTRL_REG_OFFSET 0x00000040 -#define SPA_AUX_INFO_REG_OFFSET 0x0000004C -#define SPA_STAT_POP_REG_OFFSET 0x00000050 -#define SPA_STATUS_REG_OFFSET 0x00000054 -#define SPA_KEY_SZ_REG_OFFSET 0x00000100 -#define SPA_CIPH_KEY_BASE_REG_OFFSET 0x00004000 -#define SPA_HASH_KEY_BASE_REG_OFFSET 0x00008000 -#define SPA_RC4_CTX_BASE_REG_OFFSET 0x00020000 - -#define SPA_IRQ_EN_REG_RESET 0x00000000 -#define SPA_IRQ_CTRL_REG_RESET 0x00000000 -#define SPA_FIFO_STAT_REG_RESET 0x00000000 -#define SPA_SDMA_BRST_SZ_REG_RESET 0x00000000 -#define SPA_SRC_PTR_REG_RESET 0x00000000 -#define SPA_DST_PTR_REG_RESET 0x00000000 -#define SPA_OFFSET_REG_RESET 0x00000000 -#define SPA_AAD_LEN_REG_RESET 0x00000000 -#define SPA_PROC_LEN_REG_RESET 0x00000000 -#define SPA_ICV_LEN_REG_RESET 0x00000000 -#define SPA_ICV_OFFSET_REG_RESET 0x00000000 -#define SPA_SW_CTRL_REG_RESET 0x00000000 -#define SPA_CTRL_REG_RESET 0x00000000 -#define SPA_AUX_INFO_REG_RESET 0x00000000 -#define SPA_STAT_POP_REG_RESET 0x00000000 -#define SPA_STATUS_REG_RESET 0x00000000 -#define SPA_KEY_SZ_REG_RESET 0x00000000 - -#define SPA_CTRL_HASH_ALG_IDX 4 -#define SPA_CTRL_CIPH_MODE_IDX 8 -#define SPA_CTRL_HASH_MODE_IDX 12 -#define SPA_CTRL_CTX_IDX 16 -#define SPA_CTRL_ENCRYPT_IDX 24 -#define SPA_CTRL_AAD_COPY 25 -#define SPA_CTRL_ICV_PT 26 -#define SPA_CTRL_ICV_ENC 27 -#define SPA_CTRL_ICV_APPEND 28 -#define SPA_CTRL_KEY_EXP 29 - -#define SPA_KEY_SZ_CXT_IDX 8 -#define SPA_KEY_SZ_CIPHER_IDX 31 - -#define SPA_IRQ_EN_CMD0_EN (1 << 0) -#define SPA_IRQ_EN_STAT_EN (1 << 4) -#define SPA_IRQ_EN_GLBL_EN (1 << 31) - -#define SPA_CTRL_CIPH_ALG_NULL 0x00 -#define SPA_CTRL_CIPH_ALG_DES 0x01 -#define SPA_CTRL_CIPH_ALG_AES 0x02 -#define SPA_CTRL_CIPH_ALG_RC4 0x03 -#define SPA_CTRL_CIPH_ALG_MULTI2 0x04 -#define SPA_CTRL_CIPH_ALG_KASUMI 0x05 - -#define SPA_CTRL_HASH_ALG_NULL (0x00 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_MD5 (0x01 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_SHA (0x02 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_SHA224 (0x03 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_SHA256 (0x04 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_SHA384 (0x05 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_SHA512 (0x06 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_AESMAC (0x07 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_AESCMAC (0x08 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_KASF9 (0x09 << SPA_CTRL_HASH_ALG_IDX) - -#define SPA_CTRL_CIPH_MODE_NULL (0x00 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_ECB (0x00 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_CBC (0x01 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_CTR (0x02 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_CCM (0x03 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_GCM (0x05 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_OFB (0x07 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_CFB (0x08 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_F8 (0x09 << SPA_CTRL_CIPH_MODE_IDX) - -#define SPA_CTRL_HASH_MODE_RAW (0x00 << SPA_CTRL_HASH_MODE_IDX) -#define SPA_CTRL_HASH_MODE_SSLMAC (0x01 << SPA_CTRL_HASH_MODE_IDX) -#define SPA_CTRL_HASH_MODE_HMAC (0x02 << SPA_CTRL_HASH_MODE_IDX) - -#define SPA_FIFO_STAT_EMPTY (1 << 31) -#define SPA_FIFO_CMD_FULL (1 << 7) - -#endif /* __PICOXCELL_CRYPTO_REGS_H__ */ diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index 846a3d90b41a..77783feb62b2 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -11,7 +11,7 @@ config CRYPTO_DEV_QAT select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 - select CRYPTO_AES + select CRYPTO_LIB_AES select FW_LOADER config CRYPTO_DEV_QAT_DH895xCC diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index 344bfae45bff..6a9be01fdf33 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -19,7 +19,7 @@ static struct adf_fw_config adf_4xxx_fw_config[] = { }; /* Worker thread to service arbiter mappings */ -static u32 thrd_to_arb_map[] = { +static const u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = { 0x5555555, 0x5555555, 0x5555555, 0x5555555, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0x0 @@ -119,17 +119,9 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_1; } -static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, - u32 const **arb_map_config) +static const u32 *adf_get_arbiter_mapping(void) { - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - unsigned long ae_mask = hw_device->ae_mask; - int i; - - for_each_clear_bit(i, &ae_mask, ADF_4XXX_MAX_ACCELENGINES) - thrd_to_arb_map[i] = 0; - - *arb_map_config = thrd_to_arb_map; + return thrd_to_arb_map; } static void get_arb_info(struct arb_info *arb_info) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index eb45f1b1ae3e..f5990d042c9a 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -7,8 +7,8 @@ #include "adf_c3xxx_hw_data.h" #include "icp_qat_hw.h" -/* Worker thread to service arbiter mappings based on dev SKUs */ -static const u32 thrd_to_arb_map_6_me_sku[] = { +/* Worker thread to service arbiter mappings */ +static const u32 thrd_to_arb_map[ADF_C3XXX_MAX_ACCELENGINES] = { 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA }; @@ -101,18 +101,9 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_UNKNOWN; } -static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, - u32 const **arb_map_config) +static const u32 *adf_get_arbiter_mapping(void) { - switch (accel_dev->accel_pci_dev.sku) { - case DEV_SKU_4: - *arb_map_config = thrd_to_arb_map_6_me_sku; - break; - default: - dev_err(&GET_DEV(accel_dev), - "The configuration doesn't match any SKU"); - *arb_map_config = NULL; - } + return thrd_to_arb_map; } static u32 get_pf2vf_offset(u32 i) diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index babdffbcb846..cadcf12884c8 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -7,13 +7,8 @@ #include "adf_c62x_hw_data.h" #include "icp_qat_hw.h" -/* Worker thread to service arbiter mappings based on dev SKUs */ -static const u32 thrd_to_arb_map_8_me_sku[] = { - 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, - 0x11222AAA, 0x12222AAA, 0x11222AAA, 0, 0 -}; - -static const u32 thrd_to_arb_map_10_me_sku[] = { +/* Worker thread to service arbiter mappings */ +static const u32 thrd_to_arb_map[ADF_C62X_MAX_ACCELENGINES] = { 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA }; @@ -108,21 +103,9 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_UNKNOWN; } -static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, - u32 const **arb_map_config) +static const u32 *adf_get_arbiter_mapping(void) { - switch (accel_dev->accel_pci_dev.sku) { - case DEV_SKU_2: - *arb_map_config = thrd_to_arb_map_8_me_sku; - break; - case DEV_SKU_4: - *arb_map_config = thrd_to_arb_map_10_me_sku; - break; - default: - dev_err(&GET_DEV(accel_dev), - "The configuration doesn't match any SKU"); - *arb_map_config = NULL; - } + return thrd_to_arb_map; } static u32 get_pf2vf_offset(u32 i) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index c46a5805b294..5527344546e5 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -168,8 +168,7 @@ struct adf_hw_device_data { int (*send_admin_init)(struct adf_accel_dev *accel_dev); int (*init_arb)(struct adf_accel_dev *accel_dev); void (*exit_arb)(struct adf_accel_dev *accel_dev); - void (*get_arb_mapping)(struct adf_accel_dev *accel_dev, - const u32 **cfg); + const u32 *(*get_arb_mapping)(void); void (*disable_iov)(struct adf_accel_dev *accel_dev); void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, bool enable); diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index eb9b3be9d8eb..96b437bfe3de 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -464,3 +464,4 @@ MODULE_AUTHOR("Intel"); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_ALIAS_CRYPTO("intel_qat"); MODULE_VERSION(ADF_DRV_VERSION); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index 9f5240d9488b..64e4596a24f4 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -19,6 +19,7 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; void __iomem *csr = accel_dev->transport->banks[0].csr_addr; + unsigned long ae_mask = hw_data->ae_mask; u32 arb_off, wt_off, arb_cfg; const u32 *thd_2_arb_cfg; struct arb_info info; @@ -35,12 +36,9 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) WRITE_CSR_ARB_SARCONFIG(csr, arb_off, arb, arb_cfg); /* Map worker threads to service arbiters */ - hw_data->get_arb_mapping(accel_dev, &thd_2_arb_cfg); + thd_2_arb_cfg = hw_data->get_arb_mapping(); - if (!thd_2_arb_cfg) - return -EFAULT; - - for (i = 0; i < hw_data->num_engines; i++) + for_each_set_bit(i, &ae_mask, hw_data->num_engines) WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, thd_2_arb_cfg[i]); return 0; diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 5a7030acdc33..888c1e047295 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2014 - 2020 Intel Corporation */ #include +#include #include "adf_accel_devices.h" #include "adf_transport_internal.h" #include "adf_transport_access_macros.h" @@ -246,6 +247,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, return -EFAULT; } + ring_num = array_index_nospec(ring_num, num_rings_per_bank); bank = &transport_data->banks[bank_num]; if (adf_reserve_ring(bank, ring_num)) { dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n", diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index 1205186ad51e..e69e5907f595 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -62,8 +62,8 @@ static int adf_ring_show(struct seq_file *sfile, void *v) seq_printf(sfile, "head %x, tail %x, empty: %d\n", head, tail, (empty & 1 << ring->ring_number) >> ring->ring_number); - seq_printf(sfile, "ring size %d, msg size %d\n", - ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size), + seq_printf(sfile, "ring size %lld, msg size %d\n", + (long long)ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size), ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); seq_puts(sfile, "----------- Ring data ------------\n"); return 0; diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 31c7a206a629..ff78c73c47e3 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 2c863d25327a..b0b78445418b 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -321,13 +321,13 @@ static int qat_dh_compute_value(struct kpp_request *req) qat_req->out.dh.out_tab[1] = 0; /* Mapping in.in.b or in.in_g2.xa is the same */ qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh.in.b, - sizeof(struct qat_dh_input_params), + sizeof(qat_req->in.dh.in.b), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh.r, - sizeof(struct qat_dh_output_params), + sizeof(qat_req->out.dh.r), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; @@ -716,13 +716,13 @@ static int qat_rsa_enc(struct akcipher_request *req) qat_req->in.rsa.in_tab[3] = 0; qat_req->out.rsa.out_tab[1] = 0; qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.enc.m, - sizeof(struct qat_rsa_input_params), + sizeof(qat_req->in.rsa.enc.m), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.enc.c, - sizeof(struct qat_rsa_output_params), + sizeof(qat_req->out.rsa.enc.c), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; @@ -864,13 +864,13 @@ static int qat_rsa_dec(struct akcipher_request *req) qat_req->in.rsa.in_tab[3] = 0; qat_req->out.rsa.out_tab[1] = 0; qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.dec.c, - sizeof(struct qat_rsa_input_params), + sizeof(qat_req->in.rsa.dec.c), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.dec.m, - sizeof(struct qat_rsa_output_params), + sizeof(qat_req->out.rsa.dec.m), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 1e83d9397b11..7dd7cd6c3ef8 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -7,14 +7,8 @@ #include "adf_dh895xcc_hw_data.h" #include "icp_qat_hw.h" -/* Worker thread to service arbiter mappings based on dev SKUs */ -static const u32 thrd_to_arb_map_sku4[] = { - 0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666, - 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, - 0x00000000, 0x00000000, 0x00000000, 0x00000000 -}; - -static const u32 thrd_to_arb_map_sku6[] = { +/* Worker thread to service arbiter mappings */ +static const u32 thrd_to_arb_map[ADF_DH895XCC_MAX_ACCELENGINES] = { 0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666, 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222 @@ -127,23 +121,9 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_UNKNOWN; } -static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, - u32 const **arb_map_config) +static const u32 *adf_get_arbiter_mapping(void) { - switch (accel_dev->accel_pci_dev.sku) { - case DEV_SKU_1: - *arb_map_config = thrd_to_arb_map_sku4; - break; - - case DEV_SKU_2: - case DEV_SKU_4: - *arb_map_config = thrd_to_arb_map_sku6; - break; - default: - dev_err(&GET_DEV(accel_dev), - "The configuration doesn't match any SKU"); - *arb_map_config = NULL; - } + return thrd_to_arb_map; } static u32 get_pf2vf_offset(u32 i) diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 8b5be29cb4dc..457084b344c1 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -1350,12 +1350,6 @@ static void sahara_unregister_algs(struct sahara_dev *dev) crypto_unregister_ahash(&sha_v4_algs[i]); } -static const struct platform_device_id sahara_platform_ids[] = { - { .name = "sahara-imx27" }, - { /* sentinel */ } -}; -MODULE_DEVICE_TABLE(platform, sahara_platform_ids); - static const struct of_device_id sahara_dt_ids[] = { { .compatible = "fsl,imx53-sahara" }, { .compatible = "fsl,imx27-sahara" }, @@ -1540,7 +1534,6 @@ static struct platform_driver sahara_driver = { .name = SAHARA_NAME, .of_match_table = sahara_dt_ids, }, - .id_table = sahara_platform_ids, }; module_platform_driver(sahara_driver); diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 2670c30332fa..2a4793176c71 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -1229,7 +1229,7 @@ static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) cr = stm32_cryp_read(cryp, CRYP_CR); stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN); - stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->last_ctr); + stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->last_ctr); stm32_cryp_write(cryp, CRYP_CR, cr); } diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 4fd85f31630a..25c9f825b8b5 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1093,11 +1093,12 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev, */ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, unsigned int offset, int datalen, int elen, - struct talitos_ptr *link_tbl_ptr) + struct talitos_ptr *link_tbl_ptr, int align) { int n_sg = elen ? sg_count + 1 : sg_count; int count = 0; int cryptlen = datalen + elen; + int padding = ALIGN(cryptlen, align) - cryptlen; while (cryptlen && sg && n_sg--) { unsigned int len = sg_dma_len(sg); @@ -1121,7 +1122,7 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, offset += datalen; } to_talitos_ptr(link_tbl_ptr + count, - sg_dma_address(sg) + offset, len, 0); + sg_dma_address(sg) + offset, sg_next(sg) ? len : len + padding, 0); to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); count++; cryptlen -= len; @@ -1144,10 +1145,11 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, unsigned int len, struct talitos_edesc *edesc, struct talitos_ptr *ptr, int sg_count, unsigned int offset, int tbl_off, int elen, - bool force) + bool force, int align) { struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); + int aligned_len = ALIGN(len, align); if (!src) { to_talitos_ptr(ptr, 0, 0, is_sec1); @@ -1155,22 +1157,22 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, } to_talitos_ptr_ext_set(ptr, elen, is_sec1); if (sg_count == 1 && !force) { - to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1); + to_talitos_ptr(ptr, sg_dma_address(src) + offset, aligned_len, is_sec1); return sg_count; } if (is_sec1) { - to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1); + to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, aligned_len, is_sec1); return sg_count; } sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, - &edesc->link_tbl[tbl_off]); + &edesc->link_tbl[tbl_off], align); if (sg_count == 1 && !force) { /* Only one segment now, so no link tbl needed*/ copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); return sg_count; } to_talitos_ptr(ptr, edesc->dma_link_tbl + - tbl_off * sizeof(struct talitos_ptr), len, is_sec1); + tbl_off * sizeof(struct talitos_ptr), aligned_len, is_sec1); to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); return sg_count; @@ -1182,7 +1184,7 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src, unsigned int offset, int tbl_off) { return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, - tbl_off, 0, false); + tbl_off, 0, false, 1); } /* @@ -1251,7 +1253,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], sg_count, areq->assoclen, tbl_off, elen, - false); + false, 1); if (ret > 1) { tbl_off += ret; @@ -1271,7 +1273,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, elen = 0; ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], sg_count, areq->assoclen, tbl_off, elen, - is_ipsec_esp && !encrypt); + is_ipsec_esp && !encrypt, 1); tbl_off += ret; if (!encrypt && is_ipsec_esp) { @@ -1577,6 +1579,8 @@ static int common_nonsnoop(struct talitos_edesc *edesc, bool sync_needed = false; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); + bool is_ctr = (desc->hdr & DESC_HDR_SEL0_MASK) == DESC_HDR_SEL0_AESU && + (desc->hdr & DESC_HDR_MODE0_AESU_MASK) == DESC_HDR_MODE0_AESU_CTR; /* first DWORD empty */ @@ -1597,8 +1601,8 @@ static int common_nonsnoop(struct talitos_edesc *edesc, /* * cipher in */ - sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc, - &desc->ptr[3], sg_count, 0, 0); + sg_count = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[3], + sg_count, 0, 0, 0, false, is_ctr ? 16 : 1); if (sg_count > 1) sync_needed = true; @@ -2761,6 +2765,22 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_SEL0_AESU | DESC_HDR_MODE0_AESU_CTR, }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-talitos", + .base.cra_blocksize = 1, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = skcipher_aes_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CTR, + }, { .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.skcipher = { .base.cra_name = "ecb(des)", @@ -3178,6 +3198,12 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey; t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt; t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt; + if (!strcmp(alg->cra_name, "ctr(aes)") && !has_ftr_sec1(priv) && + DESC_TYPE(t_alg->algt.desc_hdr_template) != + DESC_TYPE(DESC_HDR_TYPE_AESU_CTR_NONSNOOP)) { + devm_kfree(dev, t_alg); + return ERR_PTR(-ENOTSUPP); + } break; case CRYPTO_ALG_TYPE_AEAD: alg = &t_alg->algt.alg.aead.base; diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 1469b956948a..32825119e880 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -344,6 +344,7 @@ static inline bool has_ftr_sec1(struct talitos_private *priv) /* primary execution unit mode (MODE0) and derivatives */ #define DESC_HDR_MODE0_ENCRYPT cpu_to_be32(0x00100000) +#define DESC_HDR_MODE0_AESU_MASK cpu_to_be32(0x00600000) #define DESC_HDR_MODE0_AESU_CBC cpu_to_be32(0x00200000) #define DESC_HDR_MODE0_AESU_CTR cpu_to_be32(0x00600000) #define DESC_HDR_MODE0_DEU_CBC cpu_to_be32(0x00400000) diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c index 2bc5d4e1adf4..d05c02baebcf 100644 --- a/drivers/crypto/vmx/aes.c +++ b/drivers/crypto/vmx/aes.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "aesp8-ppc.h" diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h index 01774a4d26a2..5764d4438388 100644 --- a/drivers/crypto/vmx/aesp8-ppc.h +++ b/drivers/crypto/vmx/aesp8-ppc.h @@ -7,6 +7,12 @@ struct aes_key { int rounds; }; +extern struct shash_alg p8_ghash_alg; +extern struct crypto_alg p8_aes_alg; +extern struct skcipher_alg p8_aes_cbc_alg; +extern struct skcipher_alg p8_aes_ctr_alg; +extern struct skcipher_alg p8_aes_xts_alg; + int aes_p8_set_encrypt_key(const u8 *userKey, const int bits, struct aes_key *key); int aes_p8_set_decrypt_key(const u8 *userKey, const int bits, diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c index 3e0335fb406c..a40d08e75fc0 100644 --- a/drivers/crypto/vmx/vmx.c +++ b/drivers/crypto/vmx/vmx.c @@ -17,11 +17,7 @@ #include #include -extern struct shash_alg p8_ghash_alg; -extern struct crypto_alg p8_aes_alg; -extern struct skcipher_alg p8_aes_cbc_alg; -extern struct skcipher_alg p8_aes_ctr_alg; -extern struct skcipher_alg p8_aes_xts_alg; +#include "aesp8-ppc.h" static int __init p8_init(void) { @@ -78,3 +74,4 @@ MODULE_DESCRIPTION("IBM VMX cryptographic acceleration instructions " "support on Power 8"); MODULE_LICENSE("GPL"); MODULE_VERSION("1.0.0"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig new file mode 100644 index 000000000000..97dc4d751651 --- /dev/null +++ b/drivers/cxl/Kconfig @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig CXL_BUS + tristate "CXL (Compute Express Link) Devices Support" + depends on PCI + help + CXL is a bus that is electrically compatible with PCI Express, but + layers three protocols on that signalling (CXL.io, CXL.cache, and + CXL.mem). The CXL.cache protocol allows devices to hold cachelines + locally, the CXL.mem protocol allows devices to be fully coherent + memory targets, the CXL.io protocol is equivalent to PCI Express. + Say 'y' to enable support for the configuration and management of + devices supporting these protocols. + +if CXL_BUS + +config CXL_MEM + tristate "CXL.mem: Memory Devices" + help + The CXL.mem protocol allows a device to act as a provider of + "System RAM" and/or "Persistent Memory" that is fully coherent + as if the memory was attached to the typical CPU memory + controller. + + Say 'y/m' to enable a driver (named "cxl_mem.ko" when built as + a module) that will attach to CXL.mem devices for + configuration, provisioning, and health monitoring. This + driver is required for dynamic provisioning of CXL.mem + attached memory which is a prerequisite for persistent memory + support. Typically volatile memory is mapped by platform + firmware and included in the platform memory map, but in some + cases the OS is responsible for mapping that memory. See + Chapter 2.3 Type 3 CXL Device in the CXL 2.0 specification. + + If unsure say 'm'. + +config CXL_MEM_RAW_COMMANDS + bool "RAW Command Interface for Memory Devices" + depends on CXL_MEM + help + Enable CXL RAW command interface. + + The CXL driver ioctl interface may assign a kernel ioctl command + number for each specification defined opcode. At any given point in + time the number of opcodes that the specification defines and a device + may implement may exceed the kernel's set of associated ioctl function + numbers. The mismatch is either by omission, specification is too new, + or by design. When prototyping new hardware, or developing / debugging + the driver it is useful to be able to submit any possible command to + the hardware, even commands that may crash the kernel due to their + potential impact to memory currently in use by the kernel. + + If developing CXL hardware or the driver say Y, otherwise say N. +endif diff --git a/drivers/cxl/Makefile b/drivers/cxl/Makefile new file mode 100644 index 000000000000..a314a1891f4d --- /dev/null +++ b/drivers/cxl/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CXL_BUS) += cxl_bus.o +obj-$(CONFIG_CXL_MEM) += cxl_mem.o + +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CXL +cxl_bus-y := bus.o +cxl_mem-y := mem.o diff --git a/drivers/cxl/bus.c b/drivers/cxl/bus.c new file mode 100644 index 000000000000..58f74796d525 --- /dev/null +++ b/drivers/cxl/bus.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include +#include + +/** + * DOC: cxl bus + * + * The CXL bus provides namespace for control devices and a rendezvous + * point for cross-device interleave coordination. + */ +struct bus_type cxl_bus_type = { + .name = "cxl", +}; +EXPORT_SYMBOL_GPL(cxl_bus_type); + +static __init int cxl_bus_init(void) +{ + return bus_register(&cxl_bus_type); +} + +static void cxl_bus_exit(void) +{ + bus_unregister(&cxl_bus_type); +} + +module_init(cxl_bus_init); +module_exit(cxl_bus_exit); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h new file mode 100644 index 000000000000..6f14838c2d25 --- /dev/null +++ b/drivers/cxl/cxl.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2020 Intel Corporation. */ + +#ifndef __CXL_H__ +#define __CXL_H__ + +#include +#include +#include + +/* CXL 2.0 8.2.8.1 Device Capabilities Array Register */ +#define CXLDEV_CAP_ARRAY_OFFSET 0x0 +#define CXLDEV_CAP_ARRAY_CAP_ID 0 +#define CXLDEV_CAP_ARRAY_ID_MASK GENMASK_ULL(15, 0) +#define CXLDEV_CAP_ARRAY_COUNT_MASK GENMASK_ULL(47, 32) +/* CXL 2.0 8.2.8.2 CXL Device Capability Header Register */ +#define CXLDEV_CAP_HDR_CAP_ID_MASK GENMASK(15, 0) +/* CXL 2.0 8.2.8.2.1 CXL Device Capabilities */ +#define CXLDEV_CAP_CAP_ID_DEVICE_STATUS 0x1 +#define CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX 0x2 +#define CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX 0x3 +#define CXLDEV_CAP_CAP_ID_MEMDEV 0x4000 + +/* CXL 2.0 8.2.8.4 Mailbox Registers */ +#define CXLDEV_MBOX_CAPS_OFFSET 0x00 +#define CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK GENMASK(4, 0) +#define CXLDEV_MBOX_CTRL_OFFSET 0x04 +#define CXLDEV_MBOX_CTRL_DOORBELL BIT(0) +#define CXLDEV_MBOX_CMD_OFFSET 0x08 +#define CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0) +#define CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK GENMASK_ULL(36, 16) +#define CXLDEV_MBOX_STATUS_OFFSET 0x10 +#define CXLDEV_MBOX_STATUS_RET_CODE_MASK GENMASK_ULL(47, 32) +#define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18 +#define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20 + +/* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ +#define CXLMDEV_STATUS_OFFSET 0x0 +#define CXLMDEV_DEV_FATAL BIT(0) +#define CXLMDEV_FW_HALT BIT(1) +#define CXLMDEV_STATUS_MEDIA_STATUS_MASK GENMASK(3, 2) +#define CXLMDEV_MS_NOT_READY 0 +#define CXLMDEV_MS_READY 1 +#define CXLMDEV_MS_ERROR 2 +#define CXLMDEV_MS_DISABLED 3 +#define CXLMDEV_READY(status) \ + (FIELD_GET(CXLMDEV_STATUS_MEDIA_STATUS_MASK, status) == \ + CXLMDEV_MS_READY) +#define CXLMDEV_MBOX_IF_READY BIT(4) +#define CXLMDEV_RESET_NEEDED_MASK GENMASK(7, 5) +#define CXLMDEV_RESET_NEEDED_NOT 0 +#define CXLMDEV_RESET_NEEDED_COLD 1 +#define CXLMDEV_RESET_NEEDED_WARM 2 +#define CXLMDEV_RESET_NEEDED_HOT 3 +#define CXLMDEV_RESET_NEEDED_CXL 4 +#define CXLMDEV_RESET_NEEDED(status) \ + (FIELD_GET(CXLMDEV_RESET_NEEDED_MASK, status) != \ + CXLMDEV_RESET_NEEDED_NOT) + +struct cxl_memdev; +/** + * struct cxl_mem - A CXL memory device + * @pdev: The PCI device associated with this CXL device. + * @regs: IO mappings to the device's MMIO + * @status_regs: CXL 2.0 8.2.8.3 Device Status Registers + * @mbox_regs: CXL 2.0 8.2.8.4 Mailbox Registers + * @memdev_regs: CXL 2.0 8.2.8.5 Memory Device Registers + * @payload_size: Size of space for payload + * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) + * @mbox_mutex: Mutex to synchronize mailbox access. + * @firmware_version: Firmware version for the memory device. + * @enabled_commands: Hardware commands found enabled in CEL. + * @pmem_range: Persistent memory capacity information. + * @ram_range: Volatile memory capacity information. + */ +struct cxl_mem { + struct pci_dev *pdev; + void __iomem *regs; + struct cxl_memdev *cxlmd; + + void __iomem *status_regs; + void __iomem *mbox_regs; + void __iomem *memdev_regs; + + size_t payload_size; + struct mutex mbox_mutex; /* Protects device mailbox and firmware */ + char firmware_version[0x10]; + unsigned long *enabled_cmds; + + struct range pmem_range; + struct range ram_range; +}; + +extern struct bus_type cxl_bus_type; +#endif /* __CXL_H__ */ diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c new file mode 100644 index 000000000000..244cb7d89678 --- /dev/null +++ b/drivers/cxl/mem.c @@ -0,0 +1,1552 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pci.h" +#include "cxl.h" + +/** + * DOC: cxl mem + * + * This implements a CXL memory device ("type-3") as it is defined by the + * Compute Express Link specification. + * + * The driver has several responsibilities, mainly: + * - Create the memX device and register on the CXL bus. + * - Enumerate device's register interface and map them. + * - Probe the device attributes to establish sysfs interface. + * - Provide an IOCTL interface to userspace to communicate with the device for + * things like firmware update. + * - Support management of interleave sets. + * - Handle and manage error conditions. + */ + +/* + * An entire PCI topology full of devices should be enough for any + * config + */ +#define CXL_MEM_MAX_DEVS 65536 + +#define cxl_doorbell_busy(cxlm) \ + (readl((cxlm)->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET) & \ + CXLDEV_MBOX_CTRL_DOORBELL) + +/* CXL 2.0 - 8.2.8.4 */ +#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) + +enum opcode { + CXL_MBOX_OP_INVALID = 0x0000, + CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID, + CXL_MBOX_OP_GET_FW_INFO = 0x0200, + CXL_MBOX_OP_ACTIVATE_FW = 0x0202, + CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400, + CXL_MBOX_OP_GET_LOG = 0x0401, + CXL_MBOX_OP_IDENTIFY = 0x4000, + CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100, + CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101, + CXL_MBOX_OP_GET_LSA = 0x4102, + CXL_MBOX_OP_SET_LSA = 0x4103, + CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200, + CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204, + CXL_MBOX_OP_SCAN_MEDIA = 0x4304, + CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305, + CXL_MBOX_OP_MAX = 0x10000 +}; + +/** + * struct mbox_cmd - A command to be submitted to hardware. + * @opcode: (input) The command set and command submitted to hardware. + * @payload_in: (input) Pointer to the input payload. + * @payload_out: (output) Pointer to the output payload. Must be allocated by + * the caller. + * @size_in: (input) Number of bytes to load from @payload_in. + * @size_out: (input) Max number of bytes loaded into @payload_out. + * (output) Number of bytes generated by the device. For fixed size + * outputs commands this is always expected to be deterministic. For + * variable sized output commands, it tells the exact number of bytes + * written. + * @return_code: (output) Error code returned from hardware. + * + * This is the primary mechanism used to send commands to the hardware. + * All the fields except @payload_* correspond exactly to the fields described in + * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and + * @payload_out are written to, and read from the Command Payload Registers + * defined in CXL 2.0 8.2.8.4.8. + */ +struct mbox_cmd { + u16 opcode; + void *payload_in; + void *payload_out; + size_t size_in; + size_t size_out; + u16 return_code; +#define CXL_MBOX_SUCCESS 0 +}; + +/** + * struct cxl_memdev - CXL bus object representing a Type-3 Memory Device + * @dev: driver core device object + * @cdev: char dev core object for ioctl operations + * @cxlm: pointer to the parent device driver data + * @ops_active: active user of @cxlm in ops handlers + * @ops_dead: completion when all @cxlm ops users have exited + * @id: id number of this memdev instance. + */ +struct cxl_memdev { + struct device dev; + struct cdev cdev; + struct cxl_mem *cxlm; + struct percpu_ref ops_active; + struct completion ops_dead; + int id; +}; + +static int cxl_mem_major; +static DEFINE_IDA(cxl_memdev_ida); +static struct dentry *cxl_debugfs; +static bool cxl_raw_allow_all; + +enum { + CEL_UUID, + VENDOR_DEBUG_UUID, +}; + +/* See CXL 2.0 Table 170. Get Log Input Payload */ +static const uuid_t log_uuid[] = { + [CEL_UUID] = UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96, + 0xb1, 0x62, 0x3b, 0x3f, 0x17), + [VENDOR_DEBUG_UUID] = UUID_INIT(0xe1819d9, 0x11a9, 0x400c, 0x81, 0x1f, + 0xd6, 0x07, 0x19, 0x40, 0x3d, 0x86), +}; + +/** + * struct cxl_mem_command - Driver representation of a memory device command + * @info: Command information as it exists for the UAPI + * @opcode: The actual bits used for the mailbox protocol + * @flags: Set of flags effecting driver behavior. + * + * * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag + * will be enabled by the driver regardless of what hardware may have + * advertised. + * + * The cxl_mem_command is the driver's internal representation of commands that + * are supported by the driver. Some of these commands may not be supported by + * the hardware. The driver will use @info to validate the fields passed in by + * the user then submit the @opcode to the hardware. + * + * See struct cxl_command_info. + */ +struct cxl_mem_command { + struct cxl_command_info info; + enum opcode opcode; + u32 flags; +#define CXL_CMD_FLAG_NONE 0 +#define CXL_CMD_FLAG_FORCE_ENABLE BIT(0) +}; + +#define CXL_CMD(_id, sin, sout, _flags) \ + [CXL_MEM_COMMAND_ID_##_id] = { \ + .info = { \ + .id = CXL_MEM_COMMAND_ID_##_id, \ + .size_in = sin, \ + .size_out = sout, \ + }, \ + .opcode = CXL_MBOX_OP_##_id, \ + .flags = _flags, \ + } + +/* + * This table defines the supported mailbox commands for the driver. This table + * is made up of a UAPI structure. Non-negative values as parameters in the + * table will be validated against the user's input. For example, if size_in is + * 0, and the user passed in 1, it is an error. + */ +static struct cxl_mem_command mem_commands[] = { + CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE), +#ifdef CONFIG_CXL_MEM_RAW_COMMANDS + CXL_CMD(RAW, ~0, ~0, 0), +#endif + CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE), + CXL_CMD(GET_FW_INFO, 0, 0x50, 0), + CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0), + CXL_CMD(GET_LSA, 0x8, ~0, 0), + CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0), + CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE), +}; + +/* + * Commands that RAW doesn't permit. The rationale for each: + * + * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment / + * coordination of transaction timeout values at the root bridge level. + * + * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live + * and needs to be coordinated with HDM updates. + * + * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the + * driver and any writes from userspace invalidates those contents. + * + * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes + * to the device after it is marked clean, userspace can not make that + * assertion. + * + * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that + * is kept up to date with patrol notifications and error management. + */ +static u16 cxl_disabled_raw_commands[] = { + CXL_MBOX_OP_ACTIVATE_FW, + CXL_MBOX_OP_SET_PARTITION_INFO, + CXL_MBOX_OP_SET_LSA, + CXL_MBOX_OP_SET_SHUTDOWN_STATE, + CXL_MBOX_OP_SCAN_MEDIA, + CXL_MBOX_OP_GET_SCAN_MEDIA, +}; + +/* + * Command sets that RAW doesn't permit. All opcodes in this set are + * disabled because they pass plain text security payloads over the + * user/kernel boundary. This functionality is intended to be wrapped + * behind the keys ABI which allows for encrypted payloads in the UAPI + */ +static u8 security_command_sets[] = { + 0x44, /* Sanitize */ + 0x45, /* Persistent Memory Data-at-rest Security */ + 0x46, /* Security Passthrough */ +}; + +#define cxl_for_each_cmd(cmd) \ + for ((cmd) = &mem_commands[0]; \ + ((cmd) - mem_commands) < ARRAY_SIZE(mem_commands); (cmd)++) + +#define cxl_cmd_count ARRAY_SIZE(mem_commands) + +static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm) +{ + const unsigned long start = jiffies; + unsigned long end = start; + + while (cxl_doorbell_busy(cxlm)) { + end = jiffies; + + if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { + /* Check again in case preempted before timeout test */ + if (!cxl_doorbell_busy(cxlm)) + break; + return -ETIMEDOUT; + } + cpu_relax(); + } + + dev_dbg(&cxlm->pdev->dev, "Doorbell wait took %dms", + jiffies_to_msecs(end) - jiffies_to_msecs(start)); + return 0; +} + +static bool cxl_is_security_command(u16 opcode) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(security_command_sets); i++) + if (security_command_sets[i] == (opcode >> 8)) + return true; + return false; +} + +static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, + struct mbox_cmd *mbox_cmd) +{ + struct device *dev = &cxlm->pdev->dev; + + dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n", + mbox_cmd->opcode, mbox_cmd->size_in); +} + +/** + * __cxl_mem_mbox_send_cmd() - Execute a mailbox command + * @cxlm: The CXL memory device to communicate with. + * @mbox_cmd: Command to send to the memory device. + * + * Context: Any context. Expects mbox_mutex to be held. + * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. + * Caller should check the return code in @mbox_cmd to make sure it + * succeeded. + * + * This is a generic form of the CXL mailbox send command thus only using the + * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory + * devices, and perhaps other types of CXL devices may have further information + * available upon error conditions. Driver facilities wishing to send mailbox + * commands should use the wrapper command. + * + * The CXL spec allows for up to two mailboxes. The intention is for the primary + * mailbox to be OS controlled and the secondary mailbox to be used by system + * firmware. This allows the OS and firmware to communicate with the device and + * not need to coordinate with each other. The driver only uses the primary + * mailbox. + */ +static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, + struct mbox_cmd *mbox_cmd) +{ + void __iomem *payload = cxlm->mbox_regs + CXLDEV_MBOX_PAYLOAD_OFFSET; + u64 cmd_reg, status_reg; + size_t out_len; + int rc; + + lockdep_assert_held(&cxlm->mbox_mutex); + + /* + * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. + * 1. Caller reads MB Control Register to verify doorbell is clear + * 2. Caller writes Command Register + * 3. Caller writes Command Payload Registers if input payload is non-empty + * 4. Caller writes MB Control Register to set doorbell + * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured + * 6. Caller reads MB Status Register to fetch Return code + * 7. If command successful, Caller reads Command Register to get Payload Length + * 8. If output payload is non-empty, host reads Command Payload Registers + * + * Hardware is free to do whatever it wants before the doorbell is rung, + * and isn't allowed to change anything after it clears the doorbell. As + * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can + * also happen in any order (though some orders might not make sense). + */ + + /* #1 */ + if (cxl_doorbell_busy(cxlm)) { + dev_err_ratelimited(&cxlm->pdev->dev, + "Mailbox re-busy after acquiring\n"); + return -EBUSY; + } + + cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, + mbox_cmd->opcode); + if (mbox_cmd->size_in) { + if (WARN_ON(!mbox_cmd->payload_in)) + return -EINVAL; + + cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, + mbox_cmd->size_in); + memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); + } + + /* #2, #3 */ + writeq(cmd_reg, cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); + + /* #4 */ + dev_dbg(&cxlm->pdev->dev, "Sending command\n"); + writel(CXLDEV_MBOX_CTRL_DOORBELL, + cxlm->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET); + + /* #5 */ + rc = cxl_mem_wait_for_doorbell(cxlm); + if (rc == -ETIMEDOUT) { + cxl_mem_mbox_timeout(cxlm, mbox_cmd); + return rc; + } + + /* #6 */ + status_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_STATUS_OFFSET); + mbox_cmd->return_code = + FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); + + if (mbox_cmd->return_code != 0) { + dev_dbg(&cxlm->pdev->dev, "Mailbox operation had an error\n"); + return 0; + } + + /* #7 */ + cmd_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); + out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); + + /* #8 */ + if (out_len && mbox_cmd->payload_out) { + /* + * Sanitize the copy. If hardware misbehaves, out_len per the + * spec can actually be greater than the max allowed size (21 + * bits available but spec defined 1M max). The caller also may + * have requested less data than the hardware supplied even + * within spec. + */ + size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len); + + memcpy_fromio(mbox_cmd->payload_out, payload, n); + mbox_cmd->size_out = n; + } else { + mbox_cmd->size_out = 0; + } + + return 0; +} + +/** + * cxl_mem_mbox_get() - Acquire exclusive access to the mailbox. + * @cxlm: The memory device to gain access to. + * + * Context: Any context. Takes the mbox_mutex. + * Return: 0 if exclusive access was acquired. + */ +static int cxl_mem_mbox_get(struct cxl_mem *cxlm) +{ + struct device *dev = &cxlm->pdev->dev; + u64 md_status; + int rc; + + mutex_lock_io(&cxlm->mbox_mutex); + + /* + * XXX: There is some amount of ambiguity in the 2.0 version of the spec + * around the mailbox interface ready (8.2.8.5.1.1). The purpose of the + * bit is to allow firmware running on the device to notify the driver + * that it's ready to receive commands. It is unclear if the bit needs + * to be read for each transaction mailbox, ie. the firmware can switch + * it on and off as needed. Second, there is no defined timeout for + * mailbox ready, like there is for the doorbell interface. + * + * Assumptions: + * 1. The firmware might toggle the Mailbox Interface Ready bit, check + * it for every command. + * + * 2. If the doorbell is clear, the firmware should have first set the + * Mailbox Interface Ready bit. Therefore, waiting for the doorbell + * to be ready is sufficient. + */ + rc = cxl_mem_wait_for_doorbell(cxlm); + if (rc) { + dev_warn(dev, "Mailbox interface not ready\n"); + goto out; + } + + md_status = readq(cxlm->memdev_regs + CXLMDEV_STATUS_OFFSET); + if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { + dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); + rc = -EBUSY; + goto out; + } + + /* + * Hardware shouldn't allow a ready status but also have failure bits + * set. Spit out an error, this should be a bug report + */ + rc = -EFAULT; + if (md_status & CXLMDEV_DEV_FATAL) { + dev_err(dev, "mbox: reported ready, but fatal\n"); + goto out; + } + if (md_status & CXLMDEV_FW_HALT) { + dev_err(dev, "mbox: reported ready, but halted\n"); + goto out; + } + if (CXLMDEV_RESET_NEEDED(md_status)) { + dev_err(dev, "mbox: reported ready, but reset needed\n"); + goto out; + } + + /* with lock held */ + return 0; + +out: + mutex_unlock(&cxlm->mbox_mutex); + return rc; +} + +/** + * cxl_mem_mbox_put() - Release exclusive access to the mailbox. + * @cxlm: The CXL memory device to communicate with. + * + * Context: Any context. Expects mbox_mutex to be held. + */ +static void cxl_mem_mbox_put(struct cxl_mem *cxlm) +{ + mutex_unlock(&cxlm->mbox_mutex); +} + +/** + * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace. + * @cxlm: The CXL memory device to communicate with. + * @cmd: The validated command. + * @in_payload: Pointer to userspace's input payload. + * @out_payload: Pointer to userspace's output payload. + * @size_out: (Input) Max payload size to copy out. + * (Output) Payload size hardware generated. + * @retval: Hardware generated return code from the operation. + * + * Return: + * * %0 - Mailbox transaction succeeded. This implies the mailbox + * protocol completed successfully not that the operation itself + * was successful. + * * %-ENOMEM - Couldn't allocate a bounce buffer. + * * %-EFAULT - Something happened with copy_to/from_user. + * * %-EINTR - Mailbox acquisition interrupted. + * * %-EXXX - Transaction level failures. + * + * Creates the appropriate mailbox command and dispatches it on behalf of a + * userspace request. The input and output payloads are copied between + * userspace. + * + * See cxl_send_cmd(). + */ +static int handle_mailbox_cmd_from_user(struct cxl_mem *cxlm, + const struct cxl_mem_command *cmd, + u64 in_payload, u64 out_payload, + s32 *size_out, u32 *retval) +{ + struct device *dev = &cxlm->pdev->dev; + struct mbox_cmd mbox_cmd = { + .opcode = cmd->opcode, + .size_in = cmd->info.size_in, + .size_out = cmd->info.size_out, + }; + int rc; + + if (cmd->info.size_out) { + mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL); + if (!mbox_cmd.payload_out) + return -ENOMEM; + } + + if (cmd->info.size_in) { + mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload), + cmd->info.size_in); + if (IS_ERR(mbox_cmd.payload_in)) { + kvfree(mbox_cmd.payload_out); + return PTR_ERR(mbox_cmd.payload_in); + } + } + + rc = cxl_mem_mbox_get(cxlm); + if (rc) + goto out; + + dev_dbg(dev, + "Submitting %s command for user\n" + "\topcode: %x\n" + "\tsize: %ub\n", + cxl_command_names[cmd->info.id].name, mbox_cmd.opcode, + cmd->info.size_in); + + dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW, + "raw command path used\n"); + + rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); + cxl_mem_mbox_put(cxlm); + if (rc) + goto out; + + /* + * @size_out contains the max size that's allowed to be written back out + * to userspace. While the payload may have written more output than + * this it will have to be ignored. + */ + if (mbox_cmd.size_out) { + dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out, + "Invalid return size\n"); + if (copy_to_user(u64_to_user_ptr(out_payload), + mbox_cmd.payload_out, mbox_cmd.size_out)) { + rc = -EFAULT; + goto out; + } + } + + *size_out = mbox_cmd.size_out; + *retval = mbox_cmd.return_code; + +out: + kvfree(mbox_cmd.payload_in); + kvfree(mbox_cmd.payload_out); + return rc; +} + +static bool cxl_mem_raw_command_allowed(u16 opcode) +{ + int i; + + if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS)) + return false; + + if (security_locked_down(LOCKDOWN_NONE)) + return false; + + if (cxl_raw_allow_all) + return true; + + if (cxl_is_security_command(opcode)) + return false; + + for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++) + if (cxl_disabled_raw_commands[i] == opcode) + return false; + + return true; +} + +/** + * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND. + * @cxlm: &struct cxl_mem device whose mailbox will be used. + * @send_cmd: &struct cxl_send_command copied in from userspace. + * @out_cmd: Sanitized and populated &struct cxl_mem_command. + * + * Return: + * * %0 - @out_cmd is ready to send. + * * %-ENOTTY - Invalid command specified. + * * %-EINVAL - Reserved fields or invalid values were used. + * * %-ENOMEM - Input or output buffer wasn't sized properly. + * * %-EPERM - Attempted to use a protected command. + * + * The result of this command is a fully validated command in @out_cmd that is + * safe to send to the hardware. + * + * See handle_mailbox_cmd_from_user() + */ +static int cxl_validate_cmd_from_user(struct cxl_mem *cxlm, + const struct cxl_send_command *send_cmd, + struct cxl_mem_command *out_cmd) +{ + const struct cxl_command_info *info; + struct cxl_mem_command *c; + + if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX) + return -ENOTTY; + + /* + * The user can never specify an input payload larger than what hardware + * supports, but output can be arbitrarily large (simply write out as + * much data as the hardware provides). + */ + if (send_cmd->in.size > cxlm->payload_size) + return -EINVAL; + + /* + * Checks are bypassed for raw commands but a WARN/taint will occur + * later in the callchain + */ + if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) { + const struct cxl_mem_command temp = { + .info = { + .id = CXL_MEM_COMMAND_ID_RAW, + .flags = 0, + .size_in = send_cmd->in.size, + .size_out = send_cmd->out.size, + }, + .opcode = send_cmd->raw.opcode + }; + + if (send_cmd->raw.rsvd) + return -EINVAL; + + /* + * Unlike supported commands, the output size of RAW commands + * gets passed along without further checking, so it must be + * validated here. + */ + if (send_cmd->out.size > cxlm->payload_size) + return -EINVAL; + + if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode)) + return -EPERM; + + memcpy(out_cmd, &temp, sizeof(temp)); + + return 0; + } + + if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK) + return -EINVAL; + + if (send_cmd->rsvd) + return -EINVAL; + + if (send_cmd->in.rsvd || send_cmd->out.rsvd) + return -EINVAL; + + /* Convert user's command into the internal representation */ + c = &mem_commands[send_cmd->id]; + info = &c->info; + + /* Check that the command is enabled for hardware */ + if (!test_bit(info->id, cxlm->enabled_cmds)) + return -ENOTTY; + + /* Check the input buffer is the expected size */ + if (info->size_in >= 0 && info->size_in != send_cmd->in.size) + return -ENOMEM; + + /* Check the output buffer is at least large enough */ + if (info->size_out >= 0 && send_cmd->out.size < info->size_out) + return -ENOMEM; + + memcpy(out_cmd, c, sizeof(*c)); + out_cmd->info.size_in = send_cmd->in.size; + /* + * XXX: out_cmd->info.size_out will be controlled by the driver, and the + * specified number of bytes @send_cmd->out.size will be copied back out + * to userspace. + */ + + return 0; +} + +static int cxl_query_cmd(struct cxl_memdev *cxlmd, + struct cxl_mem_query_commands __user *q) +{ + struct device *dev = &cxlmd->dev; + struct cxl_mem_command *cmd; + u32 n_commands; + int j = 0; + + dev_dbg(dev, "Query IOCTL\n"); + + if (get_user(n_commands, &q->n_commands)) + return -EFAULT; + + /* returns the total number if 0 elements are requested. */ + if (n_commands == 0) + return put_user(cxl_cmd_count, &q->n_commands); + + /* + * otherwise, return max(n_commands, total commands) cxl_command_info + * structures. + */ + cxl_for_each_cmd(cmd) { + const struct cxl_command_info *info = &cmd->info; + + if (copy_to_user(&q->commands[j++], info, sizeof(*info))) + return -EFAULT; + + if (j == n_commands) + break; + } + + return 0; +} + +static int cxl_send_cmd(struct cxl_memdev *cxlmd, + struct cxl_send_command __user *s) +{ + struct cxl_mem *cxlm = cxlmd->cxlm; + struct device *dev = &cxlmd->dev; + struct cxl_send_command send; + struct cxl_mem_command c; + int rc; + + dev_dbg(dev, "Send IOCTL\n"); + + if (copy_from_user(&send, s, sizeof(send))) + return -EFAULT; + + rc = cxl_validate_cmd_from_user(cxlmd->cxlm, &send, &c); + if (rc) + return rc; + + /* Prepare to handle a full payload for variable sized output */ + if (c.info.size_out < 0) + c.info.size_out = cxlm->payload_size; + + rc = handle_mailbox_cmd_from_user(cxlm, &c, send.in.payload, + send.out.payload, &send.out.size, + &send.retval); + if (rc) + return rc; + + if (copy_to_user(s, &send, sizeof(send))) + return -EFAULT; + + return 0; +} + +static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, + unsigned long arg) +{ + switch (cmd) { + case CXL_MEM_QUERY_COMMANDS: + return cxl_query_cmd(cxlmd, (void __user *)arg); + case CXL_MEM_SEND_COMMAND: + return cxl_send_cmd(cxlmd, (void __user *)arg); + default: + return -ENOTTY; + } +} + +static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct cxl_memdev *cxlmd; + struct inode *inode; + int rc = -ENOTTY; + + inode = file_inode(file); + cxlmd = container_of(inode->i_cdev, typeof(*cxlmd), cdev); + + if (!percpu_ref_tryget_live(&cxlmd->ops_active)) + return -ENXIO; + + rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); + + percpu_ref_put(&cxlmd->ops_active); + + return rc; +} + +static const struct file_operations cxl_memdev_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = cxl_memdev_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .llseek = noop_llseek, +}; + +static inline struct cxl_mem_command *cxl_mem_find_command(u16 opcode) +{ + struct cxl_mem_command *c; + + cxl_for_each_cmd(c) + if (c->opcode == opcode) + return c; + + return NULL; +} + +/** + * cxl_mem_mbox_send_cmd() - Send a mailbox command to a memory device. + * @cxlm: The CXL memory device to communicate with. + * @opcode: Opcode for the mailbox command. + * @in: The input payload for the mailbox command. + * @in_size: The length of the input payload + * @out: Caller allocated buffer for the output. + * @out_size: Expected size of output. + * + * Context: Any context. Will acquire and release mbox_mutex. + * Return: + * * %>=0 - Number of bytes returned in @out. + * * %-E2BIG - Payload is too large for hardware. + * * %-EBUSY - Couldn't acquire exclusive mailbox access. + * * %-EFAULT - Hardware error occurred. + * * %-ENXIO - Command completed, but device reported an error. + * * %-EIO - Unexpected output size. + * + * Mailbox commands may execute successfully yet the device itself reported an + * error. While this distinction can be useful for commands from userspace, the + * kernel will only be able to use results when both are successful. + * + * See __cxl_mem_mbox_send_cmd() + */ +static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, + void *in, size_t in_size, + void *out, size_t out_size) +{ + const struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); + struct mbox_cmd mbox_cmd = { + .opcode = opcode, + .payload_in = in, + .size_in = in_size, + .size_out = out_size, + .payload_out = out, + }; + int rc; + + if (out_size > cxlm->payload_size) + return -E2BIG; + + rc = cxl_mem_mbox_get(cxlm); + if (rc) + return rc; + + rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); + cxl_mem_mbox_put(cxlm); + if (rc) + return rc; + + /* TODO: Map return code to proper kernel style errno */ + if (mbox_cmd.return_code != CXL_MBOX_SUCCESS) + return -ENXIO; + + /* + * Variable sized commands can't be validated and so it's up to the + * caller to do that if they wish. + */ + if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size) + return -EIO; + + return 0; +} + +/** + * cxl_mem_setup_regs() - Setup necessary MMIO. + * @cxlm: The CXL memory device to communicate with. + * + * Return: 0 if all necessary registers mapped. + * + * A memory device is required by spec to implement a certain set of MMIO + * regions. The purpose of this function is to enumerate and map those + * registers. + */ +static int cxl_mem_setup_regs(struct cxl_mem *cxlm) +{ + struct device *dev = &cxlm->pdev->dev; + int cap, cap_count; + u64 cap_array; + + cap_array = readq(cxlm->regs + CXLDEV_CAP_ARRAY_OFFSET); + if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != + CXLDEV_CAP_ARRAY_CAP_ID) + return -ENODEV; + + cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); + + for (cap = 1; cap <= cap_count; cap++) { + void __iomem *register_block; + u32 offset; + u16 cap_id; + + cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, + readl(cxlm->regs + cap * 0x10)); + offset = readl(cxlm->regs + cap * 0x10 + 0x4); + register_block = cxlm->regs + offset; + + switch (cap_id) { + case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: + dev_dbg(dev, "found Status capability (0x%x)\n", offset); + cxlm->status_regs = register_block; + break; + case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: + dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); + cxlm->mbox_regs = register_block; + break; + case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: + dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); + break; + case CXLDEV_CAP_CAP_ID_MEMDEV: + dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); + cxlm->memdev_regs = register_block; + break; + default: + dev_dbg(dev, "Unknown cap ID: %d (0x%x)\n", cap_id, offset); + break; + } + } + + if (!cxlm->status_regs || !cxlm->mbox_regs || !cxlm->memdev_regs) { + dev_err(dev, "registers not found: %s%s%s\n", + !cxlm->status_regs ? "status " : "", + !cxlm->mbox_regs ? "mbox " : "", + !cxlm->memdev_regs ? "memdev" : ""); + return -ENXIO; + } + + return 0; +} + +static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) +{ + const int cap = readl(cxlm->mbox_regs + CXLDEV_MBOX_CAPS_OFFSET); + + cxlm->payload_size = + 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); + + /* + * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register + * + * If the size is too small, mandatory commands will not work and so + * there's no point in going forward. If the size is too large, there's + * no harm is soft limiting it. + */ + cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M); + if (cxlm->payload_size < 256) { + dev_err(&cxlm->pdev->dev, "Mailbox is too small (%zub)", + cxlm->payload_size); + return -ENXIO; + } + + dev_dbg(&cxlm->pdev->dev, "Mailbox payload sized %zu", + cxlm->payload_size); + + return 0; +} + +static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev, u32 reg_lo, + u32 reg_hi) +{ + struct device *dev = &pdev->dev; + struct cxl_mem *cxlm; + void __iomem *regs; + u64 offset; + u8 bar; + int rc; + + cxlm = devm_kzalloc(&pdev->dev, sizeof(*cxlm), GFP_KERNEL); + if (!cxlm) { + dev_err(dev, "No memory available\n"); + return NULL; + } + + offset = ((u64)reg_hi << 32) | FIELD_GET(CXL_REGLOC_ADDR_MASK, reg_lo); + bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); + + /* Basic sanity check that BAR is big enough */ + if (pci_resource_len(pdev, bar) < offset) { + dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, + &pdev->resource[bar], (unsigned long long)offset); + return NULL; + } + + rc = pcim_iomap_regions(pdev, BIT(bar), pci_name(pdev)); + if (rc) { + dev_err(dev, "failed to map registers\n"); + return NULL; + } + regs = pcim_iomap_table(pdev)[bar]; + + mutex_init(&cxlm->mbox_mutex); + cxlm->pdev = pdev; + cxlm->regs = regs + offset; + cxlm->enabled_cmds = + devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), + sizeof(unsigned long), + GFP_KERNEL | __GFP_ZERO); + if (!cxlm->enabled_cmds) { + dev_err(dev, "No memory available for bitmap\n"); + return NULL; + } + + dev_dbg(dev, "Mapped CXL Memory Device resource\n"); + return cxlm; +} + +static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) +{ + int pos; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC); + if (!pos) + return 0; + + while (pos) { + u16 vendor, id; + + pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor); + pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id); + if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id) + return pos; + + pos = pci_find_next_ext_capability(pdev, pos, + PCI_EXT_CAP_ID_DVSEC); + } + + return 0; +} + +static struct cxl_memdev *to_cxl_memdev(struct device *dev) +{ + return container_of(dev, struct cxl_memdev, dev); +} + +static void cxl_memdev_release(struct device *dev) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + + percpu_ref_exit(&cxlmd->ops_active); + ida_free(&cxl_memdev_ida, cxlmd->id); + kfree(cxlmd); +} + +static char *cxl_memdev_devnode(struct device *dev, umode_t *mode, kuid_t *uid, + kgid_t *gid) +{ + return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); +} + +static ssize_t firmware_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + + return sprintf(buf, "%.16s\n", cxlm->firmware_version); +} +static DEVICE_ATTR_RO(firmware_version); + +static ssize_t payload_max_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + + return sprintf(buf, "%zu\n", cxlm->payload_size); +} +static DEVICE_ATTR_RO(payload_max); + +static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + unsigned long long len = range_len(&cxlm->ram_range); + + return sprintf(buf, "%#llx\n", len); +} + +static struct device_attribute dev_attr_ram_size = + __ATTR(size, 0444, ram_size_show, NULL); + +static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + unsigned long long len = range_len(&cxlm->pmem_range); + + return sprintf(buf, "%#llx\n", len); +} + +static struct device_attribute dev_attr_pmem_size = + __ATTR(size, 0444, pmem_size_show, NULL); + +static struct attribute *cxl_memdev_attributes[] = { + &dev_attr_firmware_version.attr, + &dev_attr_payload_max.attr, + NULL, +}; + +static struct attribute *cxl_memdev_pmem_attributes[] = { + &dev_attr_pmem_size.attr, + NULL, +}; + +static struct attribute *cxl_memdev_ram_attributes[] = { + &dev_attr_ram_size.attr, + NULL, +}; + +static struct attribute_group cxl_memdev_attribute_group = { + .attrs = cxl_memdev_attributes, +}; + +static struct attribute_group cxl_memdev_ram_attribute_group = { + .name = "ram", + .attrs = cxl_memdev_ram_attributes, +}; + +static struct attribute_group cxl_memdev_pmem_attribute_group = { + .name = "pmem", + .attrs = cxl_memdev_pmem_attributes, +}; + +static const struct attribute_group *cxl_memdev_attribute_groups[] = { + &cxl_memdev_attribute_group, + &cxl_memdev_ram_attribute_group, + &cxl_memdev_pmem_attribute_group, + NULL, +}; + +static const struct device_type cxl_memdev_type = { + .name = "cxl_memdev", + .release = cxl_memdev_release, + .devnode = cxl_memdev_devnode, + .groups = cxl_memdev_attribute_groups, +}; + +static void cxlmdev_unregister(void *_cxlmd) +{ + struct cxl_memdev *cxlmd = _cxlmd; + struct device *dev = &cxlmd->dev; + + percpu_ref_kill(&cxlmd->ops_active); + cdev_device_del(&cxlmd->cdev, dev); + wait_for_completion(&cxlmd->ops_dead); + cxlmd->cxlm = NULL; + put_device(dev); +} + +static void cxlmdev_ops_active_release(struct percpu_ref *ref) +{ + struct cxl_memdev *cxlmd = + container_of(ref, typeof(*cxlmd), ops_active); + + complete(&cxlmd->ops_dead); +} + +static int cxl_mem_add_memdev(struct cxl_mem *cxlm) +{ + struct pci_dev *pdev = cxlm->pdev; + struct cxl_memdev *cxlmd; + struct device *dev; + struct cdev *cdev; + int rc; + + cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); + if (!cxlmd) + return -ENOMEM; + init_completion(&cxlmd->ops_dead); + + /* + * @cxlm is deallocated when the driver unbinds so operations + * that are using it need to hold a live reference. + */ + cxlmd->cxlm = cxlm; + rc = percpu_ref_init(&cxlmd->ops_active, cxlmdev_ops_active_release, 0, + GFP_KERNEL); + if (rc) + goto err_ref; + + rc = ida_alloc_range(&cxl_memdev_ida, 0, CXL_MEM_MAX_DEVS, GFP_KERNEL); + if (rc < 0) + goto err_id; + cxlmd->id = rc; + + dev = &cxlmd->dev; + device_initialize(dev); + dev->parent = &pdev->dev; + dev->bus = &cxl_bus_type; + dev->devt = MKDEV(cxl_mem_major, cxlmd->id); + dev->type = &cxl_memdev_type; + dev_set_name(dev, "mem%d", cxlmd->id); + + cdev = &cxlmd->cdev; + cdev_init(cdev, &cxl_memdev_fops); + + rc = cdev_device_add(cdev, dev); + if (rc) + goto err_add; + + return devm_add_action_or_reset(dev->parent, cxlmdev_unregister, cxlmd); + +err_add: + ida_free(&cxl_memdev_ida, cxlmd->id); +err_id: + /* + * Theoretically userspace could have already entered the fops, + * so flush ops_active. + */ + percpu_ref_kill(&cxlmd->ops_active); + wait_for_completion(&cxlmd->ops_dead); + percpu_ref_exit(&cxlmd->ops_active); +err_ref: + kfree(cxlmd); + + return rc; +} + +static int cxl_xfer_log(struct cxl_mem *cxlm, uuid_t *uuid, u32 size, u8 *out) +{ + u32 remaining = size; + u32 offset = 0; + + while (remaining) { + u32 xfer_size = min_t(u32, remaining, cxlm->payload_size); + struct cxl_mbox_get_log { + uuid_t uuid; + __le32 offset; + __le32 length; + } __packed log = { + .uuid = *uuid, + .offset = cpu_to_le32(offset), + .length = cpu_to_le32(xfer_size) + }; + int rc; + + rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LOG, &log, + sizeof(log), out, xfer_size); + if (rc < 0) + return rc; + + out += xfer_size; + remaining -= xfer_size; + offset += xfer_size; + } + + return 0; +} + +/** + * cxl_walk_cel() - Walk through the Command Effects Log. + * @cxlm: Device. + * @size: Length of the Command Effects Log. + * @cel: CEL + * + * Iterate over each entry in the CEL and determine if the driver supports the + * command. If so, the command is enabled for the device and can be used later. + */ +static void cxl_walk_cel(struct cxl_mem *cxlm, size_t size, u8 *cel) +{ + struct cel_entry { + __le16 opcode; + __le16 effect; + } __packed * cel_entry; + const int cel_entries = size / sizeof(*cel_entry); + int i; + + cel_entry = (struct cel_entry *)cel; + + for (i = 0; i < cel_entries; i++) { + u16 opcode = le16_to_cpu(cel_entry[i].opcode); + struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); + + if (!cmd) { + dev_dbg(&cxlm->pdev->dev, + "Opcode 0x%04x unsupported by driver", opcode); + continue; + } + + set_bit(cmd->info.id, cxlm->enabled_cmds); + } +} + +struct cxl_mbox_get_supported_logs { + __le16 entries; + u8 rsvd[6]; + struct gsl_entry { + uuid_t uuid; + __le32 size; + } __packed entry[]; +} __packed; + +static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_mem *cxlm) +{ + struct cxl_mbox_get_supported_logs *ret; + int rc; + + ret = kvmalloc(cxlm->payload_size, GFP_KERNEL); + if (!ret) + return ERR_PTR(-ENOMEM); + + rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_SUPPORTED_LOGS, NULL, + 0, ret, cxlm->payload_size); + if (rc < 0) { + kvfree(ret); + return ERR_PTR(rc); + } + + return ret; +} + +/** + * cxl_mem_enumerate_cmds() - Enumerate commands for a device. + * @cxlm: The device. + * + * Returns 0 if enumerate completed successfully. + * + * CXL devices have optional support for certain commands. This function will + * determine the set of supported commands for the hardware and update the + * enabled_cmds bitmap in the @cxlm. + */ +static int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm) +{ + struct cxl_mbox_get_supported_logs *gsl; + struct device *dev = &cxlm->pdev->dev; + struct cxl_mem_command *cmd; + int i, rc; + + gsl = cxl_get_gsl(cxlm); + if (IS_ERR(gsl)) + return PTR_ERR(gsl); + + rc = -ENOENT; + for (i = 0; i < le16_to_cpu(gsl->entries); i++) { + u32 size = le32_to_cpu(gsl->entry[i].size); + uuid_t uuid = gsl->entry[i].uuid; + u8 *log; + + dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size); + + if (!uuid_equal(&uuid, &log_uuid[CEL_UUID])) + continue; + + log = kvmalloc(size, GFP_KERNEL); + if (!log) { + rc = -ENOMEM; + goto out; + } + + rc = cxl_xfer_log(cxlm, &uuid, size, log); + if (rc) { + kvfree(log); + goto out; + } + + cxl_walk_cel(cxlm, size, log); + kvfree(log); + + /* In case CEL was bogus, enable some default commands. */ + cxl_for_each_cmd(cmd) + if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE) + set_bit(cmd->info.id, cxlm->enabled_cmds); + + /* Found the required CEL */ + rc = 0; + } + +out: + kvfree(gsl); + return rc; +} + +/** + * cxl_mem_identify() - Send the IDENTIFY command to the device. + * @cxlm: The device to identify. + * + * Return: 0 if identify was executed successfully. + * + * This will dispatch the identify command to the device and on success populate + * structures to be exported to sysfs. + */ +static int cxl_mem_identify(struct cxl_mem *cxlm) +{ + struct cxl_mbox_identify { + char fw_revision[0x10]; + __le64 total_capacity; + __le64 volatile_capacity; + __le64 persistent_capacity; + __le64 partition_align; + __le16 info_event_log_size; + __le16 warning_event_log_size; + __le16 failure_event_log_size; + __le16 fatal_event_log_size; + __le32 lsa_size; + u8 poison_list_max_mer[3]; + __le16 inject_poison_limit; + u8 poison_caps; + u8 qos_telemetry_caps; + } __packed id; + int rc; + + rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id, + sizeof(id)); + if (rc < 0) + return rc; + + /* + * TODO: enumerate DPA map, as 'ram' and 'pmem' do not alias. + * For now, only the capacity is exported in sysfs + */ + cxlm->ram_range.start = 0; + cxlm->ram_range.end = le64_to_cpu(id.volatile_capacity) - 1; + + cxlm->pmem_range.start = 0; + cxlm->pmem_range.end = le64_to_cpu(id.persistent_capacity) - 1; + + memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision)); + + return 0; +} + +static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + struct cxl_mem *cxlm = NULL; + u32 regloc_size, regblocks; + int rc, regloc, i; + + rc = pcim_enable_device(pdev); + if (rc) + return rc; + + regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); + if (!regloc) { + dev_err(dev, "register location dvsec not found\n"); + return -ENXIO; + } + + /* Get the size of the Register Locator DVSEC */ + pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); + regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); + + regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; + regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; + + for (i = 0; i < regblocks; i++, regloc += 8) { + u32 reg_lo, reg_hi; + u8 reg_type; + + /* "register low and high" contain other bits */ + pci_read_config_dword(pdev, regloc, ®_lo); + pci_read_config_dword(pdev, regloc + 4, ®_hi); + + reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); + + if (reg_type == CXL_REGLOC_RBI_MEMDEV) { + cxlm = cxl_mem_create(pdev, reg_lo, reg_hi); + break; + } + } + + if (!cxlm) + return -ENODEV; + + rc = cxl_mem_setup_regs(cxlm); + if (rc) + return rc; + + rc = cxl_mem_setup_mailbox(cxlm); + if (rc) + return rc; + + rc = cxl_mem_enumerate_cmds(cxlm); + if (rc) + return rc; + + rc = cxl_mem_identify(cxlm); + if (rc) + return rc; + + return cxl_mem_add_memdev(cxlm); +} + +static const struct pci_device_id cxl_mem_pci_tbl[] = { + /* PCI class code for CXL.mem Type-3 Devices */ + { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, + { /* terminate list */ }, +}; +MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); + +static struct pci_driver cxl_mem_driver = { + .name = KBUILD_MODNAME, + .id_table = cxl_mem_pci_tbl, + .probe = cxl_mem_probe, + .driver = { + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; + +static __init int cxl_mem_init(void) +{ + struct dentry *mbox_debugfs; + dev_t devt; + int rc; + + rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); + if (rc) + return rc; + + cxl_mem_major = MAJOR(devt); + + rc = pci_register_driver(&cxl_mem_driver); + if (rc) { + unregister_chrdev_region(MKDEV(cxl_mem_major, 0), + CXL_MEM_MAX_DEVS); + return rc; + } + + cxl_debugfs = debugfs_create_dir("cxl", NULL); + mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs); + debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs, + &cxl_raw_allow_all); + + return 0; +} + +static __exit void cxl_mem_exit(void) +{ + debugfs_remove_recursive(cxl_debugfs); + pci_unregister_driver(&cxl_mem_driver); + unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); +} + +MODULE_LICENSE("GPL v2"); +module_init(cxl_mem_init); +module_exit(cxl_mem_exit); +MODULE_IMPORT_NS(CXL); diff --git a/drivers/cxl/pci.h b/drivers/cxl/pci.h new file mode 100644 index 000000000000..af3ec078cf6c --- /dev/null +++ b/drivers/cxl/pci.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#ifndef __CXL_PCI_H__ +#define __CXL_PCI_H__ + +#define CXL_MEMORY_PROGIF 0x10 + +/* + * See section 8.1 Configuration Space Registers in the CXL 2.0 + * Specification + */ +#define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20) +#define PCI_DVSEC_VENDOR_ID_CXL 0x1E98 +#define PCI_DVSEC_ID_CXL 0x0 + +#define PCI_DVSEC_ID_CXL_REGLOC_OFFSET 0x8 +#define PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET 0xC + +/* BAR Indicator Register (BIR) */ +#define CXL_REGLOC_BIR_MASK GENMASK(2, 0) + +/* Register Block Identifier (RBI) */ +#define CXL_REGLOC_RBI_MASK GENMASK(15, 8) +#define CXL_REGLOC_RBI_EMPTY 0 +#define CXL_REGLOC_RBI_COMPONENT 1 +#define CXL_REGLOC_RBI_VIRT 2 +#define CXL_REGLOC_RBI_MEMDEV 3 + +#define CXL_REGLOC_ADDR_MASK GENMASK(31, 16) + +#endif /* __CXL_PCI_H__ */ diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 737b207c9e30..452e85ae87a8 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -179,7 +179,10 @@ static int dax_bus_remove(struct device *dev) struct dax_device_driver *dax_drv = to_dax_drv(dev->driver); struct dev_dax *dev_dax = to_dev_dax(dev); - return dax_drv->remove(dev_dax); + if (dax_drv->remove) + dax_drv->remove(dev_dax); + + return 0; } static struct bus_type dax_bus_type = { @@ -1038,7 +1041,7 @@ static ssize_t range_parse(const char *opt, size_t len, struct range *range) { unsigned long long addr = 0; char *start, *end, *str; - ssize_t rc = EINVAL; + ssize_t rc = -EINVAL; str = kstrdup(opt, GFP_KERNEL); if (!str) @@ -1392,6 +1395,13 @@ int __dax_driver_register(struct dax_device_driver *dax_drv, struct device_driver *drv = &dax_drv->drv; int rc = 0; + /* + * dax_bus_probe() calls dax_drv->probe() unconditionally. + * So better be safe than sorry and ensure it is provided. + */ + if (!dax_drv->probe) + return -EINVAL; + INIT_LIST_HEAD(&dax_drv->ids); drv->owner = module; drv->name = mod_name; @@ -1409,7 +1419,15 @@ int __dax_driver_register(struct dax_device_driver *dax_drv, mutex_unlock(&dax_bus_lock); if (rc) return rc; - return driver_register(drv); + + rc = driver_register(drv); + if (rc && dax_drv->match_always) { + mutex_lock(&dax_bus_lock); + match_always_count -= dax_drv->match_always; + mutex_unlock(&dax_bus_lock); + } + + return rc; } EXPORT_SYMBOL_GPL(__dax_driver_register); diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h index 72b92f95509f..1e946ad7780a 100644 --- a/drivers/dax/bus.h +++ b/drivers/dax/bus.h @@ -39,7 +39,7 @@ struct dax_device_driver { struct list_head ids; int match_always; int (*probe)(struct dev_dax *dev); - int (*remove)(struct dev_dax *dev); + void (*remove)(struct dev_dax *dev); }; int __dax_driver_register(struct dax_device_driver *dax_drv, diff --git a/drivers/dax/device.c b/drivers/dax/device.c index 5da2980bb16b..db92573c94e8 100644 --- a/drivers/dax/device.c +++ b/drivers/dax/device.c @@ -452,15 +452,9 @@ int dev_dax_probe(struct dev_dax *dev_dax) } EXPORT_SYMBOL_GPL(dev_dax_probe); -static int dev_dax_remove(struct dev_dax *dev_dax) -{ - /* all probe actions are unwound by devm */ - return 0; -} - static struct dax_device_driver device_dax_driver = { .probe = dev_dax_probe, - .remove = dev_dax_remove, + /* all probe actions are unwound by devm, so .remove isn't necessary */ .match_always = 1, }; diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 403ec42472d1..ac231cc36359 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -136,7 +136,7 @@ err_res_name: } #ifdef CONFIG_MEMORY_HOTREMOVE -static int dev_dax_kmem_remove(struct dev_dax *dev_dax) +static void dev_dax_kmem_remove(struct dev_dax *dev_dax) { int i, success = 0; struct device *dev = &dev_dax->dev; @@ -176,11 +176,9 @@ static int dev_dax_kmem_remove(struct dev_dax *dev_dax) kfree(data); dev_set_drvdata(dev, NULL); } - - return 0; } #else -static int dev_dax_kmem_remove(struct dev_dax *dev_dax) +static void dev_dax_kmem_remove(struct dev_dax *dev_dax) { /* * Without hotremove purposely leak the request_mem_region() for the @@ -190,7 +188,6 @@ static int dev_dax_kmem_remove(struct dev_dax *dev_dax) * request_mem_region(). */ any_hotremove_failed = true; - return 0; } #endif /* CONFIG_MEMORY_HOTREMOVE */ diff --git a/drivers/dax/pmem/compat.c b/drivers/dax/pmem/compat.c index 863c114fd88c..d81dc35fd65d 100644 --- a/drivers/dax/pmem/compat.c +++ b/drivers/dax/pmem/compat.c @@ -41,10 +41,9 @@ static int dax_pmem_compat_release(struct device *dev, void *data) return 0; } -static int dax_pmem_compat_remove(struct device *dev) +static void dax_pmem_compat_remove(struct device *dev) { device_for_each_child(dev, NULL, dax_pmem_compat_release); - return 0; } static struct nd_device_driver dax_pmem_compat_driver = { diff --git a/drivers/dax/super.c b/drivers/dax/super.c index cadbd0a1a1ef..5fa6ae9dbc8b 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -480,7 +480,7 @@ static void dax_free_inode(struct inode *inode) kfree(dax_dev->host); dax_dev->host = NULL; if (inode->i_rdev) - ida_simple_remove(&dax_minor_ida, MINOR(inode->i_rdev)); + ida_simple_remove(&dax_minor_ida, iminor(inode)); kmem_cache_free(dax_cache, dax_dev); } diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig index 4f8224a6ac95..4e16c71c24b7 100644 --- a/drivers/dma-buf/Kconfig +++ b/drivers/dma-buf/Kconfig @@ -50,6 +50,14 @@ config DMABUF_MOVE_NOTIFY This is marked experimental because we don't yet have a consistent execution context and memory management between drivers. +config DMABUF_DEBUG + bool "DMA-BUF debug checks" + default y if DMA_API_DEBUG + help + This option enables additional checks for DMA-BUF importers and + exporters. Specifically it validates that importers do not peek at the + underlying struct page when they import a buffer. + config DMABUF_SELFTESTS tristate "Selftests for the dma-buf interfaces" default n diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 9ad6397aaa97..f264b70c383e 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -493,7 +493,7 @@ err_alloc_file: * * 4. Once a driver is done with a shared buffer it needs to call * dma_buf_detach() (after cleaning up any mappings) and then release the - * reference acquired with dma_buf_get by calling dma_buf_put(). + * reference acquired with dma_buf_get() by calling dma_buf_put(). * * For the detailed semantics exporters are expected to implement see * &dma_buf_ops. @@ -509,9 +509,10 @@ err_alloc_file: * by the exporter. see &struct dma_buf_export_info * for further details. * - * Returns, on success, a newly created dma_buf object, which wraps the - * supplied private data and operations for dma_buf_ops. On either missing - * ops, or error in allocating struct dma_buf, will return negative error. + * Returns, on success, a newly created struct dma_buf object, which wraps the + * supplied private data and operations for struct dma_buf_ops. On either + * missing ops, or error in allocating struct dma_buf, will return negative + * error. * * For most cases the easiest way to create @exp_info is through the * %DEFINE_DMA_BUF_EXPORT_INFO macro. @@ -597,7 +598,7 @@ err_module: EXPORT_SYMBOL_GPL(dma_buf_export); /** - * dma_buf_fd - returns a file descriptor for the given dma_buf + * dma_buf_fd - returns a file descriptor for the given struct dma_buf * @dmabuf: [in] pointer to dma_buf for which fd is required. * @flags: [in] flags to give to fd * @@ -621,10 +622,10 @@ int dma_buf_fd(struct dma_buf *dmabuf, int flags) EXPORT_SYMBOL_GPL(dma_buf_fd); /** - * dma_buf_get - returns the dma_buf structure related to an fd - * @fd: [in] fd associated with the dma_buf to be returned + * dma_buf_get - returns the struct dma_buf related to an fd + * @fd: [in] fd associated with the struct dma_buf to be returned * - * On success, returns the dma_buf structure associated with an fd; uses + * On success, returns the struct dma_buf associated with an fd; uses * file's refcounting done by fget to increase refcount. returns ERR_PTR * otherwise. */ @@ -665,9 +666,36 @@ void dma_buf_put(struct dma_buf *dmabuf) } EXPORT_SYMBOL_GPL(dma_buf_put); +static void mangle_sg_table(struct sg_table *sg_table) +{ +#ifdef CONFIG_DMABUF_DEBUG + int i; + struct scatterlist *sg; + + /* To catch abuse of the underlying struct page by importers mix + * up the bits, but take care to preserve the low SG_ bits to + * not corrupt the sgt. The mixing is undone in __unmap_dma_buf + * before passing the sgt back to the exporter. */ + for_each_sgtable_sg(sg_table, sg, i) + sg->page_link ^= ~0xffUL; +#endif + +} +static struct sg_table * __map_dma_buf(struct dma_buf_attachment *attach, + enum dma_data_direction direction) +{ + struct sg_table *sg_table; + + sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction); + + if (!IS_ERR_OR_NULL(sg_table)) + mangle_sg_table(sg_table); + + return sg_table; +} + /** - * dma_buf_dynamic_attach - Add the device to dma_buf's attachments list; optionally, - * calls attach() of dma_buf_ops to allow device-specific attach functionality + * dma_buf_dynamic_attach - Add the device to dma_buf's attachments list * @dmabuf: [in] buffer to attach device to. * @dev: [in] device to be attached. * @importer_ops: [in] importer operations for the attachment @@ -676,6 +704,9 @@ EXPORT_SYMBOL_GPL(dma_buf_put); * Returns struct dma_buf_attachment pointer for this attachment. Attachments * must be cleaned up by calling dma_buf_detach(). * + * Optionally this calls &dma_buf_ops.attach to allow device-specific attach + * functionality. + * * Returns: * * A pointer to newly created &dma_buf_attachment on success, or a negative @@ -734,7 +765,7 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, goto err_unlock; } - sgt = dmabuf->ops->map_dma_buf(attach, DMA_BIDIRECTIONAL); + sgt = __map_dma_buf(attach, DMA_BIDIRECTIONAL); if (!sgt) sgt = ERR_PTR(-ENOMEM); if (IS_ERR(sgt)) { @@ -781,13 +812,24 @@ struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf, } EXPORT_SYMBOL_GPL(dma_buf_attach); +static void __unmap_dma_buf(struct dma_buf_attachment *attach, + struct sg_table *sg_table, + enum dma_data_direction direction) +{ + /* uses XOR, hence this unmangles */ + mangle_sg_table(sg_table); + + attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction); +} + /** - * dma_buf_detach - Remove the given attachment from dmabuf's attachments list; - * optionally calls detach() of dma_buf_ops for device-specific detach + * dma_buf_detach - Remove the given attachment from dmabuf's attachments list * @dmabuf: [in] buffer to detach from. * @attach: [in] attachment to be detached; is free'd after this call. * * Clean up a device attachment obtained by calling dma_buf_attach(). + * + * Optionally this calls &dma_buf_ops.detach for device-specific detach. */ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) { @@ -798,7 +840,7 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) if (dma_buf_is_dynamic(attach->dmabuf)) dma_resv_lock(attach->dmabuf->resv, NULL); - dmabuf->ops->unmap_dma_buf(attach, attach->sgt, attach->dir); + __unmap_dma_buf(attach, attach->sgt, attach->dir); if (dma_buf_is_dynamic(attach->dmabuf)) { dma_buf_unpin(attach); @@ -818,9 +860,15 @@ EXPORT_SYMBOL_GPL(dma_buf_detach); /** * dma_buf_pin - Lock down the DMA-buf - * * @attach: [in] attachment which should be pinned * + * Only dynamic importers (who set up @attach with dma_buf_dynamic_attach()) may + * call this, and only for limited use cases like scanout and not for temporary + * pin operations. It is not permitted to allow userspace to pin arbitrary + * amounts of buffers through this interface. + * + * Buffers must be unpinned by calling dma_buf_unpin(). + * * Returns: * 0 on success, negative error code on failure. */ @@ -829,6 +877,8 @@ int dma_buf_pin(struct dma_buf_attachment *attach) struct dma_buf *dmabuf = attach->dmabuf; int ret = 0; + WARN_ON(!dma_buf_attachment_is_dynamic(attach)); + dma_resv_assert_held(dmabuf->resv); if (dmabuf->ops->pin) @@ -839,14 +889,19 @@ int dma_buf_pin(struct dma_buf_attachment *attach) EXPORT_SYMBOL_GPL(dma_buf_pin); /** - * dma_buf_unpin - Remove lock from DMA-buf - * + * dma_buf_unpin - Unpin a DMA-buf * @attach: [in] attachment which should be unpinned + * + * This unpins a buffer pinned by dma_buf_pin() and allows the exporter to move + * any mapping of @attach again and inform the importer through + * &dma_buf_attach_ops.move_notify. */ void dma_buf_unpin(struct dma_buf_attachment *attach) { struct dma_buf *dmabuf = attach->dmabuf; + WARN_ON(!dma_buf_attachment_is_dynamic(attach)); + dma_resv_assert_held(dmabuf->resv); if (dmabuf->ops->unpin) @@ -907,7 +962,7 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach, } } - sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction); + sg_table = __map_dma_buf(attach, direction); if (!sg_table) sg_table = ERR_PTR(-ENOMEM); @@ -970,7 +1025,7 @@ void dma_buf_unmap_attachment(struct dma_buf_attachment *attach, if (dma_buf_is_dynamic(attach->dmabuf)) dma_resv_assert_held(attach->dmabuf->resv); - attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction); + __unmap_dma_buf(attach, sg_table, direction); if (dma_buf_is_dynamic(attach->dmabuf) && !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) @@ -1014,15 +1069,15 @@ EXPORT_SYMBOL_GPL(dma_buf_move_notify); * vmalloc space might be limited and result in vmap calls failing. * * Interfaces:: + * * void \*dma_buf_vmap(struct dma_buf \*dmabuf) * void dma_buf_vunmap(struct dma_buf \*dmabuf, void \*vaddr) * * The vmap call can fail if there is no vmap support in the exporter, or if - * it runs out of vmalloc space. Fallback to kmap should be implemented. Note - * that the dma-buf layer keeps a reference count for all vmap access and - * calls down into the exporter's vmap function only when no vmapping exists, - * and only unmaps it once. Protection against concurrent vmap/vunmap calls is - * provided by taking the dma_buf->lock mutex. + * it runs out of vmalloc space. Note that the dma-buf layer keeps a reference + * count for all vmap access and calls down into the exporter's vmap function + * only when no vmapping exists, and only unmaps it once. Protection against + * concurrent vmap/vunmap calls is provided by taking the &dma_buf.lock mutex. * * - For full compatibility on the importer side with existing userspace * interfaces, which might already support mmap'ing buffers. This is needed in @@ -1074,11 +1129,12 @@ EXPORT_SYMBOL_GPL(dma_buf_move_notify); * shootdowns would increase the complexity quite a bit. * * Interface:: + * * int dma_buf_mmap(struct dma_buf \*, struct vm_area_struct \*, * unsigned long); * * If the importing subsystem simply provides a special-purpose mmap call to - * set up a mapping in userspace, calling do_mmap with dma_buf->file will + * set up a mapping in userspace, calling do_mmap with &dma_buf.file will * equally achieve that for a dma-buf object. */ @@ -1111,6 +1167,11 @@ static int __dma_buf_begin_cpu_access(struct dma_buf *dmabuf, * dma_buf_end_cpu_access(). Only when cpu access is braketed by both calls is * it guaranteed to be coherent with other DMA access. * + * This function will also wait for any DMA transactions tracked through + * implicit synchronization in &dma_buf.resv. For DMA transactions with explicit + * synchronization this function will only ensure cache coherency, callers must + * ensure synchronization with such DMA transactions on their own. + * * Can return negative error values, returns 0 on success. */ int dma_buf_begin_cpu_access(struct dma_buf *dmabuf, @@ -1121,6 +1182,8 @@ int dma_buf_begin_cpu_access(struct dma_buf *dmabuf, if (WARN_ON(!dmabuf)) return -EINVAL; + might_lock(&dmabuf->resv->lock.base); + if (dmabuf->ops->begin_cpu_access) ret = dmabuf->ops->begin_cpu_access(dmabuf, direction); @@ -1154,6 +1217,8 @@ int dma_buf_end_cpu_access(struct dma_buf *dmabuf, WARN_ON(!dmabuf); + might_lock(&dmabuf->resv->lock.base); + if (dmabuf->ops->end_cpu_access) ret = dmabuf->ops->end_cpu_access(dmabuf, direction); @@ -1212,7 +1277,10 @@ EXPORT_SYMBOL_GPL(dma_buf_mmap); * This call may fail due to lack of virtual mapping address space. * These calls are optional in drivers. The intended use for them * is for mapping objects linear in kernel space for high use objects. - * Please attempt to use kmap/kunmap before thinking about these interfaces. + * + * To ensure coherency users must call dma_buf_begin_cpu_access() and + * dma_buf_end_cpu_access() around any cpu access performed through this + * mapping. * * Returns 0 on success, or a negative errno code otherwise. */ diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 7475e09b0680..d64fc03929be 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -311,6 +311,83 @@ void __dma_fence_might_wait(void) #endif +/** + * dma_fence_signal_timestamp_locked - signal completion of a fence + * @fence: the fence to signal + * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain + * + * Signal completion for software callbacks on a fence, this will unblock + * dma_fence_wait() calls and run all the callbacks added with + * dma_fence_add_callback(). Can be called multiple times, but since a fence + * can only go from the unsignaled to the signaled state and not back, it will + * only be effective the first time. Set the timestamp provided as the fence + * signal timestamp. + * + * Unlike dma_fence_signal_timestamp(), this function must be called with + * &dma_fence.lock held. + * + * Returns 0 on success and a negative error value when @fence has been + * signalled already. + */ +int dma_fence_signal_timestamp_locked(struct dma_fence *fence, + ktime_t timestamp) +{ + struct dma_fence_cb *cur, *tmp; + struct list_head cb_list; + + lockdep_assert_held(fence->lock); + + if (unlikely(test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, + &fence->flags))) + return -EINVAL; + + /* Stash the cb_list before replacing it with the timestamp */ + list_replace(&fence->cb_list, &cb_list); + + fence->timestamp = timestamp; + set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); + trace_dma_fence_signaled(fence); + + list_for_each_entry_safe(cur, tmp, &cb_list, node) { + INIT_LIST_HEAD(&cur->node); + cur->func(fence, cur); + } + + return 0; +} +EXPORT_SYMBOL(dma_fence_signal_timestamp_locked); + +/** + * dma_fence_signal_timestamp - signal completion of a fence + * @fence: the fence to signal + * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain + * + * Signal completion for software callbacks on a fence, this will unblock + * dma_fence_wait() calls and run all the callbacks added with + * dma_fence_add_callback(). Can be called multiple times, but since a fence + * can only go from the unsignaled to the signaled state and not back, it will + * only be effective the first time. Set the timestamp provided as the fence + * signal timestamp. + * + * Returns 0 on success and a negative error value when @fence has been + * signalled already. + */ +int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp) +{ + unsigned long flags; + int ret; + + if (!fence) + return -EINVAL; + + spin_lock_irqsave(fence->lock, flags); + ret = dma_fence_signal_timestamp_locked(fence, timestamp); + spin_unlock_irqrestore(fence->lock, flags); + + return ret; +} +EXPORT_SYMBOL(dma_fence_signal_timestamp); + /** * dma_fence_signal_locked - signal completion of a fence * @fence: the fence to signal @@ -329,28 +406,7 @@ void __dma_fence_might_wait(void) */ int dma_fence_signal_locked(struct dma_fence *fence) { - struct dma_fence_cb *cur, *tmp; - struct list_head cb_list; - - lockdep_assert_held(fence->lock); - - if (unlikely(test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, - &fence->flags))) - return -EINVAL; - - /* Stash the cb_list before replacing it with the timestamp */ - list_replace(&fence->cb_list, &cb_list); - - fence->timestamp = ktime_get(); - set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); - trace_dma_fence_signaled(fence); - - list_for_each_entry_safe(cur, tmp, &cb_list, node) { - INIT_LIST_HEAD(&cur->node); - cur->func(fence, cur); - } - - return 0; + return dma_fence_signal_timestamp_locked(fence, ktime_get()); } EXPORT_SYMBOL(dma_fence_signal_locked); @@ -379,7 +435,7 @@ int dma_fence_signal(struct dma_fence *fence) tmp = dma_fence_begin_signalling(); spin_lock_irqsave(fence->lock, flags); - ret = dma_fence_signal_locked(fence); + ret = dma_fence_signal_timestamp_locked(fence, ktime_get()); spin_unlock_irqrestore(fence->lock, flags); dma_fence_end_signalling(tmp); diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c index afd22c9dbdcf..6b5db954569f 100644 --- a/drivers/dma-buf/dma-heap.c +++ b/drivers/dma-buf/dma-heap.c @@ -52,6 +52,9 @@ static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, unsigned int fd_flags, unsigned int heap_flags) { + struct dma_buf *dmabuf; + int fd; + /* * Allocations from all heaps have to begin * and end on page boundaries. @@ -60,7 +63,16 @@ static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, if (!len) return -EINVAL; - return heap->ops->allocate(heap, len, fd_flags, heap_flags); + dmabuf = heap->ops->allocate(heap, len, fd_flags, heap_flags); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + fd = dma_buf_fd(dmabuf, fd_flags); + if (fd < 0) { + dma_buf_put(dmabuf); + /* just return, as put will call release and that will free */ + } + return fd; } static int dma_heap_open(struct inode *inode, struct file *file) diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index 364fc2f3e499..5d64eccd21d6 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -271,10 +271,10 @@ static const struct dma_buf_ops cma_heap_buf_ops = { .release = cma_heap_dma_buf_release, }; -static int cma_heap_allocate(struct dma_heap *heap, - unsigned long len, - unsigned long fd_flags, - unsigned long heap_flags) +static struct dma_buf *cma_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) { struct cma_heap *cma_heap = dma_heap_get_drvdata(heap); struct cma_heap_buffer *buffer; @@ -289,7 +289,7 @@ static int cma_heap_allocate(struct dma_heap *heap, buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); if (!buffer) - return -ENOMEM; + return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&buffer->attachments); mutex_init(&buffer->lock); @@ -348,15 +348,7 @@ static int cma_heap_allocate(struct dma_heap *heap, ret = PTR_ERR(dmabuf); goto free_pages; } - - ret = dma_buf_fd(dmabuf, fd_flags); - if (ret < 0) { - dma_buf_put(dmabuf); - /* just return, as put will call release and that will free */ - return ret; - } - - return ret; + return dmabuf; free_pages: kfree(buffer->pages); @@ -365,7 +357,7 @@ free_cma: free_buffer: kfree(buffer); - return ret; + return ERR_PTR(ret); } static const struct dma_heap_ops cma_heap_ops = { diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index 17e0e9a68baf..29e49ac17251 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -331,10 +331,10 @@ static struct page *alloc_largest_available(unsigned long size, return NULL; } -static int system_heap_allocate(struct dma_heap *heap, - unsigned long len, - unsigned long fd_flags, - unsigned long heap_flags) +static struct dma_buf *system_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) { struct system_heap_buffer *buffer; DEFINE_DMA_BUF_EXPORT_INFO(exp_info); @@ -349,7 +349,7 @@ static int system_heap_allocate(struct dma_heap *heap, buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); if (!buffer) - return -ENOMEM; + return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&buffer->attachments); mutex_init(&buffer->lock); @@ -363,8 +363,10 @@ static int system_heap_allocate(struct dma_heap *heap, * Avoid trying to allocate memory if the process * has been killed by SIGKILL */ - if (fatal_signal_pending(current)) + if (fatal_signal_pending(current)) { + ret = -EINTR; goto free_buffer; + } page = alloc_largest_available(size_remaining, max_order); if (!page) @@ -397,14 +399,7 @@ static int system_heap_allocate(struct dma_heap *heap, ret = PTR_ERR(dmabuf); goto free_pages; } - - ret = dma_buf_fd(dmabuf, fd_flags); - if (ret < 0) { - dma_buf_put(dmabuf); - /* just return, as put will call release and that will free */ - return ret; - } - return ret; + return dmabuf; free_pages: for_each_sgtable_sg(table, sg, i) { @@ -418,7 +413,7 @@ free_buffer: __free_pages(page, compound_order(page)); kfree(buffer); - return ret; + return ERR_PTR(ret); } static const struct dma_heap_ops system_heap_ops = { diff --git a/drivers/dma-buf/st-dma-fence.c b/drivers/dma-buf/st-dma-fence.c index e593064341c8..c8a12d7ad71a 100644 --- a/drivers/dma-buf/st-dma-fence.c +++ b/drivers/dma-buf/st-dma-fence.c @@ -471,8 +471,11 @@ static int thread_signal_callback(void *arg) dma_fence_signal(f1); smp_store_mb(cb.seen, false); - if (!f2 || dma_fence_add_callback(f2, &cb.cb, simple_callback)) - miss++, cb.seen = true; + if (!f2 || + dma_fence_add_callback(f2, &cb.cb, simple_callback)) { + miss++; + cb.seen = true; + } if (!t->before) dma_fence_signal(f1); diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index d242c7632621..0c2827fd8c19 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -124,13 +124,6 @@ config BCM_SBA_RAID has the capability to offload memcpy, xor and pq computation for raid5/6. -config COH901318 - bool "ST-Ericsson COH901318 DMA support" - select DMA_ENGINE - depends on ARCH_U300 || COMPILE_TEST - help - Enable support for ST-Ericsson COH 901 318 DMA. - config DMA_BCM2835 tristate "BCM2835 DMA engine support" depends on ARCH_BCM2835 @@ -179,6 +172,7 @@ config DMA_SUN6I config DW_AXI_DMAC tristate "Synopsys DesignWare AXI DMA support" depends on OF || COMPILE_TEST + depends on HAS_IOMEM select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -378,14 +372,14 @@ config MILBEAUT_XDMAC XDMAC device. config MMP_PDMA - bool "MMP PDMA support" + tristate "MMP PDMA support" depends on ARCH_MMP || ARCH_PXA || COMPILE_TEST select DMA_ENGINE help Support the MMP PDMA engine for PXA and MMP platform. config MMP_TDMA - bool "MMP Two-Channel DMA support" + tristate "MMP Two-Channel DMA support" depends on ARCH_MMP || COMPILE_TEST select DMA_ENGINE select GENERIC_ALLOCATOR @@ -519,13 +513,6 @@ config PLX_DMA These are exposed via extra functions on the switch's upstream port. Each function exposes one DMA channel. -config SIRF_DMA - tristate "CSR SiRFprimaII/SiRFmarco DMA support" - depends on ARCH_SIRF - select DMA_ENGINE - help - Enable support for the CSR SiRFprimaII DMA engine. - config STE_DMA40 bool "ST-Ericsson DMA40 support" depends on ARCH_U8500 @@ -710,15 +697,6 @@ config XILINX_ZYNQMP_DPDMA driver provides the dmaengine required by the DisplayPort subsystem display driver. -config ZX_DMA - tristate "ZTE ZX DMA support" - depends on ARCH_ZX || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support the DMA engine for ZTE ZX family platform devices. - - # driver files source "drivers/dma/bestcomm/Kconfig" @@ -740,6 +718,8 @@ source "drivers/dma/ti/Kconfig" source "drivers/dma/fsl-dpaa2-qdma/Kconfig" +source "drivers/dma/lgm/Kconfig" + # clients comment "DMA Clients" depends on DMA_ENGINE diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 948a8da05f8b..aa69094e3547 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -20,7 +20,6 @@ obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o obj-$(CONFIG_AXI_DMAC) += dma-axi-dmac.o obj-$(CONFIG_BCM_SBA_RAID) += bcm-sba-raid.o -obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o obj-$(CONFIG_DMA_BCM2835) += bcm2835-dma.o obj-$(CONFIG_DMA_JZ4780) += dma-jz4780.o obj-$(CONFIG_DMA_SA11X0) += sa11x0-dma.o @@ -65,7 +64,6 @@ obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/ obj-$(CONFIG_PXA_DMA) += pxa_dma.o obj-$(CONFIG_RENESAS_DMA) += sh/ obj-$(CONFIG_SF_PDMA) += sf-pdma/ -obj-$(CONFIG_SIRF_DMA) += sirf-dma.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o obj-$(CONFIG_STM32_DMA) += stm32-dma.o obj-$(CONFIG_STM32_DMAMUX) += stm32-dmamux.o @@ -79,9 +77,9 @@ obj-$(CONFIG_TIMB_DMA) += timb_dma.o obj-$(CONFIG_UNIPHIER_MDMAC) += uniphier-mdmac.o obj-$(CONFIG_UNIPHIER_XDMAC) += uniphier-xdmac.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o -obj-$(CONFIG_ZX_DMA) += zx_dma.o obj-$(CONFIG_ST_FDMA) += st_fdma.o obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/ +obj-$(CONFIG_INTEL_LDMA) += lgm/ obj-y += mediatek/ obj-y += qcom/ diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 7eaee5b705b1..30ae36124b1d 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -54,6 +54,25 @@ module_param(init_nr_desc_per_channel, uint, 0644); MODULE_PARM_DESC(init_nr_desc_per_channel, "initial descriptors per channel (default: 64)"); +/** + * struct at_dma_platform_data - Controller configuration parameters + * @nr_channels: Number of channels supported by hardware (max 8) + * @cap_mask: dma_capability flags supported by the platform + */ +struct at_dma_platform_data { + unsigned int nr_channels; + dma_cap_mask_t cap_mask; +}; + +/** + * struct at_dma_slave - Controller-specific information about a slave + * @dma_dev: required DMA master device + * @cfg: Platform-specific initializer for the CFG register + */ +struct at_dma_slave { + struct device *dma_dev; + u32 cfg; +}; /* prototypes */ static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx); diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h index 80fc2fe8c77e..4d1ebc040031 100644 --- a/drivers/dma/at_hdmac_regs.h +++ b/drivers/dma/at_hdmac_regs.h @@ -7,8 +7,6 @@ #ifndef AT_HDMAC_REGS_H #define AT_HDMAC_REGS_H -#include - #define AT_DMA_MAX_NR_CHANNELS 8 @@ -148,7 +146,31 @@ #define ATC_AUTO (0x1 << 31) /* Auto multiple buffer tx enable */ /* Bitfields in CFG */ -/* are in at_hdmac.h */ +#define ATC_PER_MSB(h) ((0x30U & (h)) >> 4) /* Extract most significant bits of a handshaking identifier */ + +#define ATC_SRC_PER(h) (0xFU & (h)) /* Channel src rq associated with periph handshaking ifc h */ +#define ATC_DST_PER(h) ((0xFU & (h)) << 4) /* Channel dst rq associated with periph handshaking ifc h */ +#define ATC_SRC_REP (0x1 << 8) /* Source Replay Mod */ +#define ATC_SRC_H2SEL (0x1 << 9) /* Source Handshaking Mod */ +#define ATC_SRC_H2SEL_SW (0x0 << 9) +#define ATC_SRC_H2SEL_HW (0x1 << 9) +#define ATC_SRC_PER_MSB(h) (ATC_PER_MSB(h) << 10) /* Channel src rq (most significant bits) */ +#define ATC_DST_REP (0x1 << 12) /* Destination Replay Mod */ +#define ATC_DST_H2SEL (0x1 << 13) /* Destination Handshaking Mod */ +#define ATC_DST_H2SEL_SW (0x0 << 13) +#define ATC_DST_H2SEL_HW (0x1 << 13) +#define ATC_DST_PER_MSB(h) (ATC_PER_MSB(h) << 14) /* Channel dst rq (most significant bits) */ +#define ATC_SOD (0x1 << 16) /* Stop On Done */ +#define ATC_LOCK_IF (0x1 << 20) /* Interface Lock */ +#define ATC_LOCK_B (0x1 << 21) /* AHB Bus Lock */ +#define ATC_LOCK_IF_L (0x1 << 22) /* Master Interface Arbiter Lock */ +#define ATC_LOCK_IF_L_CHUNK (0x0 << 22) +#define ATC_LOCK_IF_L_BUFFER (0x1 << 22) +#define ATC_AHB_PROT_MASK (0x7 << 24) /* AHB Protection */ +#define ATC_FIFOCFG_MASK (0x3 << 28) /* FIFO Request Configuration */ +#define ATC_FIFOCFG_LARGESTBURST (0x0 << 28) +#define ATC_FIFOCFG_HALFFIFO (0x1 << 28) +#define ATC_FIFOCFG_ENOUGHSPACE (0x2 << 28) /* Bitfields in SPIP */ #define ATC_SPIP_HOLE(x) (0xFFFFU & (x)) diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c deleted file mode 100644 index 95b9b2f5358e..000000000000 --- a/drivers/dma/coh901318.c +++ /dev/null @@ -1,2808 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * driver/dma/coh901318.c - * - * Copyright (C) 2007-2009 ST-Ericsson - * DMA driver for COH 901 318 - * Author: Per Friden - */ - -#include -#include -#include /* printk() */ -#include /* everything... */ -#include -#include /* kmalloc() */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "coh901318.h" -#include "dmaengine.h" - -#define COH901318_MOD32_MASK (0x1F) -#define COH901318_WORD_MASK (0xFFFFFFFF) -/* INT_STATUS - Interrupt Status Registers 32bit (R/-) */ -#define COH901318_INT_STATUS1 (0x0000) -#define COH901318_INT_STATUS2 (0x0004) -/* TC_INT_STATUS - Terminal Count Interrupt Status Registers 32bit (R/-) */ -#define COH901318_TC_INT_STATUS1 (0x0008) -#define COH901318_TC_INT_STATUS2 (0x000C) -/* TC_INT_CLEAR - Terminal Count Interrupt Clear Registers 32bit (-/W) */ -#define COH901318_TC_INT_CLEAR1 (0x0010) -#define COH901318_TC_INT_CLEAR2 (0x0014) -/* RAW_TC_INT_STATUS - Raw Term Count Interrupt Status Registers 32bit (R/-) */ -#define COH901318_RAW_TC_INT_STATUS1 (0x0018) -#define COH901318_RAW_TC_INT_STATUS2 (0x001C) -/* BE_INT_STATUS - Bus Error Interrupt Status Registers 32bit (R/-) */ -#define COH901318_BE_INT_STATUS1 (0x0020) -#define COH901318_BE_INT_STATUS2 (0x0024) -/* BE_INT_CLEAR - Bus Error Interrupt Clear Registers 32bit (-/W) */ -#define COH901318_BE_INT_CLEAR1 (0x0028) -#define COH901318_BE_INT_CLEAR2 (0x002C) -/* RAW_BE_INT_STATUS - Raw Term Count Interrupt Status Registers 32bit (R/-) */ -#define COH901318_RAW_BE_INT_STATUS1 (0x0030) -#define COH901318_RAW_BE_INT_STATUS2 (0x0034) - -/* - * CX_CFG - Channel Configuration Registers 32bit (R/W) - */ -#define COH901318_CX_CFG (0x0100) -#define COH901318_CX_CFG_SPACING (0x04) -/* Channel enable activates tha dma job */ -#define COH901318_CX_CFG_CH_ENABLE (0x00000001) -#define COH901318_CX_CFG_CH_DISABLE (0x00000000) -/* Request Mode */ -#define COH901318_CX_CFG_RM_MASK (0x00000006) -#define COH901318_CX_CFG_RM_MEMORY_TO_MEMORY (0x0 << 1) -#define COH901318_CX_CFG_RM_PRIMARY_TO_MEMORY (0x1 << 1) -#define COH901318_CX_CFG_RM_MEMORY_TO_PRIMARY (0x1 << 1) -#define COH901318_CX_CFG_RM_PRIMARY_TO_SECONDARY (0x3 << 1) -#define COH901318_CX_CFG_RM_SECONDARY_TO_PRIMARY (0x3 << 1) -/* Linked channel request field. RM must == 11 */ -#define COH901318_CX_CFG_LCRF_SHIFT 3 -#define COH901318_CX_CFG_LCRF_MASK (0x000001F8) -#define COH901318_CX_CFG_LCR_DISABLE (0x00000000) -/* Terminal Counter Interrupt Request Mask */ -#define COH901318_CX_CFG_TC_IRQ_ENABLE (0x00000200) -#define COH901318_CX_CFG_TC_IRQ_DISABLE (0x00000000) -/* Bus Error interrupt Mask */ -#define COH901318_CX_CFG_BE_IRQ_ENABLE (0x00000400) -#define COH901318_CX_CFG_BE_IRQ_DISABLE (0x00000000) - -/* - * CX_STAT - Channel Status Registers 32bit (R/-) - */ -#define COH901318_CX_STAT (0x0200) -#define COH901318_CX_STAT_SPACING (0x04) -#define COH901318_CX_STAT_RBE_IRQ_IND (0x00000008) -#define COH901318_CX_STAT_RTC_IRQ_IND (0x00000004) -#define COH901318_CX_STAT_ACTIVE (0x00000002) -#define COH901318_CX_STAT_ENABLED (0x00000001) - -/* - * CX_CTRL - Channel Control Registers 32bit (R/W) - */ -#define COH901318_CX_CTRL (0x0400) -#define COH901318_CX_CTRL_SPACING (0x10) -/* Transfer Count Enable */ -#define COH901318_CX_CTRL_TC_ENABLE (0x00001000) -#define COH901318_CX_CTRL_TC_DISABLE (0x00000000) -/* Transfer Count Value 0 - 4095 */ -#define COH901318_CX_CTRL_TC_VALUE_MASK (0x00000FFF) -/* Burst count */ -#define COH901318_CX_CTRL_BURST_COUNT_MASK (0x0000E000) -#define COH901318_CX_CTRL_BURST_COUNT_64_BYTES (0x7 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_48_BYTES (0x6 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_32_BYTES (0x5 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_16_BYTES (0x4 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_8_BYTES (0x3 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_4_BYTES (0x2 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_2_BYTES (0x1 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_1_BYTE (0x0 << 13) -/* Source bus size */ -#define COH901318_CX_CTRL_SRC_BUS_SIZE_MASK (0x00030000) -#define COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS (0x2 << 16) -#define COH901318_CX_CTRL_SRC_BUS_SIZE_16_BITS (0x1 << 16) -#define COH901318_CX_CTRL_SRC_BUS_SIZE_8_BITS (0x0 << 16) -/* Source address increment */ -#define COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE (0x00040000) -#define COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE (0x00000000) -/* Destination Bus Size */ -#define COH901318_CX_CTRL_DST_BUS_SIZE_MASK (0x00180000) -#define COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS (0x2 << 19) -#define COH901318_CX_CTRL_DST_BUS_SIZE_16_BITS (0x1 << 19) -#define COH901318_CX_CTRL_DST_BUS_SIZE_8_BITS (0x0 << 19) -/* Destination address increment */ -#define COH901318_CX_CTRL_DST_ADDR_INC_ENABLE (0x00200000) -#define COH901318_CX_CTRL_DST_ADDR_INC_DISABLE (0x00000000) -/* Master Mode (Master2 is only connected to MSL) */ -#define COH901318_CX_CTRL_MASTER_MODE_MASK (0x00C00000) -#define COH901318_CX_CTRL_MASTER_MODE_M2R_M1W (0x3 << 22) -#define COH901318_CX_CTRL_MASTER_MODE_M1R_M2W (0x2 << 22) -#define COH901318_CX_CTRL_MASTER_MODE_M2RW (0x1 << 22) -#define COH901318_CX_CTRL_MASTER_MODE_M1RW (0x0 << 22) -/* Terminal Count flag to PER enable */ -#define COH901318_CX_CTRL_TCP_ENABLE (0x01000000) -#define COH901318_CX_CTRL_TCP_DISABLE (0x00000000) -/* Terminal Count flags to CPU enable */ -#define COH901318_CX_CTRL_TC_IRQ_ENABLE (0x02000000) -#define COH901318_CX_CTRL_TC_IRQ_DISABLE (0x00000000) -/* Hand shake to peripheral */ -#define COH901318_CX_CTRL_HSP_ENABLE (0x04000000) -#define COH901318_CX_CTRL_HSP_DISABLE (0x00000000) -#define COH901318_CX_CTRL_HSS_ENABLE (0x08000000) -#define COH901318_CX_CTRL_HSS_DISABLE (0x00000000) -/* DMA mode */ -#define COH901318_CX_CTRL_DDMA_MASK (0x30000000) -#define COH901318_CX_CTRL_DDMA_LEGACY (0x0 << 28) -#define COH901318_CX_CTRL_DDMA_DEMAND_DMA1 (0x1 << 28) -#define COH901318_CX_CTRL_DDMA_DEMAND_DMA2 (0x2 << 28) -/* Primary Request Data Destination */ -#define COH901318_CX_CTRL_PRDD_MASK (0x40000000) -#define COH901318_CX_CTRL_PRDD_DEST (0x1 << 30) -#define COH901318_CX_CTRL_PRDD_SOURCE (0x0 << 30) - -/* - * CX_SRC_ADDR - Channel Source Address Registers 32bit (R/W) - */ -#define COH901318_CX_SRC_ADDR (0x0404) -#define COH901318_CX_SRC_ADDR_SPACING (0x10) - -/* - * CX_DST_ADDR - Channel Destination Address Registers 32bit R/W - */ -#define COH901318_CX_DST_ADDR (0x0408) -#define COH901318_CX_DST_ADDR_SPACING (0x10) - -/* - * CX_LNK_ADDR - Channel Link Address Registers 32bit (R/W) - */ -#define COH901318_CX_LNK_ADDR (0x040C) -#define COH901318_CX_LNK_ADDR_SPACING (0x10) -#define COH901318_CX_LNK_LINK_IMMEDIATE (0x00000001) - -/** - * struct coh901318_params - parameters for DMAC configuration - * @config: DMA config register - * @ctrl_lli_last: DMA control register for the last lli in the list - * @ctrl_lli: DMA control register for an lli - * @ctrl_lli_chained: DMA control register for a chained lli - */ -struct coh901318_params { - u32 config; - u32 ctrl_lli_last; - u32 ctrl_lli; - u32 ctrl_lli_chained; -}; - -/** - * struct coh_dma_channel - dma channel base - * @name: ascii name of dma channel - * @number: channel id number - * @desc_nbr_max: number of preallocated descriptors - * @priority_high: prio of channel, 0 low otherwise high. - * @param: configuration parameters - */ -struct coh_dma_channel { - const char name[32]; - const int number; - const int desc_nbr_max; - const int priority_high; - const struct coh901318_params param; -}; - -/** - * struct powersave - DMA power save structure - * @lock: lock protecting data in this struct - * @started_channels: bit mask indicating active dma channels - */ -struct powersave { - spinlock_t lock; - u64 started_channels; -}; - -/* points out all dma slave channels. - * Syntax is [A1, B1, A2, B2, .... ,-1,-1] - * Select all channels from A to B, end of list is marked with -1,-1 - */ -static int dma_slave_channels[] = { - U300_DMA_MSL_TX_0, U300_DMA_SPI_RX, - U300_DMA_UART1_TX, U300_DMA_UART1_RX, -1, -1}; - -/* points out all dma memcpy channels. */ -static int dma_memcpy_channels[] = { - U300_DMA_GENERAL_PURPOSE_0, U300_DMA_GENERAL_PURPOSE_8, -1, -1}; - -#define flags_memcpy_config (COH901318_CX_CFG_CH_DISABLE | \ - COH901318_CX_CFG_RM_MEMORY_TO_MEMORY | \ - COH901318_CX_CFG_LCR_DISABLE | \ - COH901318_CX_CFG_TC_IRQ_ENABLE | \ - COH901318_CX_CFG_BE_IRQ_ENABLE) -#define flags_memcpy_lli_chained (COH901318_CX_CTRL_TC_ENABLE | \ - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | \ - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_MASTER_MODE_M1RW | \ - COH901318_CX_CTRL_TCP_DISABLE | \ - COH901318_CX_CTRL_TC_IRQ_DISABLE | \ - COH901318_CX_CTRL_HSP_DISABLE | \ - COH901318_CX_CTRL_HSS_DISABLE | \ - COH901318_CX_CTRL_DDMA_LEGACY | \ - COH901318_CX_CTRL_PRDD_SOURCE) -#define flags_memcpy_lli (COH901318_CX_CTRL_TC_ENABLE | \ - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | \ - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_MASTER_MODE_M1RW | \ - COH901318_CX_CTRL_TCP_DISABLE | \ - COH901318_CX_CTRL_TC_IRQ_DISABLE | \ - COH901318_CX_CTRL_HSP_DISABLE | \ - COH901318_CX_CTRL_HSS_DISABLE | \ - COH901318_CX_CTRL_DDMA_LEGACY | \ - COH901318_CX_CTRL_PRDD_SOURCE) -#define flags_memcpy_lli_last (COH901318_CX_CTRL_TC_ENABLE | \ - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | \ - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_MASTER_MODE_M1RW | \ - COH901318_CX_CTRL_TCP_DISABLE | \ - COH901318_CX_CTRL_TC_IRQ_ENABLE | \ - COH901318_CX_CTRL_HSP_DISABLE | \ - COH901318_CX_CTRL_HSS_DISABLE | \ - COH901318_CX_CTRL_DDMA_LEGACY | \ - COH901318_CX_CTRL_PRDD_SOURCE) - -static const struct coh_dma_channel chan_config[U300_DMA_CHANNELS] = { - { - .number = U300_DMA_MSL_TX_0, - .name = "MSL TX 0", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_TX_1, - .name = "MSL TX 1", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_MSL_TX_2, - .name = "MSL TX 2", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .desc_nbr_max = 10, - }, - { - .number = U300_DMA_MSL_TX_3, - .name = "MSL TX 3", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_MSL_TX_4, - .name = "MSL TX 4", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_MSL_TX_5, - .name = "MSL TX 5", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_TX_6, - .name = "MSL TX 6", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_RX_0, - .name = "MSL RX 0", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_RX_1, - .name = "MSL RX 1", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_2, - .name = "MSL RX 2", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_3, - .name = "MSL RX 3", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_4, - .name = "MSL RX 4", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_5, - .name = "MSL RX 5", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_6, - .name = "MSL RX 6", - .priority_high = 0, - }, - /* - * Don't set up device address, burst count or size of src - * or dst bus for this peripheral - handled by PrimeCell - * DMA extension. - */ - { - .number = U300_DMA_MMCSD_RX_TX, - .name = "MMCSD RX TX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - - }, - { - .number = U300_DMA_MSPRO_TX, - .name = "MSPRO TX", - .priority_high = 0, - }, - { - .number = U300_DMA_MSPRO_RX, - .name = "MSPRO RX", - .priority_high = 0, - }, - /* - * Don't set up device address, burst count or size of src - * or dst bus for this peripheral - handled by PrimeCell - * DMA extension. - */ - { - .number = U300_DMA_UART0_TX, - .name = "UART0 TX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - }, - { - .number = U300_DMA_UART0_RX, - .name = "UART0 RX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - }, - { - .number = U300_DMA_APEX_TX, - .name = "APEX TX", - .priority_high = 0, - }, - { - .number = U300_DMA_APEX_RX, - .name = "APEX RX", - .priority_high = 0, - }, - { - .number = U300_DMA_PCM_I2S0_TX, - .name = "PCM I2S0 TX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_PCM_I2S0_RX, - .name = "PCM I2S0 RX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_PCM_I2S1_TX, - .name = "PCM I2S1 TX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_PCM_I2S1_RX, - .name = "PCM I2S1 RX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_XGAM_CDI, - .name = "XGAM CDI", - .priority_high = 0, - }, - { - .number = U300_DMA_XGAM_PDI, - .name = "XGAM PDI", - .priority_high = 0, - }, - /* - * Don't set up device address, burst count or size of src - * or dst bus for this peripheral - handled by PrimeCell - * DMA extension. - */ - { - .number = U300_DMA_SPI_TX, - .name = "SPI TX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - }, - { - .number = U300_DMA_SPI_RX, - .name = "SPI RX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - - }, - { - .number = U300_DMA_GENERAL_PURPOSE_0, - .name = "GENERAL 00", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_1, - .name = "GENERAL 01", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_2, - .name = "GENERAL 02", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_3, - .name = "GENERAL 03", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_4, - .name = "GENERAL 04", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_5, - .name = "GENERAL 05", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_6, - .name = "GENERAL 06", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_7, - .name = "GENERAL 07", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_8, - .name = "GENERAL 08", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_UART1_TX, - .name = "UART1 TX", - .priority_high = 0, - }, - { - .number = U300_DMA_UART1_RX, - .name = "UART1 RX", - .priority_high = 0, - } -}; - -#define COHC_2_DEV(cohc) (&cohc->chan.dev->device) - -#ifdef VERBOSE_DEBUG -#define COH_DBG(x) ({ if (1) x; 0; }) -#else -#define COH_DBG(x) ({ if (0) x; 0; }) -#endif - -struct coh901318_desc { - struct dma_async_tx_descriptor desc; - struct list_head node; - struct scatterlist *sg; - unsigned int sg_len; - struct coh901318_lli *lli; - enum dma_transfer_direction dir; - unsigned long flags; - u32 head_config; - u32 head_ctrl; -}; - -struct coh901318_base { - struct device *dev; - void __iomem *virtbase; - unsigned int irq; - struct coh901318_pool pool; - struct powersave pm; - struct dma_device dma_slave; - struct dma_device dma_memcpy; - struct coh901318_chan *chans; -}; - -struct coh901318_chan { - spinlock_t lock; - int allocated; - int id; - int stopped; - - struct work_struct free_work; - struct dma_chan chan; - - struct tasklet_struct tasklet; - - struct list_head active; - struct list_head queue; - struct list_head free; - - unsigned long nbr_active_done; - unsigned long busy; - - struct dma_slave_config config; - u32 addr; - u32 ctrl; - - struct coh901318_base *base; -}; - -static void coh901318_list_print(struct coh901318_chan *cohc, - struct coh901318_lli *lli) -{ - struct coh901318_lli *l = lli; - int i = 0; - - while (l) { - dev_vdbg(COHC_2_DEV(cohc), "i %d, lli %p, ctrl 0x%x, src %pad" - ", dst %pad, link %pad virt_link_addr 0x%p\n", - i, l, l->control, &l->src_addr, &l->dst_addr, - &l->link_addr, l->virt_link_addr); - i++; - l = l->virt_link_addr; - } -} - -#ifdef CONFIG_DEBUG_FS - -#define COH901318_DEBUGFS_ASSIGN(x, y) (x = y) - -static struct coh901318_base *debugfs_dma_base; -static struct dentry *dma_dentry; - -static ssize_t coh901318_debugfs_read(struct file *file, char __user *buf, - size_t count, loff_t *f_pos) -{ - u64 started_channels = debugfs_dma_base->pm.started_channels; - int pool_count = debugfs_dma_base->pool.debugfs_pool_counter; - char *dev_buf; - char *tmp; - int ret; - int i; - - dev_buf = kmalloc(4*1024, GFP_KERNEL); - if (dev_buf == NULL) - return -ENOMEM; - tmp = dev_buf; - - tmp += sprintf(tmp, "DMA -- enabled dma channels\n"); - - for (i = 0; i < U300_DMA_CHANNELS; i++) { - if (started_channels & (1ULL << i)) - tmp += sprintf(tmp, "channel %d\n", i); - } - - tmp += sprintf(tmp, "Pool alloc nbr %d\n", pool_count); - - ret = simple_read_from_buffer(buf, count, f_pos, dev_buf, - tmp - dev_buf); - kfree(dev_buf); - return ret; -} - -static const struct file_operations coh901318_debugfs_status_operations = { - .open = simple_open, - .read = coh901318_debugfs_read, - .llseek = default_llseek, -}; - - -static int __init init_coh901318_debugfs(void) -{ - - dma_dentry = debugfs_create_dir("dma", NULL); - - debugfs_create_file("status", S_IFREG | S_IRUGO, dma_dentry, NULL, - &coh901318_debugfs_status_operations); - return 0; -} - -static void __exit exit_coh901318_debugfs(void) -{ - debugfs_remove_recursive(dma_dentry); -} - -module_init(init_coh901318_debugfs); -module_exit(exit_coh901318_debugfs); -#else - -#define COH901318_DEBUGFS_ASSIGN(x, y) - -#endif /* CONFIG_DEBUG_FS */ - -static inline struct coh901318_chan *to_coh901318_chan(struct dma_chan *chan) -{ - return container_of(chan, struct coh901318_chan, chan); -} - -static int coh901318_dma_set_runtimeconfig(struct dma_chan *chan, - struct dma_slave_config *config, - enum dma_transfer_direction direction); - -static inline const struct coh901318_params * -cohc_chan_param(struct coh901318_chan *cohc) -{ - return &chan_config[cohc->id].param; -} - -static inline const struct coh_dma_channel * -cohc_chan_conf(struct coh901318_chan *cohc) -{ - return &chan_config[cohc->id]; -} - -static void enable_powersave(struct coh901318_chan *cohc) -{ - unsigned long flags; - struct powersave *pm = &cohc->base->pm; - - spin_lock_irqsave(&pm->lock, flags); - - pm->started_channels &= ~(1ULL << cohc->id); - - spin_unlock_irqrestore(&pm->lock, flags); -} -static void disable_powersave(struct coh901318_chan *cohc) -{ - unsigned long flags; - struct powersave *pm = &cohc->base->pm; - - spin_lock_irqsave(&pm->lock, flags); - - pm->started_channels |= (1ULL << cohc->id); - - spin_unlock_irqrestore(&pm->lock, flags); -} - -static inline int coh901318_set_ctrl(struct coh901318_chan *cohc, u32 control) -{ - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - writel(control, - virtbase + COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING * channel); - return 0; -} - -static inline int coh901318_set_conf(struct coh901318_chan *cohc, u32 conf) -{ - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - writel(conf, - virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING*channel); - return 0; -} - - -static int coh901318_start(struct coh901318_chan *cohc) -{ - u32 val; - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - disable_powersave(cohc); - - val = readl(virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - /* Enable channel */ - val |= COH901318_CX_CFG_CH_ENABLE; - writel(val, virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - return 0; -} - -static int coh901318_prep_linked_list(struct coh901318_chan *cohc, - struct coh901318_lli *lli) -{ - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - BUG_ON(readl(virtbase + COH901318_CX_STAT + - COH901318_CX_STAT_SPACING*channel) & - COH901318_CX_STAT_ACTIVE); - - writel(lli->src_addr, - virtbase + COH901318_CX_SRC_ADDR + - COH901318_CX_SRC_ADDR_SPACING * channel); - - writel(lli->dst_addr, virtbase + - COH901318_CX_DST_ADDR + - COH901318_CX_DST_ADDR_SPACING * channel); - - writel(lli->link_addr, virtbase + COH901318_CX_LNK_ADDR + - COH901318_CX_LNK_ADDR_SPACING * channel); - - writel(lli->control, virtbase + COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING * channel); - - return 0; -} - -static struct coh901318_desc * -coh901318_desc_get(struct coh901318_chan *cohc) -{ - struct coh901318_desc *desc; - - if (list_empty(&cohc->free)) { - /* alloc new desc because we're out of used ones - * TODO: alloc a pile of descs instead of just one, - * avoid many small allocations. - */ - desc = kzalloc(sizeof(struct coh901318_desc), GFP_NOWAIT); - if (desc == NULL) - goto out; - INIT_LIST_HEAD(&desc->node); - dma_async_tx_descriptor_init(&desc->desc, &cohc->chan); - } else { - /* Reuse an old desc. */ - desc = list_first_entry(&cohc->free, - struct coh901318_desc, - node); - list_del(&desc->node); - /* Initialize it a bit so it's not insane */ - desc->sg = NULL; - desc->sg_len = 0; - desc->desc.callback = NULL; - desc->desc.callback_param = NULL; - } - - out: - return desc; -} - -static void -coh901318_desc_free(struct coh901318_chan *cohc, struct coh901318_desc *cohd) -{ - list_add_tail(&cohd->node, &cohc->free); -} - -/* call with irq lock held */ -static void -coh901318_desc_submit(struct coh901318_chan *cohc, struct coh901318_desc *desc) -{ - list_add_tail(&desc->node, &cohc->active); -} - -static struct coh901318_desc * -coh901318_first_active_get(struct coh901318_chan *cohc) -{ - return list_first_entry_or_null(&cohc->active, struct coh901318_desc, - node); -} - -static void -coh901318_desc_remove(struct coh901318_desc *cohd) -{ - list_del(&cohd->node); -} - -static void -coh901318_desc_queue(struct coh901318_chan *cohc, struct coh901318_desc *desc) -{ - list_add_tail(&desc->node, &cohc->queue); -} - -static struct coh901318_desc * -coh901318_first_queued(struct coh901318_chan *cohc) -{ - return list_first_entry_or_null(&cohc->queue, struct coh901318_desc, - node); -} - -static inline u32 coh901318_get_bytes_in_lli(struct coh901318_lli *in_lli) -{ - struct coh901318_lli *lli = in_lli; - u32 bytes = 0; - - while (lli) { - bytes += lli->control & COH901318_CX_CTRL_TC_VALUE_MASK; - lli = lli->virt_link_addr; - } - return bytes; -} - -/* - * Get the number of bytes left to transfer on this channel, - * it is unwise to call this before stopping the channel for - * absolute measures, but for a rough guess you can still call - * it. - */ -static u32 coh901318_get_bytes_left(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - struct coh901318_desc *cohd; - struct list_head *pos; - unsigned long flags; - u32 left = 0; - int i = 0; - - spin_lock_irqsave(&cohc->lock, flags); - - /* - * If there are many queued jobs, we iterate and add the - * size of them all. We take a special look on the first - * job though, since it is probably active. - */ - list_for_each(pos, &cohc->active) { - /* - * The first job in the list will be working on the - * hardware. The job can be stopped but still active, - * so that the transfer counter is somewhere inside - * the buffer. - */ - cohd = list_entry(pos, struct coh901318_desc, node); - - if (i == 0) { - struct coh901318_lli *lli; - dma_addr_t ladd; - - /* Read current transfer count value */ - left = readl(cohc->base->virtbase + - COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING * cohc->id) & - COH901318_CX_CTRL_TC_VALUE_MASK; - - /* See if the transfer is linked... */ - ladd = readl(cohc->base->virtbase + - COH901318_CX_LNK_ADDR + - COH901318_CX_LNK_ADDR_SPACING * - cohc->id) & - ~COH901318_CX_LNK_LINK_IMMEDIATE; - /* Single transaction */ - if (!ladd) - continue; - - /* - * Linked transaction, follow the lli, find the - * currently processing lli, and proceed to the next - */ - lli = cohd->lli; - while (lli && lli->link_addr != ladd) - lli = lli->virt_link_addr; - - if (lli) - lli = lli->virt_link_addr; - - /* - * Follow remaining lli links around to count the total - * number of bytes left - */ - left += coh901318_get_bytes_in_lli(lli); - } else { - left += coh901318_get_bytes_in_lli(cohd->lli); - } - i++; - } - - /* Also count bytes in the queued jobs */ - list_for_each(pos, &cohc->queue) { - cohd = list_entry(pos, struct coh901318_desc, node); - left += coh901318_get_bytes_in_lli(cohd->lli); - } - - spin_unlock_irqrestore(&cohc->lock, flags); - - return left; -} - -/* - * Pauses a transfer without losing data. Enables power save. - * Use this function in conjunction with coh901318_resume. - */ -static int coh901318_pause(struct dma_chan *chan) -{ - u32 val; - unsigned long flags; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - spin_lock_irqsave(&cohc->lock, flags); - - /* Disable channel in HW */ - val = readl(virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - /* Stopping infinite transfer */ - if ((val & COH901318_CX_CTRL_TC_ENABLE) == 0 && - (val & COH901318_CX_CFG_CH_ENABLE)) - cohc->stopped = 1; - - - val &= ~COH901318_CX_CFG_CH_ENABLE; - /* Enable twice, HW bug work around */ - writel(val, virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - writel(val, virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - /* Spin-wait for it to actually go inactive */ - while (readl(virtbase + COH901318_CX_STAT+COH901318_CX_STAT_SPACING * - channel) & COH901318_CX_STAT_ACTIVE) - cpu_relax(); - - /* Check if we stopped an active job */ - if ((readl(virtbase + COH901318_CX_CTRL+COH901318_CX_CTRL_SPACING * - channel) & COH901318_CX_CTRL_TC_VALUE_MASK) > 0) - cohc->stopped = 1; - - enable_powersave(cohc); - - spin_unlock_irqrestore(&cohc->lock, flags); - return 0; -} - -/* Resumes a transfer that has been stopped via 300_dma_stop(..). - Power save is handled. -*/ -static int coh901318_resume(struct dma_chan *chan) -{ - u32 val; - unsigned long flags; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int channel = cohc->id; - - spin_lock_irqsave(&cohc->lock, flags); - - disable_powersave(cohc); - - if (cohc->stopped) { - /* Enable channel in HW */ - val = readl(cohc->base->virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - val |= COH901318_CX_CFG_CH_ENABLE; - - writel(val, cohc->base->virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING*channel); - - cohc->stopped = 0; - } - - spin_unlock_irqrestore(&cohc->lock, flags); - return 0; -} - -bool coh901318_filter_id(struct dma_chan *chan, void *chan_id) -{ - unsigned long ch_nr = (unsigned long) chan_id; - - if (ch_nr == to_coh901318_chan(chan)->id) - return true; - - return false; -} -EXPORT_SYMBOL(coh901318_filter_id); - -struct coh901318_filter_args { - struct coh901318_base *base; - unsigned int ch_nr; -}; - -static bool coh901318_filter_base_and_id(struct dma_chan *chan, void *data) -{ - struct coh901318_filter_args *args = data; - - if (&args->base->dma_slave == chan->device && - args->ch_nr == to_coh901318_chan(chan)->id) - return true; - - return false; -} - -static struct dma_chan *coh901318_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct coh901318_filter_args args = { - .base = ofdma->of_dma_data, - .ch_nr = dma_spec->args[0], - }; - dma_cap_mask_t cap; - dma_cap_zero(cap); - dma_cap_set(DMA_SLAVE, cap); - - return dma_request_channel(cap, coh901318_filter_base_and_id, &args); -} -/* - * DMA channel allocation - */ -static int coh901318_config(struct coh901318_chan *cohc, - struct coh901318_params *param) -{ - const struct coh901318_params *p; - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - if (param) - p = param; - else - p = cohc_chan_param(cohc); - - /* Clear any pending BE or TC interrupt */ - if (channel < 32) { - writel(1 << channel, virtbase + COH901318_BE_INT_CLEAR1); - writel(1 << channel, virtbase + COH901318_TC_INT_CLEAR1); - } else { - writel(1 << (channel - 32), virtbase + - COH901318_BE_INT_CLEAR2); - writel(1 << (channel - 32), virtbase + - COH901318_TC_INT_CLEAR2); - } - - coh901318_set_conf(cohc, p->config); - coh901318_set_ctrl(cohc, p->ctrl_lli_last); - - return 0; -} - -/* must lock when calling this function - * start queued jobs, if any - * TODO: start all queued jobs in one go - * - * Returns descriptor if queued job is started otherwise NULL. - * If the queue is empty NULL is returned. - */ -static struct coh901318_desc *coh901318_queue_start(struct coh901318_chan *cohc) -{ - struct coh901318_desc *cohd; - - /* - * start queued jobs, if any - * TODO: transmit all queued jobs in one go - */ - cohd = coh901318_first_queued(cohc); - - if (cohd != NULL) { - /* Remove from queue */ - coh901318_desc_remove(cohd); - /* initiate DMA job */ - cohc->busy = 1; - - coh901318_desc_submit(cohc, cohd); - - /* Program the transaction head */ - coh901318_set_conf(cohc, cohd->head_config); - coh901318_set_ctrl(cohc, cohd->head_ctrl); - coh901318_prep_linked_list(cohc, cohd->lli); - - /* start dma job on this channel */ - coh901318_start(cohc); - - } - - return cohd; -} - -/* - * This tasklet is called from the interrupt handler to - * handle each descriptor (DMA job) that is sent to a channel. - */ -static void dma_tasklet(struct tasklet_struct *t) -{ - struct coh901318_chan *cohc = from_tasklet(cohc, t, tasklet); - struct coh901318_desc *cohd_fin; - unsigned long flags; - struct dmaengine_desc_callback cb; - - dev_vdbg(COHC_2_DEV(cohc), "[%s] chan_id %d" - " nbr_active_done %ld\n", __func__, - cohc->id, cohc->nbr_active_done); - - spin_lock_irqsave(&cohc->lock, flags); - - /* get first active descriptor entry from list */ - cohd_fin = coh901318_first_active_get(cohc); - - if (cohd_fin == NULL) - goto err; - - /* locate callback to client */ - dmaengine_desc_get_callback(&cohd_fin->desc, &cb); - - /* sign this job as completed on the channel */ - dma_cookie_complete(&cohd_fin->desc); - - /* release the lli allocation and remove the descriptor */ - coh901318_lli_free(&cohc->base->pool, &cohd_fin->lli); - - /* return desc to free-list */ - coh901318_desc_remove(cohd_fin); - coh901318_desc_free(cohc, cohd_fin); - - spin_unlock_irqrestore(&cohc->lock, flags); - - /* Call the callback when we're done */ - dmaengine_desc_callback_invoke(&cb, NULL); - - spin_lock_irqsave(&cohc->lock, flags); - - /* - * If another interrupt fired while the tasklet was scheduling, - * we don't get called twice, so we have this number of active - * counter that keep track of the number of IRQs expected to - * be handled for this channel. If there happen to be more than - * one IRQ to be ack:ed, we simply schedule this tasklet again. - */ - cohc->nbr_active_done--; - if (cohc->nbr_active_done) { - dev_dbg(COHC_2_DEV(cohc), "scheduling tasklet again, new IRQs " - "came in while we were scheduling this tasklet\n"); - if (cohc_chan_conf(cohc)->priority_high) - tasklet_hi_schedule(&cohc->tasklet); - else - tasklet_schedule(&cohc->tasklet); - } - - spin_unlock_irqrestore(&cohc->lock, flags); - - return; - - err: - spin_unlock_irqrestore(&cohc->lock, flags); - dev_err(COHC_2_DEV(cohc), "[%s] No active dma desc\n", __func__); -} - - -/* called from interrupt context */ -static void dma_tc_handle(struct coh901318_chan *cohc) -{ - /* - * If the channel is not allocated, then we shouldn't have - * any TC interrupts on it. - */ - if (!cohc->allocated) { - dev_err(COHC_2_DEV(cohc), "spurious interrupt from " - "unallocated channel\n"); - return; - } - - /* - * When we reach this point, at least one queue item - * should have been moved over from cohc->queue to - * cohc->active and run to completion, that is why we're - * getting a terminal count interrupt is it not? - * If you get this BUG() the most probable cause is that - * the individual nodes in the lli chain have IRQ enabled, - * so check your platform config for lli chain ctrl. - */ - BUG_ON(list_empty(&cohc->active)); - - cohc->nbr_active_done++; - - /* - * This attempt to take a job from cohc->queue, put it - * into cohc->active and start it. - */ - if (coh901318_queue_start(cohc) == NULL) - cohc->busy = 0; - - /* - * This tasklet will remove items from cohc->active - * and thus terminates them. - */ - if (cohc_chan_conf(cohc)->priority_high) - tasklet_hi_schedule(&cohc->tasklet); - else - tasklet_schedule(&cohc->tasklet); -} - - -static irqreturn_t dma_irq_handler(int irq, void *dev_id) -{ - u32 status1; - u32 status2; - int i; - int ch; - struct coh901318_base *base = dev_id; - struct coh901318_chan *cohc; - void __iomem *virtbase = base->virtbase; - - status1 = readl(virtbase + COH901318_INT_STATUS1); - status2 = readl(virtbase + COH901318_INT_STATUS2); - - if (unlikely(status1 == 0 && status2 == 0)) { - dev_warn(base->dev, "spurious DMA IRQ from no channel!\n"); - return IRQ_HANDLED; - } - - /* TODO: consider handle IRQ in tasklet here to - * minimize interrupt latency */ - - /* Check the first 32 DMA channels for IRQ */ - while (status1) { - /* Find first bit set, return as a number. */ - i = ffs(status1) - 1; - ch = i; - - cohc = &base->chans[ch]; - spin_lock(&cohc->lock); - - /* Mask off this bit */ - status1 &= ~(1 << i); - /* Check the individual channel bits */ - if (test_bit(i, virtbase + COH901318_BE_INT_STATUS1)) { - dev_crit(COHC_2_DEV(cohc), - "DMA bus error on channel %d!\n", ch); - BUG_ON(1); - /* Clear BE interrupt */ - __set_bit(i, virtbase + COH901318_BE_INT_CLEAR1); - } else { - /* Caused by TC, really? */ - if (unlikely(!test_bit(i, virtbase + - COH901318_TC_INT_STATUS1))) { - dev_warn(COHC_2_DEV(cohc), - "ignoring interrupt not caused by terminal count on channel %d\n", ch); - /* Clear TC interrupt */ - BUG_ON(1); - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR1); - } else { - /* Enable powersave if transfer has finished */ - if (!(readl(virtbase + COH901318_CX_STAT + - COH901318_CX_STAT_SPACING*ch) & - COH901318_CX_STAT_ENABLED)) { - enable_powersave(cohc); - } - - /* Must clear TC interrupt before calling - * dma_tc_handle - * in case tc_handle initiate a new dma job - */ - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR1); - - dma_tc_handle(cohc); - } - } - spin_unlock(&cohc->lock); - } - - /* Check the remaining 32 DMA channels for IRQ */ - while (status2) { - /* Find first bit set, return as a number. */ - i = ffs(status2) - 1; - ch = i + 32; - cohc = &base->chans[ch]; - spin_lock(&cohc->lock); - - /* Mask off this bit */ - status2 &= ~(1 << i); - /* Check the individual channel bits */ - if (test_bit(i, virtbase + COH901318_BE_INT_STATUS2)) { - dev_crit(COHC_2_DEV(cohc), - "DMA bus error on channel %d!\n", ch); - /* Clear BE interrupt */ - BUG_ON(1); - __set_bit(i, virtbase + COH901318_BE_INT_CLEAR2); - } else { - /* Caused by TC, really? */ - if (unlikely(!test_bit(i, virtbase + - COH901318_TC_INT_STATUS2))) { - dev_warn(COHC_2_DEV(cohc), - "ignoring interrupt not caused by terminal count on channel %d\n", ch); - /* Clear TC interrupt */ - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR2); - BUG_ON(1); - } else { - /* Enable powersave if transfer has finished */ - if (!(readl(virtbase + COH901318_CX_STAT + - COH901318_CX_STAT_SPACING*ch) & - COH901318_CX_STAT_ENABLED)) { - enable_powersave(cohc); - } - /* Must clear TC interrupt before calling - * dma_tc_handle - * in case tc_handle initiate a new dma job - */ - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR2); - - dma_tc_handle(cohc); - } - } - spin_unlock(&cohc->lock); - } - - return IRQ_HANDLED; -} - -static int coh901318_terminate_all(struct dma_chan *chan) -{ - unsigned long flags; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - struct coh901318_desc *cohd; - void __iomem *virtbase = cohc->base->virtbase; - - /* The remainder of this function terminates the transfer */ - coh901318_pause(chan); - spin_lock_irqsave(&cohc->lock, flags); - - /* Clear any pending BE or TC interrupt */ - if (cohc->id < 32) { - writel(1 << cohc->id, virtbase + COH901318_BE_INT_CLEAR1); - writel(1 << cohc->id, virtbase + COH901318_TC_INT_CLEAR1); - } else { - writel(1 << (cohc->id - 32), virtbase + - COH901318_BE_INT_CLEAR2); - writel(1 << (cohc->id - 32), virtbase + - COH901318_TC_INT_CLEAR2); - } - - enable_powersave(cohc); - - while ((cohd = coh901318_first_active_get(cohc))) { - /* release the lli allocation*/ - coh901318_lli_free(&cohc->base->pool, &cohd->lli); - - /* return desc to free-list */ - coh901318_desc_remove(cohd); - coh901318_desc_free(cohc, cohd); - } - - while ((cohd = coh901318_first_queued(cohc))) { - /* release the lli allocation*/ - coh901318_lli_free(&cohc->base->pool, &cohd->lli); - - /* return desc to free-list */ - coh901318_desc_remove(cohd); - coh901318_desc_free(cohc, cohd); - } - - - cohc->nbr_active_done = 0; - cohc->busy = 0; - - spin_unlock_irqrestore(&cohc->lock, flags); - - return 0; -} - -static int coh901318_alloc_chan_resources(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - unsigned long flags; - - dev_vdbg(COHC_2_DEV(cohc), "[%s] DMA channel %d\n", - __func__, cohc->id); - - if (chan->client_count > 1) - return -EBUSY; - - spin_lock_irqsave(&cohc->lock, flags); - - coh901318_config(cohc, NULL); - - cohc->allocated = 1; - dma_cookie_init(chan); - - spin_unlock_irqrestore(&cohc->lock, flags); - - return 1; -} - -static void -coh901318_free_chan_resources(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int channel = cohc->id; - unsigned long flags; - - spin_lock_irqsave(&cohc->lock, flags); - - /* Disable HW */ - writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING*channel); - writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING*channel); - - cohc->allocated = 0; - - spin_unlock_irqrestore(&cohc->lock, flags); - - coh901318_terminate_all(chan); -} - - -static dma_cookie_t -coh901318_tx_submit(struct dma_async_tx_descriptor *tx) -{ - struct coh901318_desc *cohd = container_of(tx, struct coh901318_desc, - desc); - struct coh901318_chan *cohc = to_coh901318_chan(tx->chan); - unsigned long flags; - dma_cookie_t cookie; - - spin_lock_irqsave(&cohc->lock, flags); - cookie = dma_cookie_assign(tx); - - coh901318_desc_queue(cohc, cohd); - - spin_unlock_irqrestore(&cohc->lock, flags); - - return cookie; -} - -static struct dma_async_tx_descriptor * -coh901318_prep_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, - size_t size, unsigned long flags) -{ - struct coh901318_lli *lli; - struct coh901318_desc *cohd; - unsigned long flg; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int lli_len; - u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last; - int ret; - - spin_lock_irqsave(&cohc->lock, flg); - - dev_vdbg(COHC_2_DEV(cohc), - "[%s] channel %d src %pad dest %pad size %zu\n", - __func__, cohc->id, &src, &dest, size); - - if (flags & DMA_PREP_INTERRUPT) - /* Trigger interrupt after last lli */ - ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE; - - lli_len = size >> MAX_DMA_PACKET_SIZE_SHIFT; - if ((lli_len << MAX_DMA_PACKET_SIZE_SHIFT) < size) - lli_len++; - - lli = coh901318_lli_alloc(&cohc->base->pool, lli_len); - - if (lli == NULL) - goto err; - - ret = coh901318_lli_fill_memcpy( - &cohc->base->pool, lli, src, size, dest, - cohc_chan_param(cohc)->ctrl_lli_chained, - ctrl_last); - if (ret) - goto err; - - COH_DBG(coh901318_list_print(cohc, lli)); - - /* Pick a descriptor to handle this transfer */ - cohd = coh901318_desc_get(cohc); - cohd->lli = lli; - cohd->flags = flags; - cohd->desc.tx_submit = coh901318_tx_submit; - - spin_unlock_irqrestore(&cohc->lock, flg); - - return &cohd->desc; - err: - spin_unlock_irqrestore(&cohc->lock, flg); - return NULL; -} - -static struct dma_async_tx_descriptor * -coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, - unsigned int sg_len, enum dma_transfer_direction direction, - unsigned long flags, void *context) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - struct coh901318_lli *lli; - struct coh901318_desc *cohd; - const struct coh901318_params *params; - struct scatterlist *sg; - int len = 0; - int size; - int i; - u32 ctrl_chained = cohc_chan_param(cohc)->ctrl_lli_chained; - u32 ctrl = cohc_chan_param(cohc)->ctrl_lli; - u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last; - u32 config; - unsigned long flg; - int ret; - - if (!sgl) - goto out; - if (sg_dma_len(sgl) == 0) - goto out; - - spin_lock_irqsave(&cohc->lock, flg); - - dev_vdbg(COHC_2_DEV(cohc), "[%s] sg_len %d dir %d\n", - __func__, sg_len, direction); - - if (flags & DMA_PREP_INTERRUPT) - /* Trigger interrupt after last lli */ - ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE; - - params = cohc_chan_param(cohc); - config = params->config; - /* - * Add runtime-specific control on top, make - * sure the bits you set per peripheral channel are - * cleared in the default config from the platform. - */ - ctrl_chained |= cohc->ctrl; - ctrl_last |= cohc->ctrl; - ctrl |= cohc->ctrl; - - if (direction == DMA_MEM_TO_DEV) { - u32 tx_flags = COH901318_CX_CTRL_PRDD_SOURCE | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE; - - config |= COH901318_CX_CFG_RM_MEMORY_TO_PRIMARY; - ctrl_chained |= tx_flags; - ctrl_last |= tx_flags; - ctrl |= tx_flags; - } else if (direction == DMA_DEV_TO_MEM) { - u32 rx_flags = COH901318_CX_CTRL_PRDD_DEST | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE; - - config |= COH901318_CX_CFG_RM_PRIMARY_TO_MEMORY; - ctrl_chained |= rx_flags; - ctrl_last |= rx_flags; - ctrl |= rx_flags; - } else - goto err_direction; - - /* The dma only supports transmitting packages up to - * MAX_DMA_PACKET_SIZE. Calculate to total number of - * dma elemts required to send the entire sg list - */ - for_each_sg(sgl, sg, sg_len, i) { - unsigned int factor; - size = sg_dma_len(sg); - - if (size <= MAX_DMA_PACKET_SIZE) { - len++; - continue; - } - - factor = size >> MAX_DMA_PACKET_SIZE_SHIFT; - if ((factor << MAX_DMA_PACKET_SIZE_SHIFT) < size) - factor++; - - len += factor; - } - - pr_debug("Allocate %d lli:s for this transfer\n", len); - lli = coh901318_lli_alloc(&cohc->base->pool, len); - - if (lli == NULL) - goto err_dma_alloc; - - coh901318_dma_set_runtimeconfig(chan, &cohc->config, direction); - - /* initiate allocated lli list */ - ret = coh901318_lli_fill_sg(&cohc->base->pool, lli, sgl, sg_len, - cohc->addr, - ctrl_chained, - ctrl, - ctrl_last, - direction, COH901318_CX_CTRL_TC_IRQ_ENABLE); - if (ret) - goto err_lli_fill; - - - COH_DBG(coh901318_list_print(cohc, lli)); - - /* Pick a descriptor to handle this transfer */ - cohd = coh901318_desc_get(cohc); - cohd->head_config = config; - /* - * Set the default head ctrl for the channel to the one from the - * lli, things may have changed due to odd buffer alignment - * etc. - */ - cohd->head_ctrl = lli->control; - cohd->dir = direction; - cohd->flags = flags; - cohd->desc.tx_submit = coh901318_tx_submit; - cohd->lli = lli; - - spin_unlock_irqrestore(&cohc->lock, flg); - - return &cohd->desc; - err_lli_fill: - err_dma_alloc: - err_direction: - spin_unlock_irqrestore(&cohc->lock, flg); - out: - return NULL; -} - -static enum dma_status -coh901318_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - enum dma_status ret; - - ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE || !txstate) - return ret; - - dma_set_residue(txstate, coh901318_get_bytes_left(chan)); - - if (ret == DMA_IN_PROGRESS && cohc->stopped) - ret = DMA_PAUSED; - - return ret; -} - -static void -coh901318_issue_pending(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&cohc->lock, flags); - - /* - * Busy means that pending jobs are already being processed, - * and then there is no point in starting the queue: the - * terminal count interrupt on the channel will take the next - * job on the queue and execute it anyway. - */ - if (!cohc->busy) - coh901318_queue_start(cohc); - - spin_unlock_irqrestore(&cohc->lock, flags); -} - -/* - * Here we wrap in the runtime dma control interface - */ -struct burst_table { - int burst_8bit; - int burst_16bit; - int burst_32bit; - u32 reg; -}; - -static const struct burst_table burst_sizes[] = { - { - .burst_8bit = 64, - .burst_16bit = 32, - .burst_32bit = 16, - .reg = COH901318_CX_CTRL_BURST_COUNT_64_BYTES, - }, - { - .burst_8bit = 48, - .burst_16bit = 24, - .burst_32bit = 12, - .reg = COH901318_CX_CTRL_BURST_COUNT_48_BYTES, - }, - { - .burst_8bit = 32, - .burst_16bit = 16, - .burst_32bit = 8, - .reg = COH901318_CX_CTRL_BURST_COUNT_32_BYTES, - }, - { - .burst_8bit = 16, - .burst_16bit = 8, - .burst_32bit = 4, - .reg = COH901318_CX_CTRL_BURST_COUNT_16_BYTES, - }, - { - .burst_8bit = 8, - .burst_16bit = 4, - .burst_32bit = 2, - .reg = COH901318_CX_CTRL_BURST_COUNT_8_BYTES, - }, - { - .burst_8bit = 4, - .burst_16bit = 2, - .burst_32bit = 1, - .reg = COH901318_CX_CTRL_BURST_COUNT_4_BYTES, - }, - { - .burst_8bit = 2, - .burst_16bit = 1, - .burst_32bit = 0, - .reg = COH901318_CX_CTRL_BURST_COUNT_2_BYTES, - }, - { - .burst_8bit = 1, - .burst_16bit = 0, - .burst_32bit = 0, - .reg = COH901318_CX_CTRL_BURST_COUNT_1_BYTE, - }, -}; - -static int coh901318_dma_set_runtimeconfig(struct dma_chan *chan, - struct dma_slave_config *config, - enum dma_transfer_direction direction) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - dma_addr_t addr; - enum dma_slave_buswidth addr_width; - u32 maxburst; - u32 ctrl = 0; - int i = 0; - - /* We only support mem to per or per to mem transfers */ - if (direction == DMA_DEV_TO_MEM) { - addr = config->src_addr; - addr_width = config->src_addr_width; - maxburst = config->src_maxburst; - } else if (direction == DMA_MEM_TO_DEV) { - addr = config->dst_addr; - addr_width = config->dst_addr_width; - maxburst = config->dst_maxburst; - } else { - dev_err(COHC_2_DEV(cohc), "illegal channel mode\n"); - return -EINVAL; - } - - dev_dbg(COHC_2_DEV(cohc), "configure channel for %d byte transfers\n", - addr_width); - switch (addr_width) { - case DMA_SLAVE_BUSWIDTH_1_BYTE: - ctrl |= - COH901318_CX_CTRL_SRC_BUS_SIZE_8_BITS | - COH901318_CX_CTRL_DST_BUS_SIZE_8_BITS; - - while (i < ARRAY_SIZE(burst_sizes)) { - if (burst_sizes[i].burst_8bit <= maxburst) - break; - i++; - } - - break; - case DMA_SLAVE_BUSWIDTH_2_BYTES: - ctrl |= - COH901318_CX_CTRL_SRC_BUS_SIZE_16_BITS | - COH901318_CX_CTRL_DST_BUS_SIZE_16_BITS; - - while (i < ARRAY_SIZE(burst_sizes)) { - if (burst_sizes[i].burst_16bit <= maxburst) - break; - i++; - } - - break; - case DMA_SLAVE_BUSWIDTH_4_BYTES: - /* Direction doesn't matter here, it's 32/32 bits */ - ctrl |= - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS; - - while (i < ARRAY_SIZE(burst_sizes)) { - if (burst_sizes[i].burst_32bit <= maxburst) - break; - i++; - } - - break; - default: - dev_err(COHC_2_DEV(cohc), - "bad runtimeconfig: alien address width\n"); - return -EINVAL; - } - - ctrl |= burst_sizes[i].reg; - dev_dbg(COHC_2_DEV(cohc), - "selected burst size %d bytes for address width %d bytes, maxburst %d\n", - burst_sizes[i].burst_8bit, addr_width, maxburst); - - cohc->addr = addr; - cohc->ctrl = ctrl; - - return 0; -} - -static int coh901318_dma_slave_config(struct dma_chan *chan, - struct dma_slave_config *config) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - - memcpy(&cohc->config, config, sizeof(*config)); - - return 0; -} - -static void coh901318_base_init(struct dma_device *dma, const int *pick_chans, - struct coh901318_base *base) -{ - int chans_i; - int i = 0; - struct coh901318_chan *cohc; - - INIT_LIST_HEAD(&dma->channels); - - for (chans_i = 0; pick_chans[chans_i] != -1; chans_i += 2) { - for (i = pick_chans[chans_i]; i <= pick_chans[chans_i+1]; i++) { - cohc = &base->chans[i]; - - cohc->base = base; - cohc->chan.device = dma; - cohc->id = i; - - /* TODO: do we really need this lock if only one - * client is connected to each channel? - */ - - spin_lock_init(&cohc->lock); - - cohc->nbr_active_done = 0; - cohc->busy = 0; - INIT_LIST_HEAD(&cohc->free); - INIT_LIST_HEAD(&cohc->active); - INIT_LIST_HEAD(&cohc->queue); - - tasklet_setup(&cohc->tasklet, dma_tasklet); - - list_add_tail(&cohc->chan.device_node, - &dma->channels); - } - } -} - -static int __init coh901318_probe(struct platform_device *pdev) -{ - int err = 0; - struct coh901318_base *base; - int irq; - struct resource *io; - - io = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!io) - return -ENODEV; - - /* Map DMA controller registers to virtual memory */ - if (devm_request_mem_region(&pdev->dev, - io->start, - resource_size(io), - pdev->dev.driver->name) == NULL) - return -ENOMEM; - - base = devm_kzalloc(&pdev->dev, - ALIGN(sizeof(struct coh901318_base), 4) + - U300_DMA_CHANNELS * - sizeof(struct coh901318_chan), - GFP_KERNEL); - if (!base) - return -ENOMEM; - - base->chans = ((void *)base) + ALIGN(sizeof(struct coh901318_base), 4); - - base->virtbase = devm_ioremap(&pdev->dev, io->start, resource_size(io)); - if (!base->virtbase) - return -ENOMEM; - - base->dev = &pdev->dev; - spin_lock_init(&base->pm.lock); - base->pm.started_channels = 0; - - COH901318_DEBUGFS_ASSIGN(debugfs_dma_base, base); - - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; - - err = devm_request_irq(&pdev->dev, irq, dma_irq_handler, 0, - "coh901318", base); - if (err) - return err; - - base->irq = irq; - - err = coh901318_pool_create(&base->pool, &pdev->dev, - sizeof(struct coh901318_lli), - 32); - if (err) - return err; - - /* init channels for device transfers */ - coh901318_base_init(&base->dma_slave, dma_slave_channels, - base); - - dma_cap_zero(base->dma_slave.cap_mask); - dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask); - - base->dma_slave.device_alloc_chan_resources = coh901318_alloc_chan_resources; - base->dma_slave.device_free_chan_resources = coh901318_free_chan_resources; - base->dma_slave.device_prep_slave_sg = coh901318_prep_slave_sg; - base->dma_slave.device_tx_status = coh901318_tx_status; - base->dma_slave.device_issue_pending = coh901318_issue_pending; - base->dma_slave.device_config = coh901318_dma_slave_config; - base->dma_slave.device_pause = coh901318_pause; - base->dma_slave.device_resume = coh901318_resume; - base->dma_slave.device_terminate_all = coh901318_terminate_all; - base->dma_slave.dev = &pdev->dev; - - err = dma_async_device_register(&base->dma_slave); - - if (err) - goto err_register_slave; - - /* init channels for memcpy */ - coh901318_base_init(&base->dma_memcpy, dma_memcpy_channels, - base); - - dma_cap_zero(base->dma_memcpy.cap_mask); - dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask); - - base->dma_memcpy.device_alloc_chan_resources = coh901318_alloc_chan_resources; - base->dma_memcpy.device_free_chan_resources = coh901318_free_chan_resources; - base->dma_memcpy.device_prep_dma_memcpy = coh901318_prep_memcpy; - base->dma_memcpy.device_tx_status = coh901318_tx_status; - base->dma_memcpy.device_issue_pending = coh901318_issue_pending; - base->dma_memcpy.device_config = coh901318_dma_slave_config; - base->dma_memcpy.device_pause = coh901318_pause; - base->dma_memcpy.device_resume = coh901318_resume; - base->dma_memcpy.device_terminate_all = coh901318_terminate_all; - base->dma_memcpy.dev = &pdev->dev; - /* - * This controller can only access address at even 32bit boundaries, - * i.e. 2^2 - */ - base->dma_memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES; - err = dma_async_device_register(&base->dma_memcpy); - - if (err) - goto err_register_memcpy; - - err = of_dma_controller_register(pdev->dev.of_node, coh901318_xlate, - base); - if (err) - goto err_register_of_dma; - - platform_set_drvdata(pdev, base); - dev_info(&pdev->dev, "Initialized COH901318 DMA on virtual base 0x%p\n", - base->virtbase); - - return err; - - err_register_of_dma: - dma_async_device_unregister(&base->dma_memcpy); - err_register_memcpy: - dma_async_device_unregister(&base->dma_slave); - err_register_slave: - coh901318_pool_destroy(&base->pool); - return err; -} -static void coh901318_base_remove(struct coh901318_base *base, const int *pick_chans) -{ - int chans_i; - int i = 0; - struct coh901318_chan *cohc; - - for (chans_i = 0; pick_chans[chans_i] != -1; chans_i += 2) { - for (i = pick_chans[chans_i]; i <= pick_chans[chans_i+1]; i++) { - cohc = &base->chans[i]; - - tasklet_kill(&cohc->tasklet); - } - } - -} - -static int coh901318_remove(struct platform_device *pdev) -{ - struct coh901318_base *base = platform_get_drvdata(pdev); - - devm_free_irq(&pdev->dev, base->irq, base); - - coh901318_base_remove(base, dma_slave_channels); - coh901318_base_remove(base, dma_memcpy_channels); - - of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&base->dma_memcpy); - dma_async_device_unregister(&base->dma_slave); - coh901318_pool_destroy(&base->pool); - return 0; -} - -static const struct of_device_id coh901318_dt_match[] = { - { .compatible = "stericsson,coh901318" }, - {}, -}; - -static struct platform_driver coh901318_driver = { - .remove = coh901318_remove, - .driver = { - .name = "coh901318", - .of_match_table = coh901318_dt_match, - }, -}; - -static int __init coh901318_init(void) -{ - return platform_driver_probe(&coh901318_driver, coh901318_probe); -} -subsys_initcall(coh901318_init); - -static void __exit coh901318_exit(void) -{ - platform_driver_unregister(&coh901318_driver); -} -module_exit(coh901318_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Per Friden"); diff --git a/drivers/dma/coh901318.h b/drivers/dma/coh901318.h deleted file mode 100644 index bbf533600558..000000000000 --- a/drivers/dma/coh901318.h +++ /dev/null @@ -1,141 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2007-2013 ST-Ericsson - * DMA driver for COH 901 318 - * Author: Per Friden - */ - -#ifndef COH901318_H -#define COH901318_H - -#define MAX_DMA_PACKET_SIZE_SHIFT 11 -#define MAX_DMA_PACKET_SIZE (1 << MAX_DMA_PACKET_SIZE_SHIFT) - -struct device; - -struct coh901318_pool { - spinlock_t lock; - struct dma_pool *dmapool; - struct device *dev; - -#ifdef CONFIG_DEBUG_FS - int debugfs_pool_counter; -#endif -}; - -/** - * struct coh901318_lli - linked list item for DMAC - * @control: control settings for DMAC - * @src_addr: transfer source address - * @dst_addr: transfer destination address - * @link_addr: physical address to next lli - * @virt_link_addr: virtual address of next lli (only used by pool_free) - * @phy_this: physical address of current lli (only used by pool_free) - */ -struct coh901318_lli { - u32 control; - dma_addr_t src_addr; - dma_addr_t dst_addr; - dma_addr_t link_addr; - - void *virt_link_addr; - dma_addr_t phy_this; -}; - -/** - * coh901318_pool_create() - Creates an dma pool for lli:s - * @pool: pool handle - * @dev: dma device - * @lli_nbr: number of lli:s in the pool - * @algin: address alignemtn of lli:s - * returns 0 on success otherwise none zero - */ -int coh901318_pool_create(struct coh901318_pool *pool, - struct device *dev, - size_t lli_nbr, size_t align); - -/** - * coh901318_pool_destroy() - Destroys the dma pool - * @pool: pool handle - * returns 0 on success otherwise none zero - */ -int coh901318_pool_destroy(struct coh901318_pool *pool); - -/** - * coh901318_lli_alloc() - Allocates a linked list - * - * @pool: pool handle - * @len: length to list - * return: none NULL if success otherwise NULL - */ -struct coh901318_lli * -coh901318_lli_alloc(struct coh901318_pool *pool, - unsigned int len); - -/** - * coh901318_lli_free() - Returns the linked list items to the pool - * @pool: pool handle - * @lli: reference to lli pointer to be freed - */ -void coh901318_lli_free(struct coh901318_pool *pool, - struct coh901318_lli **lli); - -/** - * coh901318_lli_fill_memcpy() - Prepares the lli:s for dma memcpy - * @pool: pool handle - * @lli: allocated lli - * @src: src address - * @size: transfer size - * @dst: destination address - * @ctrl_chained: ctrl for chained lli - * @ctrl_last: ctrl for the last lli - * returns number of CPU interrupts for the lli, negative on error. - */ -int -coh901318_lli_fill_memcpy(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t src, unsigned int size, - dma_addr_t dst, u32 ctrl_chained, u32 ctrl_last); - -/** - * coh901318_lli_fill_single() - Prepares the lli:s for dma single transfer - * @pool: pool handle - * @lli: allocated lli - * @buf: transfer buffer - * @size: transfer size - * @dev_addr: address of periphal - * @ctrl_chained: ctrl for chained lli - * @ctrl_last: ctrl for the last lli - * @dir: direction of transfer (to or from device) - * returns number of CPU interrupts for the lli, negative on error. - */ -int -coh901318_lli_fill_single(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t buf, unsigned int size, - dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl_last, - enum dma_transfer_direction dir); - -/** - * coh901318_lli_fill_single() - Prepares the lli:s for dma scatter list transfer - * @pool: pool handle - * @lli: allocated lli - * @sg: scatter gather list - * @nents: number of entries in sg - * @dev_addr: address of periphal - * @ctrl_chained: ctrl for chained lli - * @ctrl: ctrl of middle lli - * @ctrl_last: ctrl for the last lli - * @dir: direction of transfer (to or from device) - * @ctrl_irq_mask: ctrl mask for CPU interrupt - * returns number of CPU interrupts for the lli, negative on error. - */ -int -coh901318_lli_fill_sg(struct coh901318_pool *pool, - struct coh901318_lli *lli, - struct scatterlist *sg, unsigned int nents, - dma_addr_t dev_addr, u32 ctrl_chained, - u32 ctrl, u32 ctrl_last, - enum dma_transfer_direction dir, u32 ctrl_irq_mask); - -#endif /* COH901318_H */ diff --git a/drivers/dma/coh901318_lli.c b/drivers/dma/coh901318_lli.c deleted file mode 100644 index 6b6c2fd0865a..000000000000 --- a/drivers/dma/coh901318_lli.c +++ /dev/null @@ -1,313 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * driver/dma/coh901318_lli.c - * - * Copyright (C) 2007-2009 ST-Ericsson - * Support functions for handling lli for dma - * Author: Per Friden - */ - -#include -#include -#include -#include -#include - -#include "coh901318.h" - -#if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_U300_DEBUG)) -#define DEBUGFS_POOL_COUNTER_RESET(pool) (pool->debugfs_pool_counter = 0) -#define DEBUGFS_POOL_COUNTER_ADD(pool, add) (pool->debugfs_pool_counter += add) -#else -#define DEBUGFS_POOL_COUNTER_RESET(pool) -#define DEBUGFS_POOL_COUNTER_ADD(pool, add) -#endif - -static struct coh901318_lli * -coh901318_lli_next(struct coh901318_lli *data) -{ - if (data == NULL || data->link_addr == 0) - return NULL; - - return (struct coh901318_lli *) data->virt_link_addr; -} - -int coh901318_pool_create(struct coh901318_pool *pool, - struct device *dev, - size_t size, size_t align) -{ - spin_lock_init(&pool->lock); - pool->dev = dev; - pool->dmapool = dma_pool_create("lli_pool", dev, size, align, 0); - - DEBUGFS_POOL_COUNTER_RESET(pool); - return 0; -} - -int coh901318_pool_destroy(struct coh901318_pool *pool) -{ - - dma_pool_destroy(pool->dmapool); - return 0; -} - -struct coh901318_lli * -coh901318_lli_alloc(struct coh901318_pool *pool, unsigned int len) -{ - int i; - struct coh901318_lli *head; - struct coh901318_lli *lli; - struct coh901318_lli *lli_prev; - dma_addr_t phy; - - if (len == 0) - return NULL; - - spin_lock(&pool->lock); - - head = dma_pool_alloc(pool->dmapool, GFP_NOWAIT, &phy); - - if (head == NULL) - goto err; - - DEBUGFS_POOL_COUNTER_ADD(pool, 1); - - lli = head; - lli->phy_this = phy; - lli->link_addr = 0x00000000; - lli->virt_link_addr = NULL; - - for (i = 1; i < len; i++) { - lli_prev = lli; - - lli = dma_pool_alloc(pool->dmapool, GFP_NOWAIT, &phy); - - if (lli == NULL) - goto err_clean_up; - - DEBUGFS_POOL_COUNTER_ADD(pool, 1); - lli->phy_this = phy; - lli->link_addr = 0x00000000; - lli->virt_link_addr = NULL; - - lli_prev->link_addr = phy; - lli_prev->virt_link_addr = lli; - } - - spin_unlock(&pool->lock); - - return head; - - err: - spin_unlock(&pool->lock); - return NULL; - - err_clean_up: - lli_prev->link_addr = 0x00000000U; - spin_unlock(&pool->lock); - coh901318_lli_free(pool, &head); - return NULL; -} - -void coh901318_lli_free(struct coh901318_pool *pool, - struct coh901318_lli **lli) -{ - struct coh901318_lli *l; - struct coh901318_lli *next; - - if (lli == NULL) - return; - - l = *lli; - - if (l == NULL) - return; - - spin_lock(&pool->lock); - - while (l->link_addr) { - next = l->virt_link_addr; - dma_pool_free(pool->dmapool, l, l->phy_this); - DEBUGFS_POOL_COUNTER_ADD(pool, -1); - l = next; - } - dma_pool_free(pool->dmapool, l, l->phy_this); - DEBUGFS_POOL_COUNTER_ADD(pool, -1); - - spin_unlock(&pool->lock); - *lli = NULL; -} - -int -coh901318_lli_fill_memcpy(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t source, unsigned int size, - dma_addr_t destination, u32 ctrl_chained, - u32 ctrl_eom) -{ - int s = size; - dma_addr_t src = source; - dma_addr_t dst = destination; - - lli->src_addr = src; - lli->dst_addr = dst; - - while (lli->link_addr) { - lli->control = ctrl_chained | MAX_DMA_PACKET_SIZE; - lli->src_addr = src; - lli->dst_addr = dst; - - s -= MAX_DMA_PACKET_SIZE; - lli = coh901318_lli_next(lli); - - src += MAX_DMA_PACKET_SIZE; - dst += MAX_DMA_PACKET_SIZE; - } - - lli->control = ctrl_eom | s; - lli->src_addr = src; - lli->dst_addr = dst; - - return 0; -} - -int -coh901318_lli_fill_single(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t buf, unsigned int size, - dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl_eom, - enum dma_transfer_direction dir) -{ - int s = size; - dma_addr_t src; - dma_addr_t dst; - - - if (dir == DMA_MEM_TO_DEV) { - src = buf; - dst = dev_addr; - - } else if (dir == DMA_DEV_TO_MEM) { - - src = dev_addr; - dst = buf; - } else { - return -EINVAL; - } - - while (lli->link_addr) { - size_t block_size = MAX_DMA_PACKET_SIZE; - lli->control = ctrl_chained | MAX_DMA_PACKET_SIZE; - - /* If we are on the next-to-final block and there will - * be less than half a DMA packet left for the last - * block, then we want to make this block a little - * smaller to balance the sizes. This is meant to - * avoid too small transfers if the buffer size is - * (MAX_DMA_PACKET_SIZE*N + 1) */ - if (s < (MAX_DMA_PACKET_SIZE + MAX_DMA_PACKET_SIZE/2)) - block_size = MAX_DMA_PACKET_SIZE/2; - - s -= block_size; - lli->src_addr = src; - lli->dst_addr = dst; - - lli = coh901318_lli_next(lli); - - if (dir == DMA_MEM_TO_DEV) - src += block_size; - else if (dir == DMA_DEV_TO_MEM) - dst += block_size; - } - - lli->control = ctrl_eom | s; - lli->src_addr = src; - lli->dst_addr = dst; - - return 0; -} - -int -coh901318_lli_fill_sg(struct coh901318_pool *pool, - struct coh901318_lli *lli, - struct scatterlist *sgl, unsigned int nents, - dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl, - u32 ctrl_last, - enum dma_transfer_direction dir, u32 ctrl_irq_mask) -{ - int i; - struct scatterlist *sg; - u32 ctrl_sg; - dma_addr_t src = 0; - dma_addr_t dst = 0; - u32 bytes_to_transfer; - u32 elem_size; - - if (lli == NULL) - goto err; - - spin_lock(&pool->lock); - - if (dir == DMA_MEM_TO_DEV) - dst = dev_addr; - else if (dir == DMA_DEV_TO_MEM) - src = dev_addr; - else - goto err; - - for_each_sg(sgl, sg, nents, i) { - if (sg_is_chain(sg)) { - /* sg continues to the next sg-element don't - * send ctrl_finish until the last - * sg-element in the chain - */ - ctrl_sg = ctrl_chained; - } else if (i == nents - 1) - ctrl_sg = ctrl_last; - else - ctrl_sg = ctrl ? ctrl : ctrl_last; - - - if (dir == DMA_MEM_TO_DEV) - /* increment source address */ - src = sg_dma_address(sg); - else - /* increment destination address */ - dst = sg_dma_address(sg); - - bytes_to_transfer = sg_dma_len(sg); - - while (bytes_to_transfer) { - u32 val; - - if (bytes_to_transfer > MAX_DMA_PACKET_SIZE) { - elem_size = MAX_DMA_PACKET_SIZE; - val = ctrl_chained; - } else { - elem_size = bytes_to_transfer; - val = ctrl_sg; - } - - lli->control = val | elem_size; - lli->src_addr = src; - lli->dst_addr = dst; - - if (dir == DMA_DEV_TO_MEM) - dst += elem_size; - else - src += elem_size; - - BUG_ON(lli->link_addr & 3); - - bytes_to_transfer -= elem_size; - lli = coh901318_lli_next(lli); - } - - } - spin_unlock(&pool->lock); - - return 0; - err: - spin_unlock(&pool->lock); - return -EINVAL; -} diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index 612d353648cf..ebee94dbd630 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -1004,6 +1004,18 @@ static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = { JZ_SOC_DATA_BREAK_LINKS, }; +static const struct jz4780_dma_soc_data jz4760_dma_soc_data = { + .nb_channels = 5, + .transfer_ord_max = 6, + .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, +}; + +static const struct jz4780_dma_soc_data jz4760b_dma_soc_data = { + .nb_channels = 5, + .transfer_ord_max = 6, + .flags = JZ_SOC_DATA_PER_CHAN_PM, +}; + static const struct jz4780_dma_soc_data jz4770_dma_soc_data = { .nb_channels = 6, .transfer_ord_max = 6, @@ -1031,6 +1043,8 @@ static const struct jz4780_dma_soc_data x1830_dma_soc_data = { static const struct of_device_id jz4780_dma_dt_match[] = { { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, + { .compatible = "ingenic,jz4760-dma", .data = &jz4760_dma_soc_data }, + { .compatible = "ingenic,jz4760b-dma", .data = &jz4760b_dma_soc_data }, { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data }, { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data }, { .compatible = "ingenic,x1000-dma", .data = &x1000_dma_soc_data }, diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index e164f3295f5d..d9e4ac3edb4e 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -12,15 +12,20 @@ #include #include #include +#include #include #include #include +#include +#include #include #include #include +#include #include #include #include +#include #include #include "dw-axi-dmac.h" @@ -195,43 +200,56 @@ static inline const char *axi_chan_name(struct axi_dma_chan *chan) return dma_chan_name(&chan->vc.chan); } -static struct axi_dma_desc *axi_desc_get(struct axi_dma_chan *chan) +static struct axi_dma_desc *axi_desc_alloc(u32 num) { - struct dw_axi_dma *dw = chan->chip->dw; struct axi_dma_desc *desc; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->hw_desc = kcalloc(num, sizeof(*desc->hw_desc), GFP_NOWAIT); + if (!desc->hw_desc) { + kfree(desc); + return NULL; + } + + return desc; +} + +static struct axi_dma_lli *axi_desc_get(struct axi_dma_chan *chan, + dma_addr_t *addr) +{ + struct axi_dma_lli *lli; dma_addr_t phys; - desc = dma_pool_zalloc(dw->desc_pool, GFP_NOWAIT, &phys); - if (unlikely(!desc)) { + lli = dma_pool_zalloc(chan->desc_pool, GFP_NOWAIT, &phys); + if (unlikely(!lli)) { dev_err(chan2dev(chan), "%s: not enough descriptors available\n", axi_chan_name(chan)); return NULL; } atomic_inc(&chan->descs_allocated); - INIT_LIST_HEAD(&desc->xfer_list); - desc->vd.tx.phys = phys; - desc->chan = chan; + *addr = phys; - return desc; + return lli; } static void axi_desc_put(struct axi_dma_desc *desc) { struct axi_dma_chan *chan = desc->chan; - struct dw_axi_dma *dw = chan->chip->dw; - struct axi_dma_desc *child, *_next; - unsigned int descs_put = 0; + int count = atomic_read(&chan->descs_allocated); + struct axi_dma_hw_desc *hw_desc; + int descs_put; - list_for_each_entry_safe(child, _next, &desc->xfer_list, xfer_list) { - list_del(&child->xfer_list); - dma_pool_free(dw->desc_pool, child, child->vd.tx.phys); - descs_put++; + for (descs_put = 0; descs_put < count; descs_put++) { + hw_desc = &desc->hw_desc[descs_put]; + dma_pool_free(chan->desc_pool, hw_desc->lli, hw_desc->llp); } - dma_pool_free(dw->desc_pool, desc, desc->vd.tx.phys); - descs_put++; - + kfree(desc->hw_desc); + kfree(desc); atomic_sub(descs_put, &chan->descs_allocated); dev_vdbg(chan2dev(chan), "%s: %d descs put, %d still allocated\n", axi_chan_name(chan), descs_put, @@ -248,19 +266,41 @@ dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); - enum dma_status ret; + struct virt_dma_desc *vdesc; + enum dma_status status; + u32 completed_length; + unsigned long flags; + u32 completed_blocks; + size_t bytes = 0; + u32 length; + u32 len; - ret = dma_cookie_status(dchan, cookie, txstate); + status = dma_cookie_status(dchan, cookie, txstate); + if (status == DMA_COMPLETE || !txstate) + return status; - if (chan->is_paused && ret == DMA_IN_PROGRESS) - ret = DMA_PAUSED; + spin_lock_irqsave(&chan->vc.lock, flags); - return ret; + vdesc = vchan_find_desc(&chan->vc, cookie); + if (vdesc) { + length = vd_to_axi_desc(vdesc)->length; + completed_blocks = vd_to_axi_desc(vdesc)->completed_blocks; + len = vd_to_axi_desc(vdesc)->hw_desc[0].len; + completed_length = completed_blocks * len; + bytes = length - completed_length; + } else { + bytes = vd_to_axi_desc(vdesc)->length; + } + + spin_unlock_irqrestore(&chan->vc.lock, flags); + dma_set_residue(txstate, bytes); + + return status; } -static void write_desc_llp(struct axi_dma_desc *desc, dma_addr_t adr) +static void write_desc_llp(struct axi_dma_hw_desc *desc, dma_addr_t adr) { - desc->lli.llp = cpu_to_le64(adr); + desc->lli->llp = cpu_to_le64(adr); } static void write_chan_llp(struct axi_dma_chan *chan, dma_addr_t adr) @@ -268,6 +308,29 @@ static void write_chan_llp(struct axi_dma_chan *chan, dma_addr_t adr) axi_chan_iowrite64(chan, CH_LLP, adr); } +static void dw_axi_dma_set_byte_halfword(struct axi_dma_chan *chan, bool set) +{ + u32 offset = DMAC_APB_BYTE_WR_CH_EN; + u32 reg_width, val; + + if (!chan->chip->apb_regs) { + dev_dbg(chan->chip->dev, "apb_regs not initialized\n"); + return; + } + + reg_width = __ffs(chan->config.dst_addr_width); + if (reg_width == DWAXIDMAC_TRANS_WIDTH_16) + offset = DMAC_APB_HALFWORD_WR_CH_EN; + + val = ioread32(chan->chip->apb_regs + offset); + + if (set) + val |= BIT(chan->id); + else + val &= ~BIT(chan->id); + + iowrite32(val, chan->chip->apb_regs + offset); +} /* Called in chan locked context */ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, struct axi_dma_desc *first) @@ -293,9 +356,26 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, priority << CH_CFG_H_PRIORITY_POS | DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_DST_POS | DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_SRC_POS); + switch (chan->direction) { + case DMA_MEM_TO_DEV: + dw_axi_dma_set_byte_halfword(chan, true); + reg |= (chan->config.device_fc ? + DWAXIDMAC_TT_FC_MEM_TO_PER_DST : + DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC) + << CH_CFG_H_TT_FC_POS; + break; + case DMA_DEV_TO_MEM: + reg |= (chan->config.device_fc ? + DWAXIDMAC_TT_FC_PER_TO_MEM_SRC : + DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC) + << CH_CFG_H_TT_FC_POS; + break; + default: + break; + } axi_chan_iowrite32(chan, CH_CFG_H, reg); - write_chan_llp(chan, first->vd.tx.phys | lms); + write_chan_llp(chan, first->hw_desc[0].llp | lms); irq_mask = DWAXIDMAC_IRQ_DMA_TRF | DWAXIDMAC_IRQ_ALL_ERR; axi_chan_irq_sig_set(chan, irq_mask); @@ -333,6 +413,13 @@ static void dma_chan_issue_pending(struct dma_chan *dchan) spin_unlock_irqrestore(&chan->vc.lock, flags); } +static void dw_axi_dma_synchronize(struct dma_chan *dchan) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + + vchan_synchronize(&chan->vc); +} + static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); @@ -344,6 +431,15 @@ static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) return -EBUSY; } + /* LLI address must be aligned to a 64-byte boundary */ + chan->desc_pool = dma_pool_create(dev_name(chan2dev(chan)), + chan->chip->dev, + sizeof(struct axi_dma_lli), + 64, 0); + if (!chan->desc_pool) { + dev_err(chan2dev(chan), "No memory for descriptors\n"); + return -ENOMEM; + } dev_vdbg(dchan2dev(dchan), "%s: allocating\n", axi_chan_name(chan)); pm_runtime_get(chan->chip->dev); @@ -365,6 +461,8 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan) vchan_free_chan_resources(&chan->vc); + dma_pool_destroy(chan->desc_pool); + chan->desc_pool = NULL; dev_vdbg(dchan2dev(dchan), "%s: free resources, descriptor still allocated: %u\n", axi_chan_name(chan), atomic_read(&chan->descs_allocated)); @@ -372,73 +470,398 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan) pm_runtime_put(chan->chip->dev); } +static void dw_axi_dma_set_hw_channel(struct axi_dma_chip *chip, + u32 handshake_num, bool set) +{ + unsigned long start = 0; + unsigned long reg_value; + unsigned long reg_mask; + unsigned long reg_set; + unsigned long mask; + unsigned long val; + + if (!chip->apb_regs) { + dev_dbg(chip->dev, "apb_regs not initialized\n"); + return; + } + + /* + * An unused DMA channel has a default value of 0x3F. + * Lock the DMA channel by assign a handshake number to the channel. + * Unlock the DMA channel by assign 0x3F to the channel. + */ + if (set) { + reg_set = UNUSED_CHANNEL; + val = handshake_num; + } else { + reg_set = handshake_num; + val = UNUSED_CHANNEL; + } + + reg_value = lo_hi_readq(chip->apb_regs + DMAC_APB_HW_HS_SEL_0); + + for_each_set_clump8(start, reg_mask, ®_value, 64) { + if (reg_mask == reg_set) { + mask = GENMASK_ULL(start + 7, start); + reg_value &= ~mask; + reg_value |= rol64(val, start); + lo_hi_writeq(reg_value, + chip->apb_regs + DMAC_APB_HW_HS_SEL_0); + break; + } + } +} + /* * If DW_axi_dmac sees CHx_CTL.ShadowReg_Or_LLI_Last bit of the fetched LLI * as 1, it understands that the current block is the final block in the * transfer and completes the DMA transfer operation at the end of current * block transfer. */ -static void set_desc_last(struct axi_dma_desc *desc) +static void set_desc_last(struct axi_dma_hw_desc *desc) { u32 val; - val = le32_to_cpu(desc->lli.ctl_hi); + val = le32_to_cpu(desc->lli->ctl_hi); val |= CH_CTL_H_LLI_LAST; - desc->lli.ctl_hi = cpu_to_le32(val); + desc->lli->ctl_hi = cpu_to_le32(val); } -static void write_desc_sar(struct axi_dma_desc *desc, dma_addr_t adr) +static void write_desc_sar(struct axi_dma_hw_desc *desc, dma_addr_t adr) { - desc->lli.sar = cpu_to_le64(adr); + desc->lli->sar = cpu_to_le64(adr); } -static void write_desc_dar(struct axi_dma_desc *desc, dma_addr_t adr) +static void write_desc_dar(struct axi_dma_hw_desc *desc, dma_addr_t adr) { - desc->lli.dar = cpu_to_le64(adr); + desc->lli->dar = cpu_to_le64(adr); } -static void set_desc_src_master(struct axi_dma_desc *desc) +static void set_desc_src_master(struct axi_dma_hw_desc *desc) { u32 val; /* Select AXI0 for source master */ - val = le32_to_cpu(desc->lli.ctl_lo); + val = le32_to_cpu(desc->lli->ctl_lo); val &= ~CH_CTL_L_SRC_MAST; - desc->lli.ctl_lo = cpu_to_le32(val); + desc->lli->ctl_lo = cpu_to_le32(val); } -static void set_desc_dest_master(struct axi_dma_desc *desc) +static void set_desc_dest_master(struct axi_dma_hw_desc *hw_desc, + struct axi_dma_desc *desc) { u32 val; /* Select AXI1 for source master if available */ - val = le32_to_cpu(desc->lli.ctl_lo); + val = le32_to_cpu(hw_desc->lli->ctl_lo); if (desc->chan->chip->dw->hdata->nr_masters > 1) val |= CH_CTL_L_DST_MAST; else val &= ~CH_CTL_L_DST_MAST; - desc->lli.ctl_lo = cpu_to_le32(val); + hw_desc->lli->ctl_lo = cpu_to_le32(val); +} + +static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan, + struct axi_dma_hw_desc *hw_desc, + dma_addr_t mem_addr, size_t len) +{ + unsigned int data_width = BIT(chan->chip->dw->hdata->m_data_width); + unsigned int reg_width; + unsigned int mem_width; + dma_addr_t device_addr; + size_t axi_block_ts; + size_t block_ts; + u32 ctllo, ctlhi; + u32 burst_len; + + axi_block_ts = chan->chip->dw->hdata->block_size[chan->id]; + + mem_width = __ffs(data_width | mem_addr | len); + if (mem_width > DWAXIDMAC_TRANS_WIDTH_32) + mem_width = DWAXIDMAC_TRANS_WIDTH_32; + + if (!IS_ALIGNED(mem_addr, 4)) { + dev_err(chan->chip->dev, "invalid buffer alignment\n"); + return -EINVAL; + } + + switch (chan->direction) { + case DMA_MEM_TO_DEV: + reg_width = __ffs(chan->config.dst_addr_width); + device_addr = chan->config.dst_addr; + ctllo = reg_width << CH_CTL_L_DST_WIDTH_POS | + mem_width << CH_CTL_L_SRC_WIDTH_POS | + DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_DST_INC_POS | + DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS; + block_ts = len >> mem_width; + break; + case DMA_DEV_TO_MEM: + reg_width = __ffs(chan->config.src_addr_width); + device_addr = chan->config.src_addr; + ctllo = reg_width << CH_CTL_L_SRC_WIDTH_POS | + mem_width << CH_CTL_L_DST_WIDTH_POS | + DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS | + DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_SRC_INC_POS; + block_ts = len >> reg_width; + break; + default: + return -EINVAL; + } + + if (block_ts > axi_block_ts) + return -EINVAL; + + hw_desc->lli = axi_desc_get(chan, &hw_desc->llp); + if (unlikely(!hw_desc->lli)) + return -ENOMEM; + + ctlhi = CH_CTL_H_LLI_VALID; + + if (chan->chip->dw->hdata->restrict_axi_burst_len) { + burst_len = chan->chip->dw->hdata->axi_rw_burst_len; + ctlhi |= CH_CTL_H_ARLEN_EN | CH_CTL_H_AWLEN_EN | + burst_len << CH_CTL_H_ARLEN_POS | + burst_len << CH_CTL_H_AWLEN_POS; + } + + hw_desc->lli->ctl_hi = cpu_to_le32(ctlhi); + + if (chan->direction == DMA_MEM_TO_DEV) { + write_desc_sar(hw_desc, mem_addr); + write_desc_dar(hw_desc, device_addr); + } else { + write_desc_sar(hw_desc, device_addr); + write_desc_dar(hw_desc, mem_addr); + } + + hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1); + + ctllo |= DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS | + DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS; + hw_desc->lli->ctl_lo = cpu_to_le32(ctllo); + + set_desc_src_master(hw_desc); + + hw_desc->len = len; + return 0; +} + +static size_t calculate_block_len(struct axi_dma_chan *chan, + dma_addr_t dma_addr, size_t buf_len, + enum dma_transfer_direction direction) +{ + u32 data_width, reg_width, mem_width; + size_t axi_block_ts, block_len; + + axi_block_ts = chan->chip->dw->hdata->block_size[chan->id]; + + switch (direction) { + case DMA_MEM_TO_DEV: + data_width = BIT(chan->chip->dw->hdata->m_data_width); + mem_width = __ffs(data_width | dma_addr | buf_len); + if (mem_width > DWAXIDMAC_TRANS_WIDTH_32) + mem_width = DWAXIDMAC_TRANS_WIDTH_32; + + block_len = axi_block_ts << mem_width; + break; + case DMA_DEV_TO_MEM: + reg_width = __ffs(chan->config.src_addr_width); + block_len = axi_block_ts << reg_width; + break; + default: + block_len = 0; + } + + return block_len; +} + +static struct dma_async_tx_descriptor * +dw_axi_dma_chan_prep_cyclic(struct dma_chan *dchan, dma_addr_t dma_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction direction, + unsigned long flags) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + struct axi_dma_hw_desc *hw_desc = NULL; + struct axi_dma_desc *desc = NULL; + dma_addr_t src_addr = dma_addr; + u32 num_periods, num_segments; + size_t axi_block_len; + u32 total_segments; + u32 segment_len; + unsigned int i; + int status; + u64 llp = 0; + u8 lms = 0; /* Select AXI0 master for LLI fetching */ + + num_periods = buf_len / period_len; + + axi_block_len = calculate_block_len(chan, dma_addr, buf_len, direction); + if (axi_block_len == 0) + return NULL; + + num_segments = DIV_ROUND_UP(period_len, axi_block_len); + segment_len = DIV_ROUND_UP(period_len, num_segments); + + total_segments = num_periods * num_segments; + + desc = axi_desc_alloc(total_segments); + if (unlikely(!desc)) + goto err_desc_get; + + chan->direction = direction; + desc->chan = chan; + chan->cyclic = true; + desc->length = 0; + desc->period_len = period_len; + + for (i = 0; i < total_segments; i++) { + hw_desc = &desc->hw_desc[i]; + + status = dw_axi_dma_set_hw_desc(chan, hw_desc, src_addr, + segment_len); + if (status < 0) + goto err_desc_get; + + desc->length += hw_desc->len; + /* Set end-of-link to the linked descriptor, so that cyclic + * callback function can be triggered during interrupt. + */ + set_desc_last(hw_desc); + + src_addr += segment_len; + } + + llp = desc->hw_desc[0].llp; + + /* Managed transfer list */ + do { + hw_desc = &desc->hw_desc[--total_segments]; + write_desc_llp(hw_desc, llp | lms); + llp = hw_desc->llp; + } while (total_segments); + + dw_axi_dma_set_hw_channel(chan->chip, chan->hw_handshake_num, true); + + return vchan_tx_prep(&chan->vc, &desc->vd, flags); + +err_desc_get: + if (desc) + axi_desc_put(desc); + + return NULL; +} + +static struct dma_async_tx_descriptor * +dw_axi_dma_chan_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + struct axi_dma_hw_desc *hw_desc = NULL; + struct axi_dma_desc *desc = NULL; + u32 num_segments, segment_len; + unsigned int loop = 0; + struct scatterlist *sg; + size_t axi_block_len; + u32 len, num_sgs = 0; + unsigned int i; + dma_addr_t mem; + int status; + u64 llp = 0; + u8 lms = 0; /* Select AXI0 master for LLI fetching */ + + if (unlikely(!is_slave_direction(direction) || !sg_len)) + return NULL; + + mem = sg_dma_address(sgl); + len = sg_dma_len(sgl); + + axi_block_len = calculate_block_len(chan, mem, len, direction); + if (axi_block_len == 0) + return NULL; + + for_each_sg(sgl, sg, sg_len, i) + num_sgs += DIV_ROUND_UP(sg_dma_len(sg), axi_block_len); + + desc = axi_desc_alloc(num_sgs); + if (unlikely(!desc)) + goto err_desc_get; + + desc->chan = chan; + desc->length = 0; + chan->direction = direction; + + for_each_sg(sgl, sg, sg_len, i) { + mem = sg_dma_address(sg); + len = sg_dma_len(sg); + num_segments = DIV_ROUND_UP(sg_dma_len(sg), axi_block_len); + segment_len = DIV_ROUND_UP(sg_dma_len(sg), num_segments); + + do { + hw_desc = &desc->hw_desc[loop++]; + status = dw_axi_dma_set_hw_desc(chan, hw_desc, mem, segment_len); + if (status < 0) + goto err_desc_get; + + desc->length += hw_desc->len; + len -= segment_len; + mem += segment_len; + } while (len >= segment_len); + } + + /* Set end-of-link to the last link descriptor of list */ + set_desc_last(&desc->hw_desc[num_sgs - 1]); + + /* Managed transfer list */ + do { + hw_desc = &desc->hw_desc[--num_sgs]; + write_desc_llp(hw_desc, llp | lms); + llp = hw_desc->llp; + } while (num_sgs); + + dw_axi_dma_set_hw_channel(chan->chip, chan->hw_handshake_num, true); + + return vchan_tx_prep(&chan->vc, &desc->vd, flags); + +err_desc_get: + if (desc) + axi_desc_put(desc); + + return NULL; } static struct dma_async_tx_descriptor * dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, dma_addr_t src_adr, size_t len, unsigned long flags) { - struct axi_dma_desc *first = NULL, *desc = NULL, *prev = NULL; struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); size_t block_ts, max_block_ts, xfer_len; - u32 xfer_width, reg; + struct axi_dma_hw_desc *hw_desc = NULL; + struct axi_dma_desc *desc = NULL; + u32 xfer_width, reg, num; + u64 llp = 0; u8 lms = 0; /* Select AXI0 master for LLI fetching */ dev_dbg(chan2dev(chan), "%s: memcpy: src: %pad dst: %pad length: %zd flags: %#lx", axi_chan_name(chan), &src_adr, &dst_adr, len, flags); max_block_ts = chan->chip->dw->hdata->block_size[chan->id]; + xfer_width = axi_chan_get_xfer_width(chan, src_adr, dst_adr, len); + num = DIV_ROUND_UP(len, max_block_ts << xfer_width); + desc = axi_desc_alloc(num); + if (unlikely(!desc)) + goto err_desc_get; + desc->chan = chan; + num = 0; + desc->length = 0; while (len) { xfer_len = len; + hw_desc = &desc->hw_desc[num]; /* * Take care for the alignment. * Actually source and destination widths can be different, but @@ -457,13 +880,13 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, xfer_len = max_block_ts << xfer_width; } - desc = axi_desc_get(chan); - if (unlikely(!desc)) + hw_desc->lli = axi_desc_get(chan, &hw_desc->llp); + if (unlikely(!hw_desc->lli)) goto err_desc_get; - write_desc_sar(desc, src_adr); - write_desc_dar(desc, dst_adr); - desc->lli.block_ts_lo = cpu_to_le32(block_ts - 1); + write_desc_sar(hw_desc, src_adr); + write_desc_dar(hw_desc, dst_adr); + hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1); reg = CH_CTL_H_LLI_VALID; if (chan->chip->dw->hdata->restrict_axi_burst_len) { @@ -474,7 +897,7 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, CH_CTL_H_AWLEN_EN | burst_len << CH_CTL_H_AWLEN_POS); } - desc->lli.ctl_hi = cpu_to_le32(reg); + hw_desc->lli->ctl_hi = cpu_to_le32(reg); reg = (DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS | DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS | @@ -482,62 +905,68 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, xfer_width << CH_CTL_L_SRC_WIDTH_POS | DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS | DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS); - desc->lli.ctl_lo = cpu_to_le32(reg); + hw_desc->lli->ctl_lo = cpu_to_le32(reg); - set_desc_src_master(desc); - set_desc_dest_master(desc); - - /* Manage transfer list (xfer_list) */ - if (!first) { - first = desc; - } else { - list_add_tail(&desc->xfer_list, &first->xfer_list); - write_desc_llp(prev, desc->vd.tx.phys | lms); - } - prev = desc; + set_desc_src_master(hw_desc); + set_desc_dest_master(hw_desc, desc); + hw_desc->len = xfer_len; + desc->length += hw_desc->len; /* update the length and addresses for the next loop cycle */ len -= xfer_len; dst_adr += xfer_len; src_adr += xfer_len; + num++; } - /* Total len of src/dest sg == 0, so no descriptor were allocated */ - if (unlikely(!first)) - return NULL; - /* Set end-of-link to the last link descriptor of list */ - set_desc_last(desc); + set_desc_last(&desc->hw_desc[num - 1]); + /* Managed transfer list */ + do { + hw_desc = &desc->hw_desc[--num]; + write_desc_llp(hw_desc, llp | lms); + llp = hw_desc->llp; + } while (num); - return vchan_tx_prep(&chan->vc, &first->vd, flags); + return vchan_tx_prep(&chan->vc, &desc->vd, flags); err_desc_get: - if (first) - axi_desc_put(first); + if (desc) + axi_desc_put(desc); return NULL; } +static int dw_axi_dma_chan_slave_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + + memcpy(&chan->config, config, sizeof(*config)); + + return 0; +} + static void axi_chan_dump_lli(struct axi_dma_chan *chan, - struct axi_dma_desc *desc) + struct axi_dma_hw_desc *desc) { dev_err(dchan2dev(&chan->vc.chan), "SAR: 0x%llx DAR: 0x%llx LLP: 0x%llx BTS 0x%x CTL: 0x%x:%08x", - le64_to_cpu(desc->lli.sar), - le64_to_cpu(desc->lli.dar), - le64_to_cpu(desc->lli.llp), - le32_to_cpu(desc->lli.block_ts_lo), - le32_to_cpu(desc->lli.ctl_hi), - le32_to_cpu(desc->lli.ctl_lo)); + le64_to_cpu(desc->lli->sar), + le64_to_cpu(desc->lli->dar), + le64_to_cpu(desc->lli->llp), + le32_to_cpu(desc->lli->block_ts_lo), + le32_to_cpu(desc->lli->ctl_hi), + le32_to_cpu(desc->lli->ctl_lo)); } static void axi_chan_list_dump_lli(struct axi_dma_chan *chan, struct axi_dma_desc *desc_head) { - struct axi_dma_desc *desc; + int count = atomic_read(&chan->descs_allocated); + int i; - axi_chan_dump_lli(chan, desc_head); - list_for_each_entry(desc, &desc_head->xfer_list, xfer_list) - axi_chan_dump_lli(chan, desc); + for (i = 0; i < count; i++) + axi_chan_dump_lli(chan, &desc_head->hw_desc[i]); } static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) @@ -570,8 +999,13 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) { + int count = atomic_read(&chan->descs_allocated); + struct axi_dma_hw_desc *hw_desc; + struct axi_dma_desc *desc; struct virt_dma_desc *vd; unsigned long flags; + u64 llp; + int i; spin_lock_irqsave(&chan->vc.lock, flags); if (unlikely(axi_chan_is_hw_enable(chan))) { @@ -582,12 +1016,34 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) /* The completed descriptor currently is in the head of vc list */ vd = vchan_next_desc(&chan->vc); - /* Remove the completed descriptor from issued list before completing */ - list_del(&vd->node); - vchan_cookie_complete(vd); - /* Submit queued descriptors after processing the completed ones */ - axi_chan_start_first_queued(chan); + if (chan->cyclic) { + desc = vd_to_axi_desc(vd); + if (desc) { + llp = lo_hi_readq(chan->chan_regs + CH_LLP); + for (i = 0; i < count; i++) { + hw_desc = &desc->hw_desc[i]; + if (hw_desc->llp == llp) { + axi_chan_irq_clear(chan, hw_desc->lli->status_lo); + hw_desc->lli->ctl_hi |= CH_CTL_H_LLI_VALID; + desc->completed_blocks = i; + + if (((hw_desc->len * (i + 1)) % desc->period_len) == 0) + vchan_cyclic_callback(vd); + break; + } + } + + axi_chan_enable(chan); + } + } else { + /* Remove the completed descriptor from issued list before completing */ + list_del(&vd->node); + vchan_cookie_complete(vd); + + /* Submit queued descriptors after processing the completed ones */ + axi_chan_start_first_queued(chan); + } spin_unlock_irqrestore(&chan->vc.lock, flags); } @@ -627,15 +1083,31 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) static int dma_chan_terminate_all(struct dma_chan *dchan) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + u32 chan_active = BIT(chan->id) << DMAC_CHAN_EN_SHIFT; unsigned long flags; + u32 val; + int ret; LIST_HEAD(head); - spin_lock_irqsave(&chan->vc.lock, flags); - axi_chan_disable(chan); + ret = readl_poll_timeout_atomic(chan->chip->regs + DMAC_CHEN, val, + !(val & chan_active), 1000, 10000); + if (ret == -ETIMEDOUT) + dev_warn(dchan2dev(dchan), + "%s failed to stop\n", axi_chan_name(chan)); + + if (chan->direction != DMA_MEM_TO_MEM) + dw_axi_dma_set_hw_channel(chan->chip, + chan->hw_handshake_num, false); + if (chan->direction == DMA_MEM_TO_DEV) + dw_axi_dma_set_byte_halfword(chan, false); + + spin_lock_irqsave(&chan->vc.lock, flags); + vchan_get_all_descriptors(&chan->vc, &head); + chan->cyclic = false; spin_unlock_irqrestore(&chan->vc.lock, flags); vchan_dma_desc_free_list(&chan->vc, &head); @@ -746,6 +1218,22 @@ static int __maybe_unused axi_dma_runtime_resume(struct device *dev) return axi_dma_resume(chip); } +static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct dw_axi_dma *dw = ofdma->of_dma_data; + struct axi_dma_chan *chan; + struct dma_chan *dchan; + + dchan = dma_get_any_slave_channel(&dw->dma); + if (!dchan) + return NULL; + + chan = dchan_to_axi_dma_chan(dchan); + chan->hw_handshake_num = dma_spec->args[0]; + return dchan; +} + static int parse_device_properties(struct axi_dma_chip *chip) { struct device *dev = chip->dev; @@ -816,6 +1304,7 @@ static int parse_device_properties(struct axi_dma_chip *chip) static int dw_probe(struct platform_device *pdev) { + struct device_node *node = pdev->dev.of_node; struct axi_dma_chip *chip; struct resource *mem; struct dw_axi_dma *dw; @@ -848,6 +1337,12 @@ static int dw_probe(struct platform_device *pdev) if (IS_ERR(chip->regs)) return PTR_ERR(chip->regs); + if (of_device_is_compatible(node, "intel,kmb-axi-dma")) { + chip->apb_regs = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(chip->apb_regs)) + return PTR_ERR(chip->apb_regs); + } + chip->core_clk = devm_clk_get(chip->dev, "core-clk"); if (IS_ERR(chip->core_clk)) return PTR_ERR(chip->core_clk); @@ -870,13 +1365,6 @@ static int dw_probe(struct platform_device *pdev) if (ret) return ret; - /* Lli address must be aligned to a 64-byte boundary */ - dw->desc_pool = dmam_pool_create(KBUILD_MODNAME, chip->dev, - sizeof(struct axi_dma_desc), 64, 0); - if (!dw->desc_pool) { - dev_err(chip->dev, "No memory for descriptors dma pool\n"); - return -ENOMEM; - } INIT_LIST_HEAD(&dw->dma.channels); for (i = 0; i < hdata->nr_channels; i++) { @@ -893,13 +1381,16 @@ static int dw_probe(struct platform_device *pdev) /* Set capabilities */ dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask); + dma_cap_set(DMA_SLAVE, dw->dma.cap_mask); + dma_cap_set(DMA_CYCLIC, dw->dma.cap_mask); /* DMA capabilities */ dw->dma.chancnt = hdata->nr_channels; dw->dma.src_addr_widths = AXI_DMA_BUSWIDTHS; dw->dma.dst_addr_widths = AXI_DMA_BUSWIDTHS; dw->dma.directions = BIT(DMA_MEM_TO_MEM); - dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dw->dma.directions |= BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); + dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; dw->dma.dev = chip->dev; dw->dma.device_tx_status = dma_chan_tx_status; @@ -912,7 +1403,18 @@ static int dw_probe(struct platform_device *pdev) dw->dma.device_free_chan_resources = dma_chan_free_chan_resources; dw->dma.device_prep_dma_memcpy = dma_chan_prep_dma_memcpy; + dw->dma.device_synchronize = dw_axi_dma_synchronize; + dw->dma.device_config = dw_axi_dma_chan_slave_config; + dw->dma.device_prep_slave_sg = dw_axi_dma_chan_prep_slave_sg; + dw->dma.device_prep_dma_cyclic = dw_axi_dma_chan_prep_cyclic; + /* + * Synopsis DesignWare AxiDMA datasheet mentioned Maximum + * supported blocks is 1024. Device register width is 4 bytes. + * Therefore, set constraint to 1024 * 4. + */ + dw->dma.dev->dma_parms = &dw->dma_parms; + dma_set_max_seg_size(&pdev->dev, MAX_BLOCK_SIZE); platform_set_drvdata(pdev, chip); pm_runtime_enable(chip->dev); @@ -935,6 +1437,13 @@ static int dw_probe(struct platform_device *pdev) if (ret) goto err_pm_disable; + /* Register with OF helpers for DMA lookups */ + ret = of_dma_controller_register(pdev->dev.of_node, + dw_axi_dma_of_xlate, dw); + if (ret < 0) + dev_warn(&pdev->dev, + "Failed to register OF DMA controller, fallback to MEM_TO_MEM mode\n"); + dev_info(chip->dev, "DesignWare AXI DMA Controller, %d channels\n", dw->hdata->nr_channels); @@ -968,6 +1477,8 @@ static int dw_remove(struct platform_device *pdev) devm_free_irq(chip->dev, chip->irq, chip); + of_dma_controller_free(chip->dev->of_node); + list_for_each_entry_safe(chan, _chan, &dw->dma.channels, vc.chan.device_node) { list_del(&chan->vc.chan.device_node); @@ -983,6 +1494,7 @@ static const struct dev_pm_ops dw_axi_dma_pm_ops = { static const struct of_device_id dw_dma_of_id_table[] = { { .compatible = "snps,axi-dma-1.01a" }, + { .compatible = "intel,kmb-axi-dma" }, {} }; MODULE_DEVICE_TABLE(of, dw_dma_of_id_table); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index 18b6014cf9b4..b69897887c76 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -37,10 +37,16 @@ struct axi_dma_chan { struct axi_dma_chip *chip; void __iomem *chan_regs; u8 id; + u8 hw_handshake_num; atomic_t descs_allocated; + struct dma_pool *desc_pool; struct virt_dma_chan vc; + struct axi_dma_desc *desc; + struct dma_slave_config config; + enum dma_transfer_direction direction; + bool cyclic; /* these other elements are all protected by vc.lock */ bool is_paused; }; @@ -48,7 +54,7 @@ struct axi_dma_chan { struct dw_axi_dma { struct dma_device dma; struct dw_axi_dma_hcfg *hdata; - struct dma_pool *desc_pool; + struct device_dma_parameters dma_parms; /* channels */ struct axi_dma_chan *chan; @@ -58,6 +64,7 @@ struct axi_dma_chip { struct device *dev; int irq; void __iomem *regs; + void __iomem *apb_regs; struct clk *core_clk; struct clk *cfgr_clk; struct dw_axi_dma *dw; @@ -80,12 +87,20 @@ struct __packed axi_dma_lli { __le32 reserved_hi; }; +struct axi_dma_hw_desc { + struct axi_dma_lli *lli; + dma_addr_t llp; + u32 len; +}; + struct axi_dma_desc { - struct axi_dma_lli lli; + struct axi_dma_hw_desc *hw_desc; struct virt_dma_desc vd; struct axi_dma_chan *chan; - struct list_head xfer_list; + u32 completed_blocks; + u32 length; + u32 period_len; }; static inline struct device *dchan2dev(struct dma_chan *dchan) @@ -157,6 +172,19 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan) #define CH_INTSIGNAL_ENA 0x090 /* R/W Chan Interrupt Signal Enable */ #define CH_INTCLEAR 0x098 /* W Chan Interrupt Clear */ +/* These Apb registers are used by Intel KeemBay SoC */ +#define DMAC_APB_CFG 0x000 /* DMAC Apb Configuration Register */ +#define DMAC_APB_STAT 0x004 /* DMAC Apb Status Register */ +#define DMAC_APB_DEBUG_STAT_0 0x008 /* DMAC Apb Debug Status Register 0 */ +#define DMAC_APB_DEBUG_STAT_1 0x00C /* DMAC Apb Debug Status Register 1 */ +#define DMAC_APB_HW_HS_SEL_0 0x010 /* DMAC Apb HW HS register 0 */ +#define DMAC_APB_HW_HS_SEL_1 0x014 /* DMAC Apb HW HS register 1 */ +#define DMAC_APB_LPI 0x018 /* DMAC Apb Low Power Interface Reg */ +#define DMAC_APB_BYTE_WR_CH_EN 0x01C /* DMAC Apb Byte Write Enable */ +#define DMAC_APB_HALFWORD_WR_CH_EN 0x020 /* DMAC Halfword write enables */ + +#define UNUSED_CHANNEL 0x3F /* Set unused DMA channel to 0x3F */ +#define MAX_BLOCK_SIZE 0x1000 /* 1024 blocks * 4 bytes data width */ /* DMAC_CFG */ #define DMAC_EN_POS 0 diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index 0feb323bae1e..f8459cc5315d 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -1214,6 +1214,7 @@ static int fsldma_of_probe(struct platform_device *op) { struct fsldma_device *fdev; struct device_node *child; + unsigned int i; int err; fdev = kzalloc(sizeof(*fdev), GFP_KERNEL); @@ -1292,6 +1293,10 @@ static int fsldma_of_probe(struct platform_device *op) return 0; out_free_fdev: + for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { + if (fdev->chan[i]) + fsl_dma_chan_remove(fdev->chan[i]); + } irq_dispose_mapping(fdev->irq); iounmap(fdev->regs); out_free: @@ -1314,6 +1319,7 @@ static int fsldma_of_remove(struct platform_device *op) if (fdev->chan[i]) fsl_dma_chan_remove(fdev->chan[i]); } + irq_dispose_mapping(fdev->irq); iounmap(fdev->regs); kfree(fdev); diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c index 07cc7320a614..9045a6f7f589 100644 --- a/drivers/dma/hsu/pci.c +++ b/drivers/dma/hsu/pci.c @@ -26,22 +26,12 @@ static irqreturn_t hsu_pci_irq(int irq, void *dev) { struct hsu_dma_chip *chip = dev; - struct pci_dev *pdev = to_pci_dev(chip->dev); u32 dmaisr; u32 status; unsigned short i; int ret = 0; int err; - /* - * On Intel Tangier B0 and Anniedale the interrupt line, disregarding - * to have different numbers, is shared between HSU DMA and UART IPs. - * Thus on such SoCs we are expecting that IRQ handler is called in - * UART driver only. - */ - if (pdev->device == PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA) - return IRQ_HANDLED; - dmaisr = readl(chip->regs + HSU_PCI_DMAISR); for (i = 0; i < chip->hsu->nr_channels; i++) { if (dmaisr & 0x1) { @@ -105,6 +95,17 @@ static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto err_register_irq; + /* + * On Intel Tangier B0 and Anniedale the interrupt line, disregarding + * to have different numbers, is shared between HSU DMA and UART IPs. + * Thus on such SoCs we are expecting that IRQ handler is called in + * UART driver only. Instead of handling the spurious interrupt + * from HSU DMA here and waste CPU time and delay HSU UART interrupt + * handling, disable the interrupt entirely. + */ + if (pdev->device == PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA) + disable_irq_nosync(chip->irq); + pci_set_drvdata(pdev, chip); return 0; diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 71fd6e4c42cd..a15e50126434 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -165,6 +165,7 @@ int idxd_register_dma_device(struct idxd_device *idxd) INIT_LIST_HEAD(&dma->channels); dma->dev = &idxd->pdev->dev; + dma_cap_set(DMA_PRIVATE, dma->cap_mask); dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask); dma->device_release = idxd_dma_release; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index fa04acd5582a..085a0c3b62c6 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -26,12 +26,16 @@ MODULE_VERSION(IDXD_DRIVER_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Intel Corporation"); +static bool sva = true; +module_param(sva, bool, 0644); +MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); + #define DRV_NAME "idxd" bool support_enqcmd; static struct idr idxd_idrs[IDXD_TYPE_MAX]; -static struct mutex idxd_idr_lock; +static DEFINE_MUTEX(idxd_idr_lock); static struct pci_device_id idxd_pci_tbl[] = { /* DSA ver 1.0 platforms */ @@ -341,12 +345,14 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD reset complete\n"); - if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM)) { + if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { rc = idxd_enable_system_pasid(idxd); if (rc < 0) dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); else set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); + } else if (!sva) { + dev_warn(dev, "User forced SVA off via module param.\n"); } idxd_read_caps(idxd); @@ -547,7 +553,6 @@ static int __init idxd_init_module(void) else support_enqcmd = true; - mutex_init(&idxd_idr_lock); for (i = 0; i < IDXD_TYPE_MAX; i++) idr_init(&idxd_idrs[i]); diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 41ba21eea7c8..d5590c08db51 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1952,8 +1952,6 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, static int sdma_probe(struct platform_device *pdev) { - const struct of_device_id *of_id = - of_match_device(sdma_dt_ids, &pdev->dev); struct device_node *np = pdev->dev.of_node; struct device_node *spba_bus; const char *fw_name; @@ -1961,17 +1959,9 @@ static int sdma_probe(struct platform_device *pdev) int irq; struct resource *iores; struct resource spba_res; - struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev); int i; struct sdma_engine *sdma; s32 *saddr_arr; - const struct sdma_driver_data *drvdata = NULL; - - drvdata = of_id->data; - if (!drvdata) { - dev_err(&pdev->dev, "unable to find driver data\n"); - return -EINVAL; - } ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (ret) @@ -1984,7 +1974,7 @@ static int sdma_probe(struct platform_device *pdev) spin_lock_init(&sdma->channel_0_lock); sdma->dev = &pdev->dev; - sdma->drvdata = drvdata; + sdma->drvdata = of_device_get_match_data(sdma->dev); irq = platform_get_irq(pdev, 0); if (irq < 0) @@ -2063,8 +2053,6 @@ static int sdma_probe(struct platform_device *pdev) if (sdma->drvdata->script_addrs) sdma_add_scripts(sdma, sdma->drvdata->script_addrs); - if (pdata && pdata->script_addrs) - sdma_add_scripts(sdma, pdata->script_addrs); sdma->dma_device.dev = &pdev->dev; @@ -2110,30 +2098,18 @@ static int sdma_probe(struct platform_device *pdev) } /* - * Kick off firmware loading as the very last step: - * attempt to load firmware only if we're not on the error path, because - * the firmware callback requires a fully functional and allocated sdma - * instance. + * Because that device tree does not encode ROM script address, + * the RAM script in firmware is mandatory for device tree + * probe, otherwise it fails. */ - if (pdata) { - ret = sdma_get_firmware(sdma, pdata->fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware from platform data\n"); + ret = of_property_read_string(np, "fsl,sdma-ram-script-name", + &fw_name); + if (ret) { + dev_warn(&pdev->dev, "failed to get firmware name\n"); } else { - /* - * Because that device tree does not encode ROM script address, - * the RAM script in firmware is mandatory for device tree - * probe, otherwise it fails. - */ - ret = of_property_read_string(np, "fsl,sdma-ram-script-name", - &fw_name); - if (ret) { - dev_warn(&pdev->dev, "failed to get firmware name\n"); - } else { - ret = sdma_get_firmware(sdma, fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); - } + ret = sdma_get_firmware(sdma, fw_name); + if (ret) + dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); } return 0; diff --git a/drivers/dma/lgm/Kconfig b/drivers/dma/lgm/Kconfig new file mode 100644 index 000000000000..9194330ed0f2 --- /dev/null +++ b/drivers/dma/lgm/Kconfig @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +config INTEL_LDMA + bool "Lightning Mountain centralized DMA controllers" + depends on X86 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for Intel Lightning Mountain SOC DMA controllers. + These controllers provide DMA capabilities for a variety of on-chip + devices such as HSNAND and GSWIP (Gigabit Switch IP). diff --git a/drivers/dma/lgm/Makefile b/drivers/dma/lgm/Makefile new file mode 100644 index 000000000000..f318a8eff464 --- /dev/null +++ b/drivers/dma/lgm/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_INTEL_LDMA) += lgm-dma.o diff --git a/drivers/dma/lgm/lgm-dma.c b/drivers/dma/lgm/lgm-dma.c new file mode 100644 index 000000000000..efe8bd3a0e2a --- /dev/null +++ b/drivers/dma/lgm/lgm-dma.c @@ -0,0 +1,1739 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Lightning Mountain centralized DMA controller driver + * + * Copyright (c) 2016 - 2020 Intel Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../dmaengine.h" +#include "../virt-dma.h" + +#define DRIVER_NAME "lgm-dma" + +#define DMA_ID 0x0008 +#define DMA_ID_REV GENMASK(7, 0) +#define DMA_ID_PNR GENMASK(19, 16) +#define DMA_ID_CHNR GENMASK(26, 20) +#define DMA_ID_DW_128B BIT(27) +#define DMA_ID_AW_36B BIT(28) +#define DMA_VER32 0x32 +#define DMA_VER31 0x31 +#define DMA_VER22 0x0A + +#define DMA_CTRL 0x0010 +#define DMA_CTRL_RST BIT(0) +#define DMA_CTRL_DSRAM_PATH BIT(1) +#define DMA_CTRL_DBURST_WR BIT(3) +#define DMA_CTRL_VLD_DF_ACK BIT(4) +#define DMA_CTRL_CH_FL BIT(6) +#define DMA_CTRL_DS_FOD BIT(7) +#define DMA_CTRL_DRB BIT(8) +#define DMA_CTRL_ENBE BIT(9) +#define DMA_CTRL_DESC_TMOUT_CNT_V31 GENMASK(27, 16) +#define DMA_CTRL_DESC_TMOUT_EN_V31 BIT(30) +#define DMA_CTRL_PKTARB BIT(31) + +#define DMA_CPOLL 0x0014 +#define DMA_CPOLL_CNT GENMASK(15, 4) +#define DMA_CPOLL_EN BIT(31) + +#define DMA_CS 0x0018 +#define DMA_CS_MASK GENMASK(5, 0) + +#define DMA_CCTRL 0x001C +#define DMA_CCTRL_ON BIT(0) +#define DMA_CCTRL_RST BIT(1) +#define DMA_CCTRL_CH_POLL_EN BIT(2) +#define DMA_CCTRL_CH_ABC BIT(3) /* Adaptive Burst Chop */ +#define DMA_CDBA_MSB GENMASK(7, 4) +#define DMA_CCTRL_DIR_TX BIT(8) +#define DMA_CCTRL_CLASS GENMASK(11, 9) +#define DMA_CCTRL_CLASSH GENMASK(19, 18) +#define DMA_CCTRL_WR_NP_EN BIT(21) +#define DMA_CCTRL_PDEN BIT(23) +#define DMA_MAX_CLASS (SZ_32 - 1) + +#define DMA_CDBA 0x0020 +#define DMA_CDLEN 0x0024 +#define DMA_CIS 0x0028 +#define DMA_CIE 0x002C +#define DMA_CI_EOP BIT(1) +#define DMA_CI_DUR BIT(2) +#define DMA_CI_DESCPT BIT(3) +#define DMA_CI_CHOFF BIT(4) +#define DMA_CI_RDERR BIT(5) +#define DMA_CI_ALL \ + (DMA_CI_EOP | DMA_CI_DUR | DMA_CI_DESCPT | DMA_CI_CHOFF | DMA_CI_RDERR) + +#define DMA_PS 0x0040 +#define DMA_PCTRL 0x0044 +#define DMA_PCTRL_RXBL16 BIT(0) +#define DMA_PCTRL_TXBL16 BIT(1) +#define DMA_PCTRL_RXBL GENMASK(3, 2) +#define DMA_PCTRL_RXBL_8 3 +#define DMA_PCTRL_TXBL GENMASK(5, 4) +#define DMA_PCTRL_TXBL_8 3 +#define DMA_PCTRL_PDEN BIT(6) +#define DMA_PCTRL_RXBL32 BIT(7) +#define DMA_PCTRL_RXENDI GENMASK(9, 8) +#define DMA_PCTRL_TXENDI GENMASK(11, 10) +#define DMA_PCTRL_TXBL32 BIT(15) +#define DMA_PCTRL_MEM_FLUSH BIT(16) + +#define DMA_IRNEN1 0x00E8 +#define DMA_IRNCR1 0x00EC +#define DMA_IRNEN 0x00F4 +#define DMA_IRNCR 0x00F8 +#define DMA_C_DP_TICK 0x100 +#define DMA_C_DP_TICK_TIKNARB GENMASK(15, 0) +#define DMA_C_DP_TICK_TIKARB GENMASK(31, 16) + +#define DMA_C_HDRM 0x110 +/* + * If header mode is set in DMA descriptor, + * If bit 30 is disabled, HDR_LEN must be configured according to channel + * requirement. + * If bit 30 is enabled(checksum with heade mode), HDR_LEN has no need to + * be configured. It will enable check sum for switch + * If header mode is not set in DMA descriptor, + * This register setting doesn't matter + */ +#define DMA_C_HDRM_HDR_SUM BIT(30) + +#define DMA_C_BOFF 0x120 +#define DMA_C_BOFF_BOF_LEN GENMASK(7, 0) +#define DMA_C_BOFF_EN BIT(31) + +#define DMA_ORRC 0x190 +#define DMA_ORRC_ORRCNT GENMASK(8, 4) +#define DMA_ORRC_EN BIT(31) + +#define DMA_C_ENDIAN 0x200 +#define DMA_C_END_DATAENDI GENMASK(1, 0) +#define DMA_C_END_DE_EN BIT(7) +#define DMA_C_END_DESENDI GENMASK(9, 8) +#define DMA_C_END_DES_EN BIT(16) + +/* DMA controller capability */ +#define DMA_ADDR_36BIT BIT(0) +#define DMA_DATA_128BIT BIT(1) +#define DMA_CHAN_FLOW_CTL BIT(2) +#define DMA_DESC_FOD BIT(3) +#define DMA_DESC_IN_SRAM BIT(4) +#define DMA_EN_BYTE_EN BIT(5) +#define DMA_DBURST_WR BIT(6) +#define DMA_VALID_DESC_FETCH_ACK BIT(7) +#define DMA_DFT_DRB BIT(8) + +#define DMA_ORRC_MAX_CNT (SZ_32 - 1) +#define DMA_DFT_POLL_CNT SZ_4 +#define DMA_DFT_BURST_V22 SZ_2 +#define DMA_BURSTL_8DW SZ_8 +#define DMA_BURSTL_16DW SZ_16 +#define DMA_BURSTL_32DW SZ_32 +#define DMA_DFT_BURST DMA_BURSTL_16DW +#define DMA_MAX_DESC_NUM (SZ_8K - 1) +#define DMA_CHAN_BOFF_MAX (SZ_256 - 1) +#define DMA_DFT_ENDIAN 0 + +#define DMA_DFT_DESC_TCNT 50 +#define DMA_HDR_LEN_MAX (SZ_16K - 1) + +/* DMA flags */ +#define DMA_TX_CH BIT(0) +#define DMA_RX_CH BIT(1) +#define DEVICE_ALLOC_DESC BIT(2) +#define CHAN_IN_USE BIT(3) +#define DMA_HW_DESC BIT(4) + +/* Descriptor fields */ +#define DESC_DATA_LEN GENMASK(15, 0) +#define DESC_BYTE_OFF GENMASK(25, 23) +#define DESC_EOP BIT(28) +#define DESC_SOP BIT(29) +#define DESC_C BIT(30) +#define DESC_OWN BIT(31) + +#define DMA_CHAN_RST 1 +#define DMA_MAX_SIZE (BIT(16) - 1) +#define MAX_LOWER_CHANS 32 +#define MASK_LOWER_CHANS GENMASK(4, 0) +#define DMA_OWN 1 +#define HIGH_4_BITS GENMASK(3, 0) +#define DMA_DFT_DESC_NUM 1 +#define DMA_PKT_DROP_DIS 0 + +enum ldma_chan_on_off { + DMA_CH_OFF = 0, + DMA_CH_ON = 1, +}; + +enum { + DMA_TYPE_TX = 0, + DMA_TYPE_RX, + DMA_TYPE_MCPY, +}; + +struct ldma_dev; +struct ldma_port; + +struct ldma_chan { + struct virt_dma_chan vchan; + struct ldma_port *port; /* back pointer */ + char name[8]; /* Channel name */ + int nr; /* Channel id in hardware */ + u32 flags; /* central way or channel based way */ + enum ldma_chan_on_off onoff; + dma_addr_t desc_phys; + void *desc_base; /* Virtual address */ + u32 desc_cnt; /* Number of descriptors */ + int rst; + u32 hdrm_len; + bool hdrm_csum; + u32 boff_len; + u32 data_endian; + u32 desc_endian; + bool pden; + bool desc_rx_np; + bool data_endian_en; + bool desc_endian_en; + bool abc_en; + bool desc_init; + struct dma_pool *desc_pool; /* Descriptors pool */ + u32 desc_num; + struct dw2_desc_sw *ds; + struct work_struct work; + struct dma_slave_config config; +}; + +struct ldma_port { + struct ldma_dev *ldev; /* back pointer */ + u32 portid; + u32 rxbl; + u32 txbl; + u32 rxendi; + u32 txendi; + u32 pkt_drop; +}; + +/* Instance specific data */ +struct ldma_inst_data { + bool desc_in_sram; + bool chan_fc; + bool desc_fod; /* Fetch On Demand */ + bool valid_desc_fetch_ack; + u32 orrc; /* Outstanding read count */ + const char *name; + u32 type; +}; + +struct ldma_dev { + struct device *dev; + void __iomem *base; + struct reset_control *rst; + struct clk *core_clk; + struct dma_device dma_dev; + u32 ver; + int irq; + struct ldma_port *ports; + struct ldma_chan *chans; /* channel list on this DMA or port */ + spinlock_t dev_lock; /* Controller register exclusive */ + u32 chan_nrs; + u32 port_nrs; + u32 channels_mask; + u32 flags; + u32 pollcnt; + const struct ldma_inst_data *inst; + struct workqueue_struct *wq; +}; + +struct dw2_desc { + u32 field; + u32 addr; +} __packed __aligned(8); + +struct dw2_desc_sw { + struct virt_dma_desc vdesc; + struct ldma_chan *chan; + dma_addr_t desc_phys; + size_t desc_cnt; + size_t size; + struct dw2_desc *desc_hw; +}; + +static inline void +ldma_update_bits(struct ldma_dev *d, u32 mask, u32 val, u32 ofs) +{ + u32 old_val, new_val; + + old_val = readl(d->base + ofs); + new_val = (old_val & ~mask) | (val & mask); + + if (new_val != old_val) + writel(new_val, d->base + ofs); +} + +static inline struct ldma_chan *to_ldma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct ldma_chan, vchan.chan); +} + +static inline struct ldma_dev *to_ldma_dev(struct dma_device *dma_dev) +{ + return container_of(dma_dev, struct ldma_dev, dma_dev); +} + +static inline struct dw2_desc_sw *to_lgm_dma_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct dw2_desc_sw, vdesc); +} + +static inline bool ldma_chan_tx(struct ldma_chan *c) +{ + return !!(c->flags & DMA_TX_CH); +} + +static inline bool ldma_chan_is_hw_desc(struct ldma_chan *c) +{ + return !!(c->flags & DMA_HW_DESC); +} + +static void ldma_dev_reset(struct ldma_dev *d) + +{ + unsigned long flags; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CTRL_RST, DMA_CTRL_RST, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_pkt_arb_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_PKTARB; + u32 val = enable ? DMA_CTRL_PKTARB : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_sram_desc_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_DSRAM_PATH; + u32 val = enable ? DMA_CTRL_DSRAM_PATH : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_chan_flow_ctl_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type != DMA_TYPE_TX) + return; + + mask = DMA_CTRL_CH_FL; + val = enable ? DMA_CTRL_CH_FL : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_global_polling_enable(struct ldma_dev *d) +{ + unsigned long flags; + u32 mask = DMA_CPOLL_EN | DMA_CPOLL_CNT; + u32 val = DMA_CPOLL_EN; + + val |= FIELD_PREP(DMA_CPOLL_CNT, d->pollcnt); + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CPOLL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_desc_fetch_on_demand_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type == DMA_TYPE_MCPY) + return; + + mask = DMA_CTRL_DS_FOD; + val = enable ? DMA_CTRL_DS_FOD : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_byte_enable_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_ENBE; + u32 val = enable ? DMA_CTRL_ENBE : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_orrc_cfg(struct ldma_dev *d) +{ + unsigned long flags; + u32 val = 0; + u32 mask; + + if (d->inst->type == DMA_TYPE_RX) + return; + + mask = DMA_ORRC_EN | DMA_ORRC_ORRCNT; + if (d->inst->orrc > 0 && d->inst->orrc <= DMA_ORRC_MAX_CNT) + val = DMA_ORRC_EN | FIELD_PREP(DMA_ORRC_ORRCNT, d->inst->orrc); + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_ORRC); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_df_tout_cfg(struct ldma_dev *d, bool enable, int tcnt) +{ + u32 mask = DMA_CTRL_DESC_TMOUT_CNT_V31; + unsigned long flags; + u32 val; + + if (enable) + val = DMA_CTRL_DESC_TMOUT_EN_V31 | FIELD_PREP(DMA_CTRL_DESC_TMOUT_CNT_V31, tcnt); + else + val = 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_dburst_wr_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type != DMA_TYPE_RX && d->inst->type != DMA_TYPE_MCPY) + return; + + mask = DMA_CTRL_DBURST_WR; + val = enable ? DMA_CTRL_DBURST_WR : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_vld_fetch_ack_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type != DMA_TYPE_TX) + return; + + mask = DMA_CTRL_VLD_DF_ACK; + val = enable ? DMA_CTRL_VLD_DF_ACK : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_drb_cfg(struct ldma_dev *d, int enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_DRB; + u32 val = enable ? DMA_CTRL_DRB : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static int ldma_dev_cfg(struct ldma_dev *d) +{ + bool enable; + + ldma_dev_pkt_arb_cfg(d, true); + ldma_dev_global_polling_enable(d); + + enable = !!(d->flags & DMA_DFT_DRB); + ldma_dev_drb_cfg(d, enable); + + enable = !!(d->flags & DMA_EN_BYTE_EN); + ldma_dev_byte_enable_cfg(d, enable); + + enable = !!(d->flags & DMA_CHAN_FLOW_CTL); + ldma_dev_chan_flow_ctl_cfg(d, enable); + + enable = !!(d->flags & DMA_DESC_FOD); + ldma_dev_desc_fetch_on_demand_cfg(d, enable); + + enable = !!(d->flags & DMA_DESC_IN_SRAM); + ldma_dev_sram_desc_cfg(d, enable); + + enable = !!(d->flags & DMA_DBURST_WR); + ldma_dev_dburst_wr_cfg(d, enable); + + enable = !!(d->flags & DMA_VALID_DESC_FETCH_ACK); + ldma_dev_vld_fetch_ack_cfg(d, enable); + + if (d->ver > DMA_VER22) { + ldma_dev_orrc_cfg(d); + ldma_dev_df_tout_cfg(d, true, DMA_DFT_DESC_TCNT); + } + + dev_dbg(d->dev, "%s Controller 0x%08x configuration done\n", + d->inst->name, readl(d->base + DMA_CTRL)); + + return 0; +} + +static int ldma_chan_cctrl_cfg(struct ldma_chan *c, u32 val) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 class_low, class_high; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + reg = readl(d->base + DMA_CCTRL); + /* Read from hardware */ + if (reg & DMA_CCTRL_DIR_TX) + c->flags |= DMA_TX_CH; + else + c->flags |= DMA_RX_CH; + + /* Keep the class value unchanged */ + class_low = FIELD_GET(DMA_CCTRL_CLASS, reg); + class_high = FIELD_GET(DMA_CCTRL_CLASSH, reg); + val &= ~DMA_CCTRL_CLASS; + val |= FIELD_PREP(DMA_CCTRL_CLASS, class_low); + val &= ~DMA_CCTRL_CLASSH; + val |= FIELD_PREP(DMA_CCTRL_CLASSH, class_high); + writel(val, d->base + DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + return 0; +} + +static void ldma_chan_irq_init(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 enofs, crofs; + u32 cn_bit; + + if (c->nr < MAX_LOWER_CHANS) { + enofs = DMA_IRNEN; + crofs = DMA_IRNCR; + } else { + enofs = DMA_IRNEN1; + crofs = DMA_IRNCR1; + } + + cn_bit = BIT(c->nr & MASK_LOWER_CHANS); + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + + /* Clear all interrupts and disabled it */ + writel(0, d->base + DMA_CIE); + writel(DMA_CI_ALL, d->base + DMA_CIS); + + ldma_update_bits(d, cn_bit, 0, enofs); + writel(cn_bit, d->base + crofs); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_chan_set_class(struct ldma_chan *c, u32 val) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 class_val; + + if (d->inst->type == DMA_TYPE_MCPY || val > DMA_MAX_CLASS) + return; + + /* 3 bits low */ + class_val = FIELD_PREP(DMA_CCTRL_CLASS, val & 0x7); + /* 2 bits high */ + class_val |= FIELD_PREP(DMA_CCTRL_CLASSH, (val >> 3) & 0x3); + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_CLASS | DMA_CCTRL_CLASSH, class_val, + DMA_CCTRL); +} + +static int ldma_chan_on(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + /* If descriptors not configured, not allow to turn on channel */ + if (WARN_ON(!c->desc_init)) + return -EINVAL; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_ON, DMA_CCTRL_ON, DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + c->onoff = DMA_CH_ON; + + return 0; +} + +static int ldma_chan_off(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 val; + int ret; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_ON, 0, DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val, + !(val & DMA_CCTRL_ON), 0, 10000); + if (ret) + return ret; + + c->onoff = DMA_CH_OFF; + + return 0; +} + +static void ldma_chan_desc_hw_cfg(struct ldma_chan *c, dma_addr_t desc_base, + int desc_num) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + writel(lower_32_bits(desc_base), d->base + DMA_CDBA); + + /* Higher 4 bits of 36 bit addressing */ + if (IS_ENABLED(CONFIG_64BIT)) { + u32 hi = upper_32_bits(desc_base) & HIGH_4_BITS; + + ldma_update_bits(d, DMA_CDBA_MSB, + FIELD_PREP(DMA_CDBA_MSB, hi), DMA_CCTRL); + } + writel(desc_num, d->base + DMA_CDLEN); + spin_unlock_irqrestore(&d->dev_lock, flags); + + c->desc_init = true; +} + +static struct dma_async_tx_descriptor * +ldma_chan_desc_cfg(struct dma_chan *chan, dma_addr_t desc_base, int desc_num) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + struct dma_async_tx_descriptor *tx; + struct dw2_desc_sw *ds; + + if (!desc_num) { + dev_err(d->dev, "Channel %d must allocate descriptor first\n", + c->nr); + return NULL; + } + + if (desc_num > DMA_MAX_DESC_NUM) { + dev_err(d->dev, "Channel %d descriptor number out of range %d\n", + c->nr, desc_num); + return NULL; + } + + ldma_chan_desc_hw_cfg(c, desc_base, desc_num); + + c->flags |= DMA_HW_DESC; + c->desc_cnt = desc_num; + c->desc_phys = desc_base; + + ds = kzalloc(sizeof(*ds), GFP_NOWAIT); + if (!ds) + return NULL; + + tx = &ds->vdesc.tx; + dma_async_tx_descriptor_init(tx, chan); + + return tx; +} + +static int ldma_chan_reset(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 val; + int ret; + + ret = ldma_chan_off(c); + if (ret) + return ret; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_RST, DMA_CCTRL_RST, DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val, + !(val & DMA_CCTRL_RST), 0, 10000); + if (ret) + return ret; + + c->rst = 1; + c->desc_init = false; + + return 0; +} + +static void ldma_chan_byte_offset_cfg(struct ldma_chan *c, u32 boff_len) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask = DMA_C_BOFF_EN | DMA_C_BOFF_BOF_LEN; + u32 val; + + if (boff_len > 0 && boff_len <= DMA_CHAN_BOFF_MAX) + val = FIELD_PREP(DMA_C_BOFF_BOF_LEN, boff_len) | DMA_C_BOFF_EN; + else + val = 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_BOFF); +} + +static void ldma_chan_data_endian_cfg(struct ldma_chan *c, bool enable, + u32 endian_type) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask = DMA_C_END_DE_EN | DMA_C_END_DATAENDI; + u32 val; + + if (enable) + val = DMA_C_END_DE_EN | FIELD_PREP(DMA_C_END_DATAENDI, endian_type); + else + val = 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_ENDIAN); +} + +static void ldma_chan_desc_endian_cfg(struct ldma_chan *c, bool enable, + u32 endian_type) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask = DMA_C_END_DES_EN | DMA_C_END_DESENDI; + u32 val; + + if (enable) + val = DMA_C_END_DES_EN | FIELD_PREP(DMA_C_END_DESENDI, endian_type); + else + val = 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_ENDIAN); +} + +static void ldma_chan_hdr_mode_cfg(struct ldma_chan *c, u32 hdr_len, bool csum) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask, val; + + /* NB, csum disabled, hdr length must be provided */ + if (!csum && (!hdr_len || hdr_len > DMA_HDR_LEN_MAX)) + return; + + mask = DMA_C_HDRM_HDR_SUM; + val = DMA_C_HDRM_HDR_SUM; + + if (!csum && hdr_len) + val = hdr_len; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_HDRM); +} + +static void ldma_chan_rxwr_np_cfg(struct ldma_chan *c, bool enable) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask, val; + + /* Only valid for RX channel */ + if (ldma_chan_tx(c)) + return; + + mask = DMA_CCTRL_WR_NP_EN; + val = enable ? DMA_CCTRL_WR_NP_EN : 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_CCTRL); +} + +static void ldma_chan_abc_cfg(struct ldma_chan *c, bool enable) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask, val; + + if (d->ver < DMA_VER32 || ldma_chan_tx(c)) + return; + + mask = DMA_CCTRL_CH_ABC; + val = enable ? DMA_CCTRL_CH_ABC : 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_CCTRL); +} + +static int ldma_port_cfg(struct ldma_port *p) +{ + unsigned long flags; + struct ldma_dev *d; + u32 reg; + + d = p->ldev; + reg = FIELD_PREP(DMA_PCTRL_TXENDI, p->txendi); + reg |= FIELD_PREP(DMA_PCTRL_RXENDI, p->rxendi); + + if (d->ver == DMA_VER22) { + reg |= FIELD_PREP(DMA_PCTRL_TXBL, p->txbl); + reg |= FIELD_PREP(DMA_PCTRL_RXBL, p->rxbl); + } else { + reg |= FIELD_PREP(DMA_PCTRL_PDEN, p->pkt_drop); + + if (p->txbl == DMA_BURSTL_32DW) + reg |= DMA_PCTRL_TXBL32; + else if (p->txbl == DMA_BURSTL_16DW) + reg |= DMA_PCTRL_TXBL16; + else + reg |= FIELD_PREP(DMA_PCTRL_TXBL, DMA_PCTRL_TXBL_8); + + if (p->rxbl == DMA_BURSTL_32DW) + reg |= DMA_PCTRL_RXBL32; + else if (p->rxbl == DMA_BURSTL_16DW) + reg |= DMA_PCTRL_RXBL16; + else + reg |= FIELD_PREP(DMA_PCTRL_RXBL, DMA_PCTRL_RXBL_8); + } + + spin_lock_irqsave(&d->dev_lock, flags); + writel(p->portid, d->base + DMA_PS); + writel(reg, d->base + DMA_PCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + reg = readl(d->base + DMA_PCTRL); /* read back */ + dev_dbg(d->dev, "Port Control 0x%08x configuration done\n", reg); + + return 0; +} + +static int ldma_chan_cfg(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 reg; + + reg = c->pden ? DMA_CCTRL_PDEN : 0; + reg |= c->onoff ? DMA_CCTRL_ON : 0; + reg |= c->rst ? DMA_CCTRL_RST : 0; + + ldma_chan_cctrl_cfg(c, reg); + ldma_chan_irq_init(c); + + if (d->ver <= DMA_VER22) + return 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_chan_set_class(c, c->nr); + ldma_chan_byte_offset_cfg(c, c->boff_len); + ldma_chan_data_endian_cfg(c, c->data_endian_en, c->data_endian); + ldma_chan_desc_endian_cfg(c, c->desc_endian_en, c->desc_endian); + ldma_chan_hdr_mode_cfg(c, c->hdrm_len, c->hdrm_csum); + ldma_chan_rxwr_np_cfg(c, c->desc_rx_np); + ldma_chan_abc_cfg(c, c->abc_en); + spin_unlock_irqrestore(&d->dev_lock, flags); + + if (ldma_chan_is_hw_desc(c)) + ldma_chan_desc_hw_cfg(c, c->desc_phys, c->desc_cnt); + + return 0; +} + +static void ldma_dev_init(struct ldma_dev *d) +{ + unsigned long ch_mask = (unsigned long)d->channels_mask; + struct ldma_port *p; + struct ldma_chan *c; + int i; + u32 j; + + spin_lock_init(&d->dev_lock); + ldma_dev_reset(d); + ldma_dev_cfg(d); + + /* DMA port initialization */ + for (i = 0; i < d->port_nrs; i++) { + p = &d->ports[i]; + ldma_port_cfg(p); + } + + /* DMA channel initialization */ + for_each_set_bit(j, &ch_mask, d->chan_nrs) { + c = &d->chans[j]; + ldma_chan_cfg(c); + } +} + +static int ldma_cfg_init(struct ldma_dev *d) +{ + struct fwnode_handle *fwnode = dev_fwnode(d->dev); + struct ldma_port *p; + int i; + + if (fwnode_property_read_bool(fwnode, "intel,dma-byte-en")) + d->flags |= DMA_EN_BYTE_EN; + + if (fwnode_property_read_bool(fwnode, "intel,dma-dburst-wr")) + d->flags |= DMA_DBURST_WR; + + if (fwnode_property_read_bool(fwnode, "intel,dma-drb")) + d->flags |= DMA_DFT_DRB; + + if (fwnode_property_read_u32(fwnode, "intel,dma-poll-cnt", + &d->pollcnt)) + d->pollcnt = DMA_DFT_POLL_CNT; + + if (d->inst->chan_fc) + d->flags |= DMA_CHAN_FLOW_CTL; + + if (d->inst->desc_fod) + d->flags |= DMA_DESC_FOD; + + if (d->inst->desc_in_sram) + d->flags |= DMA_DESC_IN_SRAM; + + if (d->inst->valid_desc_fetch_ack) + d->flags |= DMA_VALID_DESC_FETCH_ACK; + + if (d->ver > DMA_VER22) { + if (!d->port_nrs) + return -EINVAL; + + for (i = 0; i < d->port_nrs; i++) { + p = &d->ports[i]; + p->rxendi = DMA_DFT_ENDIAN; + p->txendi = DMA_DFT_ENDIAN; + p->rxbl = DMA_DFT_BURST; + p->txbl = DMA_DFT_BURST; + p->pkt_drop = DMA_PKT_DROP_DIS; + } + } + + return 0; +} + +static void dma_free_desc_resource(struct virt_dma_desc *vdesc) +{ + struct dw2_desc_sw *ds = to_lgm_dma_desc(vdesc); + struct ldma_chan *c = ds->chan; + + dma_pool_free(c->desc_pool, ds->desc_hw, ds->desc_phys); + kfree(ds); +} + +static struct dw2_desc_sw * +dma_alloc_desc_resource(int num, struct ldma_chan *c) +{ + struct device *dev = c->vchan.chan.device->dev; + struct dw2_desc_sw *ds; + + if (num > c->desc_num) { + dev_err(dev, "sg num %d exceed max %d\n", num, c->desc_num); + return NULL; + } + + ds = kzalloc(sizeof(*ds), GFP_NOWAIT); + if (!ds) + return NULL; + + ds->chan = c; + ds->desc_hw = dma_pool_zalloc(c->desc_pool, GFP_ATOMIC, + &ds->desc_phys); + if (!ds->desc_hw) { + dev_dbg(dev, "out of memory for link descriptor\n"); + kfree(ds); + return NULL; + } + ds->desc_cnt = num; + + return ds; +} + +static void ldma_chan_irq_en(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + spin_lock_irqsave(&d->dev_lock, flags); + writel(c->nr, d->base + DMA_CS); + writel(DMA_CI_EOP, d->base + DMA_CIE); + writel(BIT(c->nr), d->base + DMA_IRNEN); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_issue_pending(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + if (d->ver == DMA_VER22) { + spin_lock_irqsave(&c->vchan.lock, flags); + if (vchan_issue_pending(&c->vchan)) { + struct virt_dma_desc *vdesc; + + /* Get the next descriptor */ + vdesc = vchan_next_desc(&c->vchan); + if (!vdesc) { + c->ds = NULL; + spin_unlock_irqrestore(&c->vchan.lock, flags); + return; + } + list_del(&vdesc->node); + c->ds = to_lgm_dma_desc(vdesc); + ldma_chan_desc_hw_cfg(c, c->ds->desc_phys, c->ds->desc_cnt); + ldma_chan_irq_en(c); + } + spin_unlock_irqrestore(&c->vchan.lock, flags); + } + ldma_chan_on(c); +} + +static void ldma_synchronize(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + /* + * clear any pending work if any. In that + * case the resource needs to be free here. + */ + cancel_work_sync(&c->work); + vchan_synchronize(&c->vchan); + if (c->ds) + dma_free_desc_resource(&c->ds->vdesc); +} + +static int ldma_terminate_all(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&c->vchan.lock, flags); + vchan_get_all_descriptors(&c->vchan, &head); + spin_unlock_irqrestore(&c->vchan.lock, flags); + vchan_dma_desc_free_list(&c->vchan, &head); + + return ldma_chan_reset(c); +} + +static int ldma_resume_chan(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + ldma_chan_on(c); + + return 0; +} + +static int ldma_pause_chan(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + return ldma_chan_off(c); +} + +static enum dma_status +ldma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + enum dma_status status = DMA_COMPLETE; + + if (d->ver == DMA_VER22) + status = dma_cookie_status(chan, cookie, txstate); + + return status; +} + +static void dma_chan_irq(int irq, void *data) +{ + struct ldma_chan *c = data; + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 stat; + + /* Disable channel interrupts */ + writel(c->nr, d->base + DMA_CS); + stat = readl(d->base + DMA_CIS); + if (!stat) + return; + + writel(readl(d->base + DMA_CIE) & ~DMA_CI_ALL, d->base + DMA_CIE); + writel(stat, d->base + DMA_CIS); + queue_work(d->wq, &c->work); +} + +static irqreturn_t dma_interrupt(int irq, void *dev_id) +{ + struct ldma_dev *d = dev_id; + struct ldma_chan *c; + unsigned long irncr; + u32 cid; + + irncr = readl(d->base + DMA_IRNCR); + if (!irncr) { + dev_err(d->dev, "dummy interrupt\n"); + return IRQ_NONE; + } + + for_each_set_bit(cid, &irncr, d->chan_nrs) { + /* Mask */ + writel(readl(d->base + DMA_IRNEN) & ~BIT(cid), d->base + DMA_IRNEN); + /* Ack */ + writel(readl(d->base + DMA_IRNCR) | BIT(cid), d->base + DMA_IRNCR); + + c = &d->chans[cid]; + dma_chan_irq(irq, c); + } + + return IRQ_HANDLED; +} + +static void prep_slave_burst_len(struct ldma_chan *c) +{ + struct ldma_port *p = c->port; + struct dma_slave_config *cfg = &c->config; + + if (cfg->dst_maxburst) + cfg->src_maxburst = cfg->dst_maxburst; + + /* TX and RX has the same burst length */ + p->txbl = ilog2(cfg->src_maxburst); + p->rxbl = p->txbl; +} + +static struct dma_async_tx_descriptor * +ldma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sglen, enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + size_t len, avail, total = 0; + struct dw2_desc *hw_ds; + struct dw2_desc_sw *ds; + struct scatterlist *sg; + int num = sglen, i; + dma_addr_t addr; + + if (!sgl) + return NULL; + + if (d->ver > DMA_VER22) + return ldma_chan_desc_cfg(chan, sgl->dma_address, sglen); + + for_each_sg(sgl, sg, sglen, i) { + avail = sg_dma_len(sg); + if (avail > DMA_MAX_SIZE) + num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; + } + + ds = dma_alloc_desc_resource(num, c); + if (!ds) + return NULL; + + c->ds = ds; + + num = 0; + /* sop and eop has to be handled nicely */ + for_each_sg(sgl, sg, sglen, i) { + addr = sg_dma_address(sg); + avail = sg_dma_len(sg); + total += avail; + + do { + len = min_t(size_t, avail, DMA_MAX_SIZE); + + hw_ds = &ds->desc_hw[num]; + switch (sglen) { + case 1: + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 1); + + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 1); + break; + default: + if (num == 0) { + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 1); + + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 0); + } else if (num == (sglen - 1)) { + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 0); + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 1); + } else { + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 0); + + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 0); + } + break; + } + /* Only 32 bit address supported */ + hw_ds->addr = (u32)addr; + + hw_ds->field &= ~DESC_DATA_LEN; + hw_ds->field |= FIELD_PREP(DESC_DATA_LEN, len); + + hw_ds->field &= ~DESC_C; + hw_ds->field |= FIELD_PREP(DESC_C, 0); + + hw_ds->field &= ~DESC_BYTE_OFF; + hw_ds->field |= FIELD_PREP(DESC_BYTE_OFF, addr & 0x3); + + /* Ensure data ready before ownership change */ + wmb(); + hw_ds->field &= ~DESC_OWN; + hw_ds->field |= FIELD_PREP(DESC_OWN, DMA_OWN); + + /* Ensure ownership changed before moving forward */ + wmb(); + num++; + addr += len; + avail -= len; + } while (avail); + } + + ds->size = total; + prep_slave_burst_len(c); + + return vchan_tx_prep(&c->vchan, &ds->vdesc, DMA_CTRL_ACK); +} + +static int +ldma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + memcpy(&c->config, cfg, sizeof(c->config)); + + return 0; +} + +static int ldma_alloc_chan_resources(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + struct device *dev = c->vchan.chan.device->dev; + size_t desc_sz; + + if (d->ver > DMA_VER22) { + c->flags |= CHAN_IN_USE; + return 0; + } + + if (c->desc_pool) + return c->desc_num; + + desc_sz = c->desc_num * sizeof(struct dw2_desc); + c->desc_pool = dma_pool_create(c->name, dev, desc_sz, + __alignof__(struct dw2_desc), 0); + + if (!c->desc_pool) { + dev_err(dev, "unable to allocate descriptor pool\n"); + return -ENOMEM; + } + + return c->desc_num; +} + +static void ldma_free_chan_resources(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + + if (d->ver == DMA_VER22) { + dma_pool_destroy(c->desc_pool); + c->desc_pool = NULL; + vchan_free_chan_resources(to_virt_chan(chan)); + ldma_chan_reset(c); + } else { + c->flags &= ~CHAN_IN_USE; + } +} + +static void dma_work(struct work_struct *work) +{ + struct ldma_chan *c = container_of(work, struct ldma_chan, work); + struct dma_async_tx_descriptor *tx = &c->ds->vdesc.tx; + struct virt_dma_chan *vc = &c->vchan; + struct dmaengine_desc_callback cb; + struct virt_dma_desc *vd, *_vd; + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&c->vchan.lock, flags); + list_splice_tail_init(&vc->desc_completed, &head); + spin_unlock_irqrestore(&c->vchan.lock, flags); + dmaengine_desc_get_callback(tx, &cb); + dma_cookie_complete(tx); + dmaengine_desc_callback_invoke(&cb, NULL); + + list_for_each_entry_safe(vd, _vd, &head, node) { + dmaengine_desc_get_callback(tx, &cb); + dma_cookie_complete(tx); + list_del(&vd->node); + dmaengine_desc_callback_invoke(&cb, NULL); + + vchan_vdesc_fini(vd); + } + c->ds = NULL; +} + +static void +update_burst_len_v22(struct ldma_chan *c, struct ldma_port *p, u32 burst) +{ + if (ldma_chan_tx(c)) + p->txbl = ilog2(burst); + else + p->rxbl = ilog2(burst); +} + +static void +update_burst_len_v3X(struct ldma_chan *c, struct ldma_port *p, u32 burst) +{ + if (ldma_chan_tx(c)) + p->txbl = burst; + else + p->rxbl = burst; +} + +static int +update_client_configs(struct of_dma *ofdma, struct of_phandle_args *spec) +{ + struct ldma_dev *d = ofdma->of_dma_data; + u32 chan_id = spec->args[0]; + u32 port_id = spec->args[1]; + u32 burst = spec->args[2]; + struct ldma_port *p; + struct ldma_chan *c; + + if (chan_id >= d->chan_nrs || port_id >= d->port_nrs) + return 0; + + p = &d->ports[port_id]; + c = &d->chans[chan_id]; + c->port = p; + + if (d->ver == DMA_VER22) + update_burst_len_v22(c, p, burst); + else + update_burst_len_v3X(c, p, burst); + + ldma_port_cfg(p); + + return 1; +} + +static struct dma_chan *ldma_xlate(struct of_phandle_args *spec, + struct of_dma *ofdma) +{ + struct ldma_dev *d = ofdma->of_dma_data; + u32 chan_id = spec->args[0]; + int ret; + + if (!spec->args_count) + return NULL; + + /* if args_count is 1 driver use default settings */ + if (spec->args_count > 1) { + ret = update_client_configs(ofdma, spec); + if (!ret) + return NULL; + } + + return dma_get_slave_channel(&d->chans[chan_id].vchan.chan); +} + +static void ldma_dma_init_v22(int i, struct ldma_dev *d) +{ + struct ldma_chan *c; + + c = &d->chans[i]; + c->nr = i; /* Real channel number */ + c->rst = DMA_CHAN_RST; + c->desc_num = DMA_DFT_DESC_NUM; + snprintf(c->name, sizeof(c->name), "chan%d", c->nr); + INIT_WORK(&c->work, dma_work); + c->vchan.desc_free = dma_free_desc_resource; + vchan_init(&c->vchan, &d->dma_dev); +} + +static void ldma_dma_init_v3X(int i, struct ldma_dev *d) +{ + struct ldma_chan *c; + + c = &d->chans[i]; + c->data_endian = DMA_DFT_ENDIAN; + c->desc_endian = DMA_DFT_ENDIAN; + c->data_endian_en = false; + c->desc_endian_en = false; + c->desc_rx_np = false; + c->flags |= DEVICE_ALLOC_DESC; + c->onoff = DMA_CH_OFF; + c->rst = DMA_CHAN_RST; + c->abc_en = true; + c->hdrm_csum = false; + c->boff_len = 0; + c->nr = i; + c->vchan.desc_free = dma_free_desc_resource; + vchan_init(&c->vchan, &d->dma_dev); +} + +static int ldma_init_v22(struct ldma_dev *d, struct platform_device *pdev) +{ + int ret; + + ret = device_property_read_u32(d->dev, "dma-channels", &d->chan_nrs); + if (ret < 0) { + dev_err(d->dev, "unable to read dma-channels property\n"); + return ret; + } + + d->irq = platform_get_irq(pdev, 0); + if (d->irq < 0) + return d->irq; + + ret = devm_request_irq(&pdev->dev, d->irq, dma_interrupt, 0, + DRIVER_NAME, d); + if (ret) + return ret; + + d->wq = alloc_ordered_workqueue("dma_wq", WQ_MEM_RECLAIM | + WQ_HIGHPRI); + if (!d->wq) + return -ENOMEM; + + return 0; +} + +static void ldma_clk_disable(void *data) +{ + struct ldma_dev *d = data; + + clk_disable_unprepare(d->core_clk); + reset_control_assert(d->rst); +} + +static const struct ldma_inst_data dma0 = { + .name = "dma0", + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = false, + .valid_desc_fetch_ack = false, +}; + +static const struct ldma_inst_data dma2tx = { + .name = "dma2tx", + .type = DMA_TYPE_TX, + .orrc = 16, + .chan_fc = true, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data dma1rx = { + .name = "dma1rx", + .type = DMA_TYPE_RX, + .orrc = 16, + .chan_fc = false, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = false, +}; + +static const struct ldma_inst_data dma1tx = { + .name = "dma1tx", + .type = DMA_TYPE_TX, + .orrc = 16, + .chan_fc = true, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data dma0tx = { + .name = "dma0tx", + .type = DMA_TYPE_TX, + .orrc = 16, + .chan_fc = true, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data dma3 = { + .name = "dma3", + .type = DMA_TYPE_MCPY, + .orrc = 16, + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = true, + .valid_desc_fetch_ack = false, +}; + +static const struct ldma_inst_data toe_dma30 = { + .name = "toe_dma30", + .type = DMA_TYPE_MCPY, + .orrc = 16, + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data toe_dma31 = { + .name = "toe_dma31", + .type = DMA_TYPE_MCPY, + .orrc = 16, + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct of_device_id intel_ldma_match[] = { + { .compatible = "intel,lgm-cdma", .data = &dma0}, + { .compatible = "intel,lgm-dma2tx", .data = &dma2tx}, + { .compatible = "intel,lgm-dma1rx", .data = &dma1rx}, + { .compatible = "intel,lgm-dma1tx", .data = &dma1tx}, + { .compatible = "intel,lgm-dma0tx", .data = &dma0tx}, + { .compatible = "intel,lgm-dma3", .data = &dma3}, + { .compatible = "intel,lgm-toe-dma30", .data = &toe_dma30}, + { .compatible = "intel,lgm-toe-dma31", .data = &toe_dma31}, + {} +}; + +static int intel_ldma_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dma_device *dma_dev; + unsigned long ch_mask; + struct ldma_chan *c; + struct ldma_port *p; + struct ldma_dev *d; + u32 id, bitn = 32, j; + int i, ret; + + d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL); + if (!d) + return -ENOMEM; + + /* Link controller to platform device */ + d->dev = &pdev->dev; + + d->inst = device_get_match_data(dev); + if (!d->inst) { + dev_err(dev, "No device match found\n"); + return -ENODEV; + } + + d->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(d->base)) + return PTR_ERR(d->base); + + /* Power up and reset the dma engine, some DMAs always on?? */ + d->core_clk = devm_clk_get_optional(dev, NULL); + if (IS_ERR(d->core_clk)) + return PTR_ERR(d->core_clk); + clk_prepare_enable(d->core_clk); + + d->rst = devm_reset_control_get_optional(dev, NULL); + if (IS_ERR(d->rst)) + return PTR_ERR(d->rst); + reset_control_deassert(d->rst); + + ret = devm_add_action_or_reset(dev, ldma_clk_disable, d); + if (ret) { + dev_err(dev, "Failed to devm_add_action_or_reset, %d\n", ret); + return ret; + } + + id = readl(d->base + DMA_ID); + d->chan_nrs = FIELD_GET(DMA_ID_CHNR, id); + d->port_nrs = FIELD_GET(DMA_ID_PNR, id); + d->ver = FIELD_GET(DMA_ID_REV, id); + + if (id & DMA_ID_AW_36B) + d->flags |= DMA_ADDR_36BIT; + + if (IS_ENABLED(CONFIG_64BIT) && (id & DMA_ID_AW_36B)) + bitn = 36; + + if (id & DMA_ID_DW_128B) + d->flags |= DMA_DATA_128BIT; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(bitn)); + if (ret) { + dev_err(dev, "No usable DMA configuration\n"); + return ret; + } + + if (d->ver == DMA_VER22) { + ret = ldma_init_v22(d, pdev); + if (ret) + return ret; + } + + ret = device_property_read_u32(dev, "dma-channel-mask", &d->channels_mask); + if (ret < 0) + d->channels_mask = GENMASK(d->chan_nrs - 1, 0); + + dma_dev = &d->dma_dev; + + dma_cap_zero(dma_dev->cap_mask); + dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); + + /* Channel initializations */ + INIT_LIST_HEAD(&dma_dev->channels); + + /* Port Initializations */ + d->ports = devm_kcalloc(dev, d->port_nrs, sizeof(*p), GFP_KERNEL); + if (!d->ports) + return -ENOMEM; + + /* Channels Initializations */ + d->chans = devm_kcalloc(d->dev, d->chan_nrs, sizeof(*c), GFP_KERNEL); + if (!d->chans) + return -ENOMEM; + + for (i = 0; i < d->port_nrs; i++) { + p = &d->ports[i]; + p->portid = i; + p->ldev = d; + } + + ret = ldma_cfg_init(d); + if (ret) + return ret; + + dma_dev->dev = &pdev->dev; + + ch_mask = (unsigned long)d->channels_mask; + for_each_set_bit(j, &ch_mask, d->chan_nrs) { + if (d->ver == DMA_VER22) + ldma_dma_init_v22(j, d); + else + ldma_dma_init_v3X(j, d); + } + + dma_dev->device_alloc_chan_resources = ldma_alloc_chan_resources; + dma_dev->device_free_chan_resources = ldma_free_chan_resources; + dma_dev->device_terminate_all = ldma_terminate_all; + dma_dev->device_issue_pending = ldma_issue_pending; + dma_dev->device_tx_status = ldma_tx_status; + dma_dev->device_resume = ldma_resume_chan; + dma_dev->device_pause = ldma_pause_chan; + dma_dev->device_prep_slave_sg = ldma_prep_slave_sg; + + if (d->ver == DMA_VER22) { + dma_dev->device_config = ldma_slave_config; + dma_dev->device_synchronize = ldma_synchronize; + dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + dma_dev->directions = BIT(DMA_MEM_TO_DEV) | + BIT(DMA_DEV_TO_MEM); + dma_dev->residue_granularity = + DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + } + + platform_set_drvdata(pdev, d); + + ldma_dev_init(d); + + ret = dma_async_device_register(dma_dev); + if (ret) { + dev_err(dev, "Failed to register slave DMA engine device\n"); + return ret; + } + + ret = of_dma_controller_register(pdev->dev.of_node, ldma_xlate, d); + if (ret) { + dev_err(dev, "Failed to register of DMA controller\n"); + dma_async_device_unregister(dma_dev); + return ret; + } + + dev_info(dev, "Init done - rev: %x, ports: %d channels: %d\n", d->ver, + d->port_nrs, d->chan_nrs); + + return 0; +} + +static struct platform_driver intel_ldma_driver = { + .probe = intel_ldma_probe, + .driver = { + .name = DRIVER_NAME, + .of_match_table = intel_ldma_match, + }, +}; + +/* + * Perform this driver as device_initcall to make sure initialization happens + * before its DMA clients of some are platform specific and also to provide + * registered DMA channels and DMA capabilities to clients before their + * initialization. + */ +static int __init intel_ldma_init(void) +{ + return platform_driver_register(&intel_ldma_driver); +} + +device_initcall(intel_ldma_init); diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index b84303be8edf..89f1814ff27a 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -18,7 +18,6 @@ #include #include #include -#include #include "dmaengine.h" @@ -1148,19 +1147,6 @@ static struct platform_driver mmp_pdma_driver = { .remove = mmp_pdma_remove, }; -bool mmp_pdma_filter_fn(struct dma_chan *chan, void *param) -{ - struct mmp_pdma_chan *c = to_mmp_pdma_chan(chan); - - if (chan->device->dev->driver != &mmp_pdma_driver.driver) - return false; - - c->drcmr = *(unsigned int *)param; - - return true; -} -EXPORT_SYMBOL_GPL(mmp_pdma_filter_fn); - module_platform_driver(mmp_pdma_driver); MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver"); diff --git a/drivers/dma/owl-dma.c b/drivers/dma/owl-dma.c index 9fede32641e9..1f0bbaed4643 100644 --- a/drivers/dma/owl-dma.c +++ b/drivers/dma/owl-dma.c @@ -1080,8 +1080,9 @@ static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec, } static const struct of_device_id owl_dma_match[] = { - { .compatible = "actions,s900-dma", .data = (void *)S900_DMA,}, + { .compatible = "actions,s500-dma", .data = (void *)S900_DMA,}, { .compatible = "actions,s700-dma", .data = (void *)S700_DMA,}, + { .compatible = "actions,s900-dma", .data = (void *)S900_DMA,}, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, owl_dma_match); @@ -1245,6 +1246,7 @@ static int owl_dma_remove(struct platform_device *pdev) owl_dma_free(od); clk_disable_unprepare(od->clk); + dma_pool_destroy(od->lli_pool); return 0; } diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index bc0f66af0f11..fd8d2bc3be9f 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -3195,7 +3195,7 @@ probe_err2: return ret; } -static int pl330_remove(struct amba_device *adev) +static void pl330_remove(struct amba_device *adev) { struct pl330_dmac *pl330 = amba_get_drvdata(adev); struct dma_pl330_chan *pch, *_p; @@ -3235,7 +3235,6 @@ static int pl330_remove(struct amba_device *adev) if (pl330->rstc) reset_control_assert(pl330->rstc); - return 0; } static const struct amba_id pl330_ids[] = { diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 88579857ca1d..c8a77b428b52 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -1270,13 +1270,13 @@ static int bam_dma_probe(struct platform_device *pdev) dev_err(bdev->dev, "num-ees unspecified in dt\n"); } - bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); - if (IS_ERR(bdev->bamclk)) { - if (!bdev->controlled_remotely) - return PTR_ERR(bdev->bamclk); + if (bdev->controlled_remotely) + bdev->bamclk = devm_clk_get_optional(bdev->dev, "bam_clk"); + else + bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); - bdev->bamclk = NULL; - } + if (IS_ERR(bdev->bamclk)) + return PTR_ERR(bdev->bamclk); ret = clk_prepare_enable(bdev->bamclk); if (ret) { @@ -1350,7 +1350,7 @@ static int bam_dma_probe(struct platform_device *pdev) if (ret) goto err_unregister_dma; - if (bdev->controlled_remotely) { + if (!bdev->bamclk) { pm_runtime_disable(&pdev->dev); return 0; } @@ -1438,10 +1438,10 @@ static int __maybe_unused bam_dma_suspend(struct device *dev) { struct bam_device *bdev = dev_get_drvdata(dev); - if (!bdev->controlled_remotely) + if (bdev->bamclk) { pm_runtime_force_suspend(dev); - - clk_unprepare(bdev->bamclk); + clk_unprepare(bdev->bamclk); + } return 0; } @@ -1451,12 +1451,13 @@ static int __maybe_unused bam_dma_resume(struct device *dev) struct bam_device *bdev = dev_get_drvdata(dev); int ret; - ret = clk_prepare(bdev->bamclk); - if (ret) - return ret; + if (bdev->bamclk) { + ret = clk_prepare(bdev->bamclk); + if (ret) + return ret; - if (!bdev->controlled_remotely) pm_runtime_force_resume(dev); + } return 0; } diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index 1a0bf6b0567a..57f5ee4235c7 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -584,7 +584,7 @@ static inline void gpi_write_reg_field(struct gpii *gpii, void __iomem *addr, gpi_write_reg(gpii, addr, val); } -static inline void +static __always_inline void gpi_update_reg(struct gpii *gpii, u32 offset, u32 mask, u32 val) { void __iomem *addr = gpii->regs + offset; @@ -1700,7 +1700,7 @@ static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc, tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); - }; + } for (i = 0; i < tre_idx; i++) dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0], diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index a57705356e8b..d530c1bf11d9 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -189,7 +189,8 @@ struct rcar_dmac_chan { * struct rcar_dmac - R-Car Gen2 DMA Controller * @engine: base DMA engine object * @dev: the hardware device - * @iomem: remapped I/O memory base + * @dmac_base: remapped base register block + * @chan_base: remapped channel register block (optional) * @n_channels: number of available channels * @channels: array of DMAC channels * @channels_mask: bitfield of which DMA channels are managed by this driver @@ -198,7 +199,8 @@ struct rcar_dmac_chan { struct rcar_dmac { struct dma_device engine; struct device *dev; - void __iomem *iomem; + void __iomem *dmac_base; + void __iomem *chan_base; unsigned int n_channels; struct rcar_dmac_chan *channels; @@ -209,6 +211,10 @@ struct rcar_dmac { #define to_rcar_dmac(d) container_of(d, struct rcar_dmac, engine) +#define for_each_rcar_dmac_chan(i, dmac, chan) \ + for (i = 0, chan = &(dmac)->channels[0]; i < (dmac)->n_channels; i++, chan++) \ + if (!((dmac)->channels_mask & BIT(i))) continue; else + /* * struct rcar_dmac_of_data - This driver's OF data * @chan_offset_base: DMAC channels base offset @@ -230,7 +236,7 @@ struct rcar_dmac_of_data { #define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8) #define RCAR_DMAOR_AE (1 << 2) #define RCAR_DMAOR_DME (1 << 0) -#define RCAR_DMACHCLR 0x0080 +#define RCAR_DMACHCLR 0x0080 /* Not on R-Car V3U */ #define RCAR_DMADPSEC 0x00a0 #define RCAR_DMASAR 0x0000 @@ -293,6 +299,9 @@ struct rcar_dmac_of_data { #define RCAR_DMAFIXDAR 0x0014 #define RCAR_DMAFIXDPBASE 0x0060 +/* For R-Car V3U */ +#define RCAR_V3U_DMACHCLR 0x0100 + /* Hardcode the MEMCPY transfer size to 4 bytes. */ #define RCAR_DMAC_MEMCPY_XFER_SIZE 4 @@ -303,17 +312,17 @@ struct rcar_dmac_of_data { static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data) { if (reg == RCAR_DMAOR) - writew(data, dmac->iomem + reg); + writew(data, dmac->dmac_base + reg); else - writel(data, dmac->iomem + reg); + writel(data, dmac->dmac_base + reg); } static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg) { if (reg == RCAR_DMAOR) - return readw(dmac->iomem + reg); + return readw(dmac->dmac_base + reg); else - return readl(dmac->iomem + reg); + return readl(dmac->dmac_base + reg); } static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg) @@ -332,6 +341,28 @@ static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data) writel(data, chan->iomem + reg); } +static void rcar_dmac_chan_clear(struct rcar_dmac *dmac, + struct rcar_dmac_chan *chan) +{ + if (dmac->chan_base) + rcar_dmac_chan_write(chan, RCAR_V3U_DMACHCLR, 1); + else + rcar_dmac_write(dmac, RCAR_DMACHCLR, BIT(chan->index)); +} + +static void rcar_dmac_chan_clear_all(struct rcar_dmac *dmac) +{ + struct rcar_dmac_chan *chan; + unsigned int i; + + if (dmac->chan_base) { + for_each_rcar_dmac_chan(i, dmac, chan) + rcar_dmac_chan_write(chan, RCAR_V3U_DMACHCLR, 1); + } else { + rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask); + } +} + /* ----------------------------------------------------------------------------- * Initialization and configuration */ @@ -447,7 +478,7 @@ static int rcar_dmac_init(struct rcar_dmac *dmac) u16 dmaor; /* Clear all channels and enable the DMAC globally. */ - rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask); + rcar_dmac_chan_clear_all(dmac); rcar_dmac_write(dmac, RCAR_DMAOR, RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME); @@ -817,15 +848,11 @@ static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan) static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac) { + struct rcar_dmac_chan *chan; unsigned int i; /* Stop all channels. */ - for (i = 0; i < dmac->n_channels; ++i) { - struct rcar_dmac_chan *chan = &dmac->channels[i]; - - if (!(dmac->channels_mask & BIT(i))) - continue; - + for_each_rcar_dmac_chan(i, dmac, chan) { /* Stop and reinitialize the channel. */ spin_lock_irq(&chan->lock); rcar_dmac_chan_halt(chan); @@ -1566,7 +1593,7 @@ static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev) * because channel is already stopped in error case. * We need to clear register and check DE bit as recovery. */ - rcar_dmac_write(dmac, RCAR_DMACHCLR, 1 << chan->index); + rcar_dmac_chan_clear(dmac, chan); rcar_dmac_chcr_de_barrier(chan); reinit = true; goto spin_lock_end; @@ -1732,9 +1759,7 @@ static const struct dev_pm_ops rcar_dmac_pm = { */ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, - struct rcar_dmac_chan *rchan, - const struct rcar_dmac_of_data *data, - unsigned int index) + struct rcar_dmac_chan *rchan) { struct platform_device *pdev = to_platform_device(dmac->dev); struct dma_chan *chan = &rchan->chan; @@ -1742,9 +1767,6 @@ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, char *irqname; int ret; - rchan->index = index; - rchan->iomem = dmac->iomem + data->chan_offset_base + - data->chan_offset_stride * index; rchan->mid_rid = -EINVAL; spin_lock_init(&rchan->lock); @@ -1756,13 +1778,13 @@ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, INIT_LIST_HEAD(&rchan->desc.wait); /* Request the channel interrupt. */ - sprintf(pdev_irqname, "ch%u", index); + sprintf(pdev_irqname, "ch%u", rchan->index); rchan->irq = platform_get_irq_byname(pdev, pdev_irqname); if (rchan->irq < 0) return -ENODEV; irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", - dev_name(dmac->dev), index); + dev_name(dmac->dev), rchan->index); if (!irqname) return -ENOMEM; @@ -1828,9 +1850,11 @@ static int rcar_dmac_probe(struct platform_device *pdev) DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES; - struct dma_device *engine; - struct rcar_dmac *dmac; const struct rcar_dmac_of_data *data; + struct rcar_dmac_chan *chan; + struct dma_device *engine; + void __iomem *chan_base; + struct rcar_dmac *dmac; unsigned int i; int ret; @@ -1868,9 +1892,24 @@ static int rcar_dmac_probe(struct platform_device *pdev) return -ENOMEM; /* Request resources. */ - dmac->iomem = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(dmac->iomem)) - return PTR_ERR(dmac->iomem); + dmac->dmac_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dmac->dmac_base)) + return PTR_ERR(dmac->dmac_base); + + if (!data->chan_offset_base) { + dmac->chan_base = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(dmac->chan_base)) + return PTR_ERR(dmac->chan_base); + + chan_base = dmac->chan_base; + } else { + chan_base = dmac->dmac_base + data->chan_offset_base; + } + + for_each_rcar_dmac_chan(i, dmac, chan) { + chan->index = i; + chan->iomem = chan_base + i * data->chan_offset_stride; + } /* Enable runtime PM and initialize the device. */ pm_runtime_enable(&pdev->dev); @@ -1916,11 +1955,8 @@ static int rcar_dmac_probe(struct platform_device *pdev) INIT_LIST_HEAD(&engine->channels); - for (i = 0; i < dmac->n_channels; ++i) { - if (!(dmac->channels_mask & BIT(i))) - continue; - - ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], data, i); + for_each_rcar_dmac_chan(i, dmac, chan) { + ret = rcar_dmac_chan_probe(dmac, chan); if (ret < 0) goto error; } @@ -1968,14 +2004,22 @@ static void rcar_dmac_shutdown(struct platform_device *pdev) } static const struct rcar_dmac_of_data rcar_dmac_data = { - .chan_offset_base = 0x8000, - .chan_offset_stride = 0x80, + .chan_offset_base = 0x8000, + .chan_offset_stride = 0x80, +}; + +static const struct rcar_dmac_of_data rcar_v3u_dmac_data = { + .chan_offset_base = 0x0, + .chan_offset_stride = 0x1000, }; static const struct of_device_id rcar_dmac_of_ids[] = { { .compatible = "renesas,rcar-dmac", .data = &rcar_dmac_data, + }, { + .compatible = "renesas,dmac-r8a779a0", + .data = &rcar_v3u_dmac_data, }, { /* Sentinel */ } }; diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c deleted file mode 100644 index a5c2843384fd..000000000000 --- a/drivers/dma/sirf-dma.c +++ /dev/null @@ -1,1170 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * DMA controller driver for CSR SiRFprimaII - * - * Copyright (c) 2011 Cambridge Silicon Radio Limited, a CSR plc group company. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dmaengine.h" - -#define SIRFSOC_DMA_VER_A7V1 1 -#define SIRFSOC_DMA_VER_A7V2 2 -#define SIRFSOC_DMA_VER_A6 4 - -#define SIRFSOC_DMA_DESCRIPTORS 16 -#define SIRFSOC_DMA_CHANNELS 16 -#define SIRFSOC_DMA_TABLE_NUM 256 - -#define SIRFSOC_DMA_CH_ADDR 0x00 -#define SIRFSOC_DMA_CH_XLEN 0x04 -#define SIRFSOC_DMA_CH_YLEN 0x08 -#define SIRFSOC_DMA_CH_CTRL 0x0C - -#define SIRFSOC_DMA_WIDTH_0 0x100 -#define SIRFSOC_DMA_CH_VALID 0x140 -#define SIRFSOC_DMA_CH_INT 0x144 -#define SIRFSOC_DMA_INT_EN 0x148 -#define SIRFSOC_DMA_INT_EN_CLR 0x14C -#define SIRFSOC_DMA_CH_LOOP_CTRL 0x150 -#define SIRFSOC_DMA_CH_LOOP_CTRL_CLR 0x154 -#define SIRFSOC_DMA_WIDTH_ATLAS7 0x10 -#define SIRFSOC_DMA_VALID_ATLAS7 0x14 -#define SIRFSOC_DMA_INT_ATLAS7 0x18 -#define SIRFSOC_DMA_INT_EN_ATLAS7 0x1c -#define SIRFSOC_DMA_LOOP_CTRL_ATLAS7 0x20 -#define SIRFSOC_DMA_CUR_DATA_ADDR 0x34 -#define SIRFSOC_DMA_MUL_ATLAS7 0x38 -#define SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7 0x158 -#define SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7 0x15C -#define SIRFSOC_DMA_IOBG_SCMD_EN 0x800 -#define SIRFSOC_DMA_EARLY_RESP_SET 0x818 -#define SIRFSOC_DMA_EARLY_RESP_CLR 0x81C - -#define SIRFSOC_DMA_MODE_CTRL_BIT 4 -#define SIRFSOC_DMA_DIR_CTRL_BIT 5 -#define SIRFSOC_DMA_MODE_CTRL_BIT_ATLAS7 2 -#define SIRFSOC_DMA_CHAIN_CTRL_BIT_ATLAS7 3 -#define SIRFSOC_DMA_DIR_CTRL_BIT_ATLAS7 4 -#define SIRFSOC_DMA_TAB_NUM_ATLAS7 7 -#define SIRFSOC_DMA_CHAIN_INT_BIT_ATLAS7 5 -#define SIRFSOC_DMA_CHAIN_FLAG_SHIFT_ATLAS7 25 -#define SIRFSOC_DMA_CHAIN_ADDR_SHIFT 32 - -#define SIRFSOC_DMA_INT_FINI_INT_ATLAS7 BIT(0) -#define SIRFSOC_DMA_INT_CNT_INT_ATLAS7 BIT(1) -#define SIRFSOC_DMA_INT_PAU_INT_ATLAS7 BIT(2) -#define SIRFSOC_DMA_INT_LOOP_INT_ATLAS7 BIT(3) -#define SIRFSOC_DMA_INT_INV_INT_ATLAS7 BIT(4) -#define SIRFSOC_DMA_INT_END_INT_ATLAS7 BIT(5) -#define SIRFSOC_DMA_INT_ALL_ATLAS7 0x3F - -/* xlen and dma_width register is in 4 bytes boundary */ -#define SIRFSOC_DMA_WORD_LEN 4 -#define SIRFSOC_DMA_XLEN_MAX_V1 0x800 -#define SIRFSOC_DMA_XLEN_MAX_V2 0x1000 - -struct sirfsoc_dma_desc { - struct dma_async_tx_descriptor desc; - struct list_head node; - - /* SiRFprimaII 2D-DMA parameters */ - - int xlen; /* DMA xlen */ - int ylen; /* DMA ylen */ - int width; /* DMA width */ - int dir; - bool cyclic; /* is loop DMA? */ - bool chain; /* is chain DMA? */ - u32 addr; /* DMA buffer address */ - u64 chain_table[SIRFSOC_DMA_TABLE_NUM]; /* chain tbl */ -}; - -struct sirfsoc_dma_chan { - struct dma_chan chan; - struct list_head free; - struct list_head prepared; - struct list_head queued; - struct list_head active; - struct list_head completed; - unsigned long happened_cyclic; - unsigned long completed_cyclic; - - /* Lock for this structure */ - spinlock_t lock; - - int mode; -}; - -struct sirfsoc_dma_regs { - u32 ctrl[SIRFSOC_DMA_CHANNELS]; - u32 interrupt_en; -}; - -struct sirfsoc_dma { - struct dma_device dma; - struct tasklet_struct tasklet; - struct sirfsoc_dma_chan channels[SIRFSOC_DMA_CHANNELS]; - void __iomem *base; - int irq; - struct clk *clk; - int type; - void (*exec_desc)(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base); - struct sirfsoc_dma_regs regs_save; -}; - -struct sirfsoc_dmadata { - void (*exec)(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base); - int type; -}; - -enum sirfsoc_dma_chain_flag { - SIRFSOC_DMA_CHAIN_NORMAL = 0x01, - SIRFSOC_DMA_CHAIN_PAUSE = 0x02, - SIRFSOC_DMA_CHAIN_LOOP = 0x03, - SIRFSOC_DMA_CHAIN_END = 0x04 -}; - -#define DRV_NAME "sirfsoc_dma" - -static int sirfsoc_dma_runtime_suspend(struct device *dev); - -/* Convert struct dma_chan to struct sirfsoc_dma_chan */ -static inline -struct sirfsoc_dma_chan *dma_chan_to_sirfsoc_dma_chan(struct dma_chan *c) -{ - return container_of(c, struct sirfsoc_dma_chan, chan); -} - -/* Convert struct dma_chan to struct sirfsoc_dma */ -static inline struct sirfsoc_dma *dma_chan_to_sirfsoc_dma(struct dma_chan *c) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(c); - return container_of(schan, struct sirfsoc_dma, channels[c->chan_id]); -} - -static void sirfsoc_dma_execute_hw_a7v2(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base) -{ - if (sdesc->chain) { - /* DMA v2 HW chain mode */ - writel_relaxed((sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT_ATLAS7) | - (sdesc->chain << - SIRFSOC_DMA_CHAIN_CTRL_BIT_ATLAS7) | - (0x8 << SIRFSOC_DMA_TAB_NUM_ATLAS7) | 0x3, - base + SIRFSOC_DMA_CH_CTRL); - } else { - /* DMA v2 legacy mode */ - writel_relaxed(sdesc->xlen, base + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, base + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(sdesc->width, base + SIRFSOC_DMA_WIDTH_ATLAS7); - writel_relaxed((sdesc->width*((sdesc->ylen+1)>>1)), - base + SIRFSOC_DMA_MUL_ATLAS7); - writel_relaxed((sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT_ATLAS7) | - (sdesc->chain << - SIRFSOC_DMA_CHAIN_CTRL_BIT_ATLAS7) | - 0x3, base + SIRFSOC_DMA_CH_CTRL); - } - writel_relaxed(sdesc->chain ? SIRFSOC_DMA_INT_END_INT_ATLAS7 : - (SIRFSOC_DMA_INT_FINI_INT_ATLAS7 | - SIRFSOC_DMA_INT_LOOP_INT_ATLAS7), - base + SIRFSOC_DMA_INT_EN_ATLAS7); - writel(sdesc->addr, base + SIRFSOC_DMA_CH_ADDR); - if (sdesc->cyclic) - writel(0x10001, base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); -} - -static void sirfsoc_dma_execute_hw_a7v1(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base) -{ - writel_relaxed(1, base + SIRFSOC_DMA_IOBG_SCMD_EN); - writel_relaxed((1 << cid), base + SIRFSOC_DMA_EARLY_RESP_SET); - writel_relaxed(sdesc->width, base + SIRFSOC_DMA_WIDTH_0 + cid * 4); - writel_relaxed(cid | (burst_mode << SIRFSOC_DMA_MODE_CTRL_BIT) | - (sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT), - base + cid * 0x10 + SIRFSOC_DMA_CH_CTRL); - writel_relaxed(sdesc->xlen, base + cid * 0x10 + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, base + cid * 0x10 + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(readl_relaxed(base + SIRFSOC_DMA_INT_EN) | - (1 << cid), base + SIRFSOC_DMA_INT_EN); - writel(sdesc->addr >> 2, base + cid * 0x10 + SIRFSOC_DMA_CH_ADDR); - if (sdesc->cyclic) { - writel((1 << cid) | 1 << (cid + 16) | - readl_relaxed(base + SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7), - base + SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7); - } - -} - -static void sirfsoc_dma_execute_hw_a6(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base) -{ - writel_relaxed(sdesc->width, base + SIRFSOC_DMA_WIDTH_0 + cid * 4); - writel_relaxed(cid | (burst_mode << SIRFSOC_DMA_MODE_CTRL_BIT) | - (sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT), - base + cid * 0x10 + SIRFSOC_DMA_CH_CTRL); - writel_relaxed(sdesc->xlen, base + cid * 0x10 + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, base + cid * 0x10 + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(readl_relaxed(base + SIRFSOC_DMA_INT_EN) | - (1 << cid), base + SIRFSOC_DMA_INT_EN); - writel(sdesc->addr >> 2, base + cid * 0x10 + SIRFSOC_DMA_CH_ADDR); - if (sdesc->cyclic) { - writel((1 << cid) | 1 << (cid + 16) | - readl_relaxed(base + SIRFSOC_DMA_CH_LOOP_CTRL), - base + SIRFSOC_DMA_CH_LOOP_CTRL); - } - -} - -/* Execute all queued DMA descriptors */ -static void sirfsoc_dma_execute(struct sirfsoc_dma_chan *schan) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - struct sirfsoc_dma_desc *sdesc = NULL; - void __iomem *base; - - /* - * lock has been held by functions calling this, so we don't hold - * lock again - */ - base = sdma->base; - sdesc = list_first_entry(&schan->queued, struct sirfsoc_dma_desc, - node); - /* Move the first queued descriptor to active list */ - list_move_tail(&sdesc->node, &schan->active); - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) - cid = 0; - - /* Start the DMA transfer */ - sdma->exec_desc(sdesc, cid, schan->mode, base); - - if (sdesc->cyclic) - schan->happened_cyclic = schan->completed_cyclic = 0; -} - -/* Interrupt handler */ -static irqreturn_t sirfsoc_dma_irq(int irq, void *data) -{ - struct sirfsoc_dma *sdma = data; - struct sirfsoc_dma_chan *schan; - struct sirfsoc_dma_desc *sdesc = NULL; - u32 is; - bool chain; - int ch; - void __iomem *reg; - - switch (sdma->type) { - case SIRFSOC_DMA_VER_A6: - case SIRFSOC_DMA_VER_A7V1: - is = readl(sdma->base + SIRFSOC_DMA_CH_INT); - reg = sdma->base + SIRFSOC_DMA_CH_INT; - while ((ch = fls(is) - 1) >= 0) { - is &= ~(1 << ch); - writel_relaxed(1 << ch, reg); - schan = &sdma->channels[ch]; - spin_lock(&schan->lock); - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, node); - if (!sdesc->cyclic) { - /* Execute queued descriptors */ - list_splice_tail_init(&schan->active, - &schan->completed); - dma_cookie_complete(&sdesc->desc); - if (!list_empty(&schan->queued)) - sirfsoc_dma_execute(schan); - } else - schan->happened_cyclic++; - spin_unlock(&schan->lock); - } - break; - - case SIRFSOC_DMA_VER_A7V2: - is = readl(sdma->base + SIRFSOC_DMA_INT_ATLAS7); - - reg = sdma->base + SIRFSOC_DMA_INT_ATLAS7; - writel_relaxed(SIRFSOC_DMA_INT_ALL_ATLAS7, reg); - schan = &sdma->channels[0]; - spin_lock(&schan->lock); - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, node); - if (!sdesc->cyclic) { - chain = sdesc->chain; - if ((chain && (is & SIRFSOC_DMA_INT_END_INT_ATLAS7)) || - (!chain && - (is & SIRFSOC_DMA_INT_FINI_INT_ATLAS7))) { - /* Execute queued descriptors */ - list_splice_tail_init(&schan->active, - &schan->completed); - dma_cookie_complete(&sdesc->desc); - if (!list_empty(&schan->queued)) - sirfsoc_dma_execute(schan); - } - } else if (sdesc->cyclic && (is & - SIRFSOC_DMA_INT_LOOP_INT_ATLAS7)) - schan->happened_cyclic++; - - spin_unlock(&schan->lock); - break; - - default: - break; - } - - /* Schedule tasklet */ - tasklet_schedule(&sdma->tasklet); - - return IRQ_HANDLED; -} - -/* process completed descriptors */ -static void sirfsoc_dma_process_completed(struct sirfsoc_dma *sdma) -{ - dma_cookie_t last_cookie = 0; - struct sirfsoc_dma_chan *schan; - struct sirfsoc_dma_desc *sdesc; - struct dma_async_tx_descriptor *desc; - unsigned long flags; - unsigned long happened_cyclic; - LIST_HEAD(list); - int i; - - for (i = 0; i < sdma->dma.chancnt; i++) { - schan = &sdma->channels[i]; - - /* Get all completed descriptors */ - spin_lock_irqsave(&schan->lock, flags); - if (!list_empty(&schan->completed)) { - list_splice_tail_init(&schan->completed, &list); - spin_unlock_irqrestore(&schan->lock, flags); - - /* Execute callbacks and run dependencies */ - list_for_each_entry(sdesc, &list, node) { - desc = &sdesc->desc; - - dmaengine_desc_get_callback_invoke(desc, NULL); - last_cookie = desc->cookie; - dma_run_dependencies(desc); - } - - /* Free descriptors */ - spin_lock_irqsave(&schan->lock, flags); - list_splice_tail_init(&list, &schan->free); - schan->chan.completed_cookie = last_cookie; - spin_unlock_irqrestore(&schan->lock, flags); - } else { - if (list_empty(&schan->active)) { - spin_unlock_irqrestore(&schan->lock, flags); - continue; - } - - /* for cyclic channel, desc is always in active list */ - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, node); - - /* cyclic DMA */ - happened_cyclic = schan->happened_cyclic; - spin_unlock_irqrestore(&schan->lock, flags); - - desc = &sdesc->desc; - while (happened_cyclic != schan->completed_cyclic) { - dmaengine_desc_get_callback_invoke(desc, NULL); - schan->completed_cyclic++; - } - } - } -} - -/* DMA Tasklet */ -static void sirfsoc_dma_tasklet(struct tasklet_struct *t) -{ - struct sirfsoc_dma *sdma = from_tasklet(sdma, t, tasklet); - - sirfsoc_dma_process_completed(sdma); -} - -/* Submit descriptor to hardware */ -static dma_cookie_t sirfsoc_dma_tx_submit(struct dma_async_tx_descriptor *txd) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(txd->chan); - struct sirfsoc_dma_desc *sdesc; - unsigned long flags; - dma_cookie_t cookie; - - sdesc = container_of(txd, struct sirfsoc_dma_desc, desc); - - spin_lock_irqsave(&schan->lock, flags); - - /* Move descriptor to queue */ - list_move_tail(&sdesc->node, &schan->queued); - - cookie = dma_cookie_assign(txd); - - spin_unlock_irqrestore(&schan->lock, flags); - - return cookie; -} - -static int sirfsoc_dma_slave_config(struct dma_chan *chan, - struct dma_slave_config *config) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - unsigned long flags; - - if ((config->src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) || - (config->dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)) - return -EINVAL; - - spin_lock_irqsave(&schan->lock, flags); - schan->mode = (config->src_maxburst == 4 ? 1 : 0); - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -static int sirfsoc_dma_terminate_all(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - - switch (sdma->type) { - case SIRFSOC_DMA_VER_A7V1: - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_INT_EN_CLR); - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_INT); - writel_relaxed((1 << cid) | 1 << (cid + 16), - sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7); - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_VALID); - break; - case SIRFSOC_DMA_VER_A7V2: - writel_relaxed(0, sdma->base + SIRFSOC_DMA_INT_EN_ATLAS7); - writel_relaxed(SIRFSOC_DMA_INT_ALL_ATLAS7, - sdma->base + SIRFSOC_DMA_INT_ATLAS7); - writel_relaxed(0, sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); - writel_relaxed(0, sdma->base + SIRFSOC_DMA_VALID_ATLAS7); - break; - case SIRFSOC_DMA_VER_A6: - writel_relaxed(readl_relaxed(sdma->base + SIRFSOC_DMA_INT_EN) & - ~(1 << cid), sdma->base + SIRFSOC_DMA_INT_EN); - writel_relaxed(readl_relaxed(sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL) & - ~((1 << cid) | 1 << (cid + 16)), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL); - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_VALID); - break; - default: - break; - } - - list_splice_tail_init(&schan->active, &schan->free); - list_splice_tail_init(&schan->queued, &schan->free); - - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -static int sirfsoc_dma_pause_chan(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - - switch (sdma->type) { - case SIRFSOC_DMA_VER_A7V1: - writel_relaxed((1 << cid) | 1 << (cid + 16), - sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7); - break; - case SIRFSOC_DMA_VER_A7V2: - writel_relaxed(0, sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); - break; - case SIRFSOC_DMA_VER_A6: - writel_relaxed(readl_relaxed(sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL) & - ~((1 << cid) | 1 << (cid + 16)), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL); - break; - - default: - break; - } - - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -static int sirfsoc_dma_resume_chan(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - switch (sdma->type) { - case SIRFSOC_DMA_VER_A7V1: - writel_relaxed((1 << cid) | 1 << (cid + 16), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7); - break; - case SIRFSOC_DMA_VER_A7V2: - writel_relaxed(0x10001, - sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); - break; - case SIRFSOC_DMA_VER_A6: - writel_relaxed(readl_relaxed(sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL) | - ((1 << cid) | 1 << (cid + 16)), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL); - break; - - default: - break; - } - - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -/* Alloc channel resources */ -static int sirfsoc_dma_alloc_chan_resources(struct dma_chan *chan) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma_desc *sdesc; - unsigned long flags; - LIST_HEAD(descs); - int i; - - pm_runtime_get_sync(sdma->dma.dev); - - /* Alloc descriptors for this channel */ - for (i = 0; i < SIRFSOC_DMA_DESCRIPTORS; i++) { - sdesc = kzalloc(sizeof(*sdesc), GFP_KERNEL); - if (!sdesc) { - dev_notice(sdma->dma.dev, "Memory allocation error. " - "Allocated only %u descriptors\n", i); - break; - } - - dma_async_tx_descriptor_init(&sdesc->desc, chan); - sdesc->desc.flags = DMA_CTRL_ACK; - sdesc->desc.tx_submit = sirfsoc_dma_tx_submit; - - list_add_tail(&sdesc->node, &descs); - } - - /* Return error only if no descriptors were allocated */ - if (i == 0) - return -ENOMEM; - - spin_lock_irqsave(&schan->lock, flags); - - list_splice_tail_init(&descs, &schan->free); - spin_unlock_irqrestore(&schan->lock, flags); - - return i; -} - -/* Free channel resources */ -static void sirfsoc_dma_free_chan_resources(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_desc *sdesc, *tmp; - unsigned long flags; - LIST_HEAD(descs); - - spin_lock_irqsave(&schan->lock, flags); - - /* Channel must be idle */ - BUG_ON(!list_empty(&schan->prepared)); - BUG_ON(!list_empty(&schan->queued)); - BUG_ON(!list_empty(&schan->active)); - BUG_ON(!list_empty(&schan->completed)); - - /* Move data */ - list_splice_tail_init(&schan->free, &descs); - - spin_unlock_irqrestore(&schan->lock, flags); - - /* Free descriptors */ - list_for_each_entry_safe(sdesc, tmp, &descs, node) - kfree(sdesc); - - pm_runtime_put(sdma->dma.dev); -} - -/* Send pending descriptor to hardware */ -static void sirfsoc_dma_issue_pending(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - - if (list_empty(&schan->active) && !list_empty(&schan->queued)) - sirfsoc_dma_execute(schan); - - spin_unlock_irqrestore(&schan->lock, flags); -} - -/* Check request completion status */ -static enum dma_status -sirfsoc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - unsigned long flags; - enum dma_status ret; - struct sirfsoc_dma_desc *sdesc; - int cid = schan->chan.chan_id; - unsigned long dma_pos; - unsigned long dma_request_bytes; - unsigned long residue; - - spin_lock_irqsave(&schan->lock, flags); - - if (list_empty(&schan->active)) { - ret = dma_cookie_status(chan, cookie, txstate); - dma_set_residue(txstate, 0); - spin_unlock_irqrestore(&schan->lock, flags); - return ret; - } - sdesc = list_first_entry(&schan->active, struct sirfsoc_dma_desc, node); - if (sdesc->cyclic) - dma_request_bytes = (sdesc->xlen + 1) * (sdesc->ylen + 1) * - (sdesc->width * SIRFSOC_DMA_WORD_LEN); - else - dma_request_bytes = sdesc->xlen * SIRFSOC_DMA_WORD_LEN; - - ret = dma_cookie_status(chan, cookie, txstate); - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) - cid = 0; - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - dma_pos = readl_relaxed(sdma->base + SIRFSOC_DMA_CUR_DATA_ADDR); - } else { - dma_pos = readl_relaxed( - sdma->base + cid * 0x10 + SIRFSOC_DMA_CH_ADDR) << 2; - } - - residue = dma_request_bytes - (dma_pos - sdesc->addr); - dma_set_residue(txstate, residue); - - spin_unlock_irqrestore(&schan->lock, flags); - - return ret; -} - -static struct dma_async_tx_descriptor *sirfsoc_dma_prep_interleaved( - struct dma_chan *chan, struct dma_interleaved_template *xt, - unsigned long flags) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma_desc *sdesc = NULL; - unsigned long iflags; - int ret; - - if ((xt->dir != DMA_MEM_TO_DEV) && (xt->dir != DMA_DEV_TO_MEM)) { - ret = -EINVAL; - goto err_dir; - } - - /* Get free descriptor */ - spin_lock_irqsave(&schan->lock, iflags); - if (!list_empty(&schan->free)) { - sdesc = list_first_entry(&schan->free, struct sirfsoc_dma_desc, - node); - list_del(&sdesc->node); - } - spin_unlock_irqrestore(&schan->lock, iflags); - - if (!sdesc) { - /* try to free completed descriptors */ - sirfsoc_dma_process_completed(sdma); - ret = 0; - goto no_desc; - } - - /* Place descriptor in prepared list */ - spin_lock_irqsave(&schan->lock, iflags); - - /* - * Number of chunks in a frame can only be 1 for prima2 - * and ylen (number of frame - 1) must be at least 0 - */ - if ((xt->frame_size == 1) && (xt->numf > 0)) { - sdesc->cyclic = 0; - sdesc->xlen = xt->sgl[0].size / SIRFSOC_DMA_WORD_LEN; - sdesc->width = (xt->sgl[0].size + xt->sgl[0].icg) / - SIRFSOC_DMA_WORD_LEN; - sdesc->ylen = xt->numf - 1; - if (xt->dir == DMA_MEM_TO_DEV) { - sdesc->addr = xt->src_start; - sdesc->dir = 1; - } else { - sdesc->addr = xt->dst_start; - sdesc->dir = 0; - } - - list_add_tail(&sdesc->node, &schan->prepared); - } else { - pr_err("sirfsoc DMA Invalid xfer\n"); - ret = -EINVAL; - goto err_xfer; - } - spin_unlock_irqrestore(&schan->lock, iflags); - - return &sdesc->desc; -err_xfer: - spin_unlock_irqrestore(&schan->lock, iflags); -no_desc: -err_dir: - return ERR_PTR(ret); -} - -static struct dma_async_tx_descriptor * -sirfsoc_dma_prep_cyclic(struct dma_chan *chan, dma_addr_t addr, - size_t buf_len, size_t period_len, - enum dma_transfer_direction direction, unsigned long flags) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma_desc *sdesc = NULL; - unsigned long iflags; - - /* - * we only support cycle transfer with 2 period - * If the X-length is set to 0, it would be the loop mode. - * The DMA address keeps increasing until reaching the end of a loop - * area whose size is defined by (DMA_WIDTH x (Y_LENGTH + 1)). Then - * the DMA address goes back to the beginning of this area. - * In loop mode, the DMA data region is divided into two parts, BUFA - * and BUFB. DMA controller generates interrupts twice in each loop: - * when the DMA address reaches the end of BUFA or the end of the - * BUFB - */ - if (buf_len != 2 * period_len) - return ERR_PTR(-EINVAL); - - /* Get free descriptor */ - spin_lock_irqsave(&schan->lock, iflags); - if (!list_empty(&schan->free)) { - sdesc = list_first_entry(&schan->free, struct sirfsoc_dma_desc, - node); - list_del(&sdesc->node); - } - spin_unlock_irqrestore(&schan->lock, iflags); - - if (!sdesc) - return NULL; - - /* Place descriptor in prepared list */ - spin_lock_irqsave(&schan->lock, iflags); - sdesc->addr = addr; - sdesc->cyclic = 1; - sdesc->xlen = 0; - sdesc->ylen = buf_len / SIRFSOC_DMA_WORD_LEN - 1; - sdesc->width = 1; - list_add_tail(&sdesc->node, &schan->prepared); - spin_unlock_irqrestore(&schan->lock, iflags); - - return &sdesc->desc; -} - -/* - * The DMA controller consists of 16 independent DMA channels. - * Each channel is allocated to a different function - */ -bool sirfsoc_dma_filter_id(struct dma_chan *chan, void *chan_id) -{ - unsigned int ch_nr = (unsigned int) chan_id; - - if (ch_nr == chan->chan_id + - chan->device->dev_id * SIRFSOC_DMA_CHANNELS) - return true; - - return false; -} -EXPORT_SYMBOL(sirfsoc_dma_filter_id); - -#define SIRFSOC_DMA_BUSWIDTHS \ - (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ - BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ - BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) - -static struct dma_chan *of_dma_sirfsoc_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct sirfsoc_dma *sdma = ofdma->of_dma_data; - unsigned int request = dma_spec->args[0]; - - if (request >= SIRFSOC_DMA_CHANNELS) - return NULL; - - return dma_get_slave_channel(&sdma->channels[request].chan); -} - -static int sirfsoc_dma_probe(struct platform_device *op) -{ - struct device_node *dn = op->dev.of_node; - struct device *dev = &op->dev; - struct dma_device *dma; - struct sirfsoc_dma *sdma; - struct sirfsoc_dma_chan *schan; - struct sirfsoc_dmadata *data; - struct resource res; - ulong regs_start, regs_size; - u32 id; - int ret, i; - - sdma = devm_kzalloc(dev, sizeof(*sdma), GFP_KERNEL); - if (!sdma) - return -ENOMEM; - - data = (struct sirfsoc_dmadata *) - (of_match_device(op->dev.driver->of_match_table, - &op->dev)->data); - sdma->exec_desc = data->exec; - sdma->type = data->type; - - if (of_property_read_u32(dn, "cell-index", &id)) { - dev_err(dev, "Fail to get DMAC index\n"); - return -ENODEV; - } - - sdma->irq = irq_of_parse_and_map(dn, 0); - if (!sdma->irq) { - dev_err(dev, "Error mapping IRQ!\n"); - return -EINVAL; - } - - sdma->clk = devm_clk_get(dev, NULL); - if (IS_ERR(sdma->clk)) { - dev_err(dev, "failed to get a clock.\n"); - return PTR_ERR(sdma->clk); - } - - ret = of_address_to_resource(dn, 0, &res); - if (ret) { - dev_err(dev, "Error parsing memory region!\n"); - goto irq_dispose; - } - - regs_start = res.start; - regs_size = resource_size(&res); - - sdma->base = devm_ioremap(dev, regs_start, regs_size); - if (!sdma->base) { - dev_err(dev, "Error mapping memory region!\n"); - ret = -ENOMEM; - goto irq_dispose; - } - - ret = request_irq(sdma->irq, &sirfsoc_dma_irq, 0, DRV_NAME, sdma); - if (ret) { - dev_err(dev, "Error requesting IRQ!\n"); - ret = -EINVAL; - goto irq_dispose; - } - - dma = &sdma->dma; - dma->dev = dev; - - dma->device_alloc_chan_resources = sirfsoc_dma_alloc_chan_resources; - dma->device_free_chan_resources = sirfsoc_dma_free_chan_resources; - dma->device_issue_pending = sirfsoc_dma_issue_pending; - dma->device_config = sirfsoc_dma_slave_config; - dma->device_pause = sirfsoc_dma_pause_chan; - dma->device_resume = sirfsoc_dma_resume_chan; - dma->device_terminate_all = sirfsoc_dma_terminate_all; - dma->device_tx_status = sirfsoc_dma_tx_status; - dma->device_prep_interleaved_dma = sirfsoc_dma_prep_interleaved; - dma->device_prep_dma_cyclic = sirfsoc_dma_prep_cyclic; - dma->src_addr_widths = SIRFSOC_DMA_BUSWIDTHS; - dma->dst_addr_widths = SIRFSOC_DMA_BUSWIDTHS; - dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); - - INIT_LIST_HEAD(&dma->channels); - dma_cap_set(DMA_SLAVE, dma->cap_mask); - dma_cap_set(DMA_CYCLIC, dma->cap_mask); - dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); - dma_cap_set(DMA_PRIVATE, dma->cap_mask); - - for (i = 0; i < SIRFSOC_DMA_CHANNELS; i++) { - schan = &sdma->channels[i]; - - schan->chan.device = dma; - dma_cookie_init(&schan->chan); - - INIT_LIST_HEAD(&schan->free); - INIT_LIST_HEAD(&schan->prepared); - INIT_LIST_HEAD(&schan->queued); - INIT_LIST_HEAD(&schan->active); - INIT_LIST_HEAD(&schan->completed); - - spin_lock_init(&schan->lock); - list_add_tail(&schan->chan.device_node, &dma->channels); - } - - tasklet_setup(&sdma->tasklet, sirfsoc_dma_tasklet); - - /* Register DMA engine */ - dev_set_drvdata(dev, sdma); - - ret = dma_async_device_register(dma); - if (ret) - goto free_irq; - - /* Device-tree DMA controller registration */ - ret = of_dma_controller_register(dn, of_dma_sirfsoc_xlate, sdma); - if (ret) { - dev_err(dev, "failed to register DMA controller\n"); - goto unreg_dma_dev; - } - - pm_runtime_enable(&op->dev); - dev_info(dev, "initialized SIRFSOC DMAC driver\n"); - - return 0; - -unreg_dma_dev: - dma_async_device_unregister(dma); -free_irq: - free_irq(sdma->irq, sdma); -irq_dispose: - irq_dispose_mapping(sdma->irq); - return ret; -} - -static int sirfsoc_dma_remove(struct platform_device *op) -{ - struct device *dev = &op->dev; - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - - of_dma_controller_free(op->dev.of_node); - dma_async_device_unregister(&sdma->dma); - free_irq(sdma->irq, sdma); - tasklet_kill(&sdma->tasklet); - irq_dispose_mapping(sdma->irq); - pm_runtime_disable(&op->dev); - if (!pm_runtime_status_suspended(&op->dev)) - sirfsoc_dma_runtime_suspend(&op->dev); - - return 0; -} - -static int __maybe_unused sirfsoc_dma_runtime_suspend(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - - clk_disable_unprepare(sdma->clk); - return 0; -} - -static int __maybe_unused sirfsoc_dma_runtime_resume(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - int ret; - - ret = clk_prepare_enable(sdma->clk); - if (ret < 0) { - dev_err(dev, "clk_enable failed: %d\n", ret); - return ret; - } - return 0; -} - -static int __maybe_unused sirfsoc_dma_pm_suspend(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - struct sirfsoc_dma_regs *save = &sdma->regs_save; - struct sirfsoc_dma_chan *schan; - int ch; - int ret; - int count; - u32 int_offset; - - /* - * if we were runtime-suspended before, resume to enable clock - * before accessing register - */ - if (pm_runtime_status_suspended(dev)) { - ret = sirfsoc_dma_runtime_resume(dev); - if (ret < 0) - return ret; - } - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - count = 1; - int_offset = SIRFSOC_DMA_INT_EN_ATLAS7; - } else { - count = SIRFSOC_DMA_CHANNELS; - int_offset = SIRFSOC_DMA_INT_EN; - } - - /* - * DMA controller will lose all registers while suspending - * so we need to save registers for active channels - */ - for (ch = 0; ch < count; ch++) { - schan = &sdma->channels[ch]; - if (list_empty(&schan->active)) - continue; - save->ctrl[ch] = readl_relaxed(sdma->base + - ch * 0x10 + SIRFSOC_DMA_CH_CTRL); - } - save->interrupt_en = readl_relaxed(sdma->base + int_offset); - - /* Disable clock */ - sirfsoc_dma_runtime_suspend(dev); - - return 0; -} - -static int __maybe_unused sirfsoc_dma_pm_resume(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - struct sirfsoc_dma_regs *save = &sdma->regs_save; - struct sirfsoc_dma_desc *sdesc; - struct sirfsoc_dma_chan *schan; - int ch; - int ret; - int count; - u32 int_offset; - u32 width_offset; - - /* Enable clock before accessing register */ - ret = sirfsoc_dma_runtime_resume(dev); - if (ret < 0) - return ret; - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - count = 1; - int_offset = SIRFSOC_DMA_INT_EN_ATLAS7; - width_offset = SIRFSOC_DMA_WIDTH_ATLAS7; - } else { - count = SIRFSOC_DMA_CHANNELS; - int_offset = SIRFSOC_DMA_INT_EN; - width_offset = SIRFSOC_DMA_WIDTH_0; - } - - writel_relaxed(save->interrupt_en, sdma->base + int_offset); - for (ch = 0; ch < count; ch++) { - schan = &sdma->channels[ch]; - if (list_empty(&schan->active)) - continue; - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, - node); - writel_relaxed(sdesc->width, - sdma->base + width_offset + ch * 4); - writel_relaxed(sdesc->xlen, - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(save->ctrl[ch], - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_CTRL); - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - writel_relaxed(sdesc->addr, - sdma->base + SIRFSOC_DMA_CH_ADDR); - } else { - writel_relaxed(sdesc->addr >> 2, - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_ADDR); - - } - } - - /* if we were runtime-suspended before, suspend again */ - if (pm_runtime_status_suspended(dev)) - sirfsoc_dma_runtime_suspend(dev); - - return 0; -} - -static const struct dev_pm_ops sirfsoc_dma_pm_ops = { - SET_RUNTIME_PM_OPS(sirfsoc_dma_runtime_suspend, sirfsoc_dma_runtime_resume, NULL) - SET_SYSTEM_SLEEP_PM_OPS(sirfsoc_dma_pm_suspend, sirfsoc_dma_pm_resume) -}; - -static struct sirfsoc_dmadata sirfsoc_dmadata_a6 = { - .exec = sirfsoc_dma_execute_hw_a6, - .type = SIRFSOC_DMA_VER_A6, -}; - -static struct sirfsoc_dmadata sirfsoc_dmadata_a7v1 = { - .exec = sirfsoc_dma_execute_hw_a7v1, - .type = SIRFSOC_DMA_VER_A7V1, -}; - -static struct sirfsoc_dmadata sirfsoc_dmadata_a7v2 = { - .exec = sirfsoc_dma_execute_hw_a7v2, - .type = SIRFSOC_DMA_VER_A7V2, -}; - -static const struct of_device_id sirfsoc_dma_match[] = { - { .compatible = "sirf,prima2-dmac", .data = &sirfsoc_dmadata_a6,}, - { .compatible = "sirf,atlas7-dmac", .data = &sirfsoc_dmadata_a7v1,}, - { .compatible = "sirf,atlas7-dmac-v2", .data = &sirfsoc_dmadata_a7v2,}, - {}, -}; -MODULE_DEVICE_TABLE(of, sirfsoc_dma_match); - -static struct platform_driver sirfsoc_dma_driver = { - .probe = sirfsoc_dma_probe, - .remove = sirfsoc_dma_remove, - .driver = { - .name = DRV_NAME, - .pm = &sirfsoc_dma_pm_ops, - .of_match_table = sirfsoc_dma_match, - }, -}; - -static __init int sirfsoc_dma_init(void) -{ - return platform_driver_register(&sirfsoc_dma_driver); -} - -static void __exit sirfsoc_dma_exit(void) -{ - platform_driver_unregister(&sirfsoc_dma_driver); -} - -subsys_initcall(sirfsoc_dma_init); -module_exit(sirfsoc_dma_exit); - -MODULE_AUTHOR("Rongjun Ying "); -MODULE_AUTHOR("Barry Song "); -MODULE_DESCRIPTION("SIRFSOC DMA control driver"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 4256e55bbf25..265d7c07b348 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -78,7 +78,7 @@ static int dma40_memcpy_channels[] = { DB8500_DMA_MEMCPY_EV_5, }; -/* Default configuration for physcial memcpy */ +/* Default configuration for physical memcpy */ static const struct stedma40_chan_cfg dma40_memcpy_conf_phy = { .mode = STEDMA40_MODE_PHYSICAL, .dir = DMA_MEM_TO_MEM, diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index f474a1232335..96ad21869ba7 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -121,6 +121,11 @@ struct udma_oes_offsets { #define UDMA_FLAG_PDMA_ACC32 BIT(0) #define UDMA_FLAG_PDMA_BURST BIT(1) #define UDMA_FLAG_TDTYPE BIT(2) +#define UDMA_FLAG_BURST_SIZE BIT(3) +#define UDMA_FLAGS_J7_CLASS (UDMA_FLAG_PDMA_ACC32 | \ + UDMA_FLAG_PDMA_BURST | \ + UDMA_FLAG_TDTYPE | \ + UDMA_FLAG_BURST_SIZE) struct udma_match_data { enum k3_dma_type type; @@ -128,6 +133,7 @@ struct udma_match_data { bool enable_memcpy_support; u32 flags; u32 statictr_z_mask; + u8 burst_size[3]; }; struct udma_soc_data { @@ -436,6 +442,18 @@ static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel) } } +static u8 udma_get_chan_tpl_index(struct udma_tpl *tpl_map, int chan_id) +{ + int i; + + for (i = 0; i < tpl_map->levels; i++) { + if (chan_id >= tpl_map->start_idx[i]) + return i; + } + + return 0; +} + static void udma_reset_uchan(struct udma_chan *uc) { memset(&uc->config, 0, sizeof(uc->config)); @@ -1811,13 +1829,21 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct udma_tchan *tchan = uc->tchan; struct udma_rchan *rchan = uc->rchan; - int ret = 0; + u8 burst_size = 0; + int ret; + u8 tpl; /* Non synchronized - mem to mem type of transfer */ int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; + if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) { + tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, tchan->id); + + burst_size = ud->match_data->burst_size[tpl]; + } + req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; req_tx.index = tchan->id; @@ -1825,6 +1851,10 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; req_tx.txcq_qnum = tc_ring; req_tx.tx_atype = ud->atype; + if (burst_size) { + req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; + req_tx.tx_burst_size = burst_size; + } ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) { @@ -1839,6 +1869,10 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) req_rx.rxcq_qnum = tc_ring; req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; req_rx.rx_atype = ud->atype; + if (burst_size) { + req_rx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; + req_rx.rx_burst_size = burst_size; + } ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); if (ret) @@ -1854,12 +1888,24 @@ static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; struct udma_bchan *bchan = uc->bchan; - int ret = 0; + u8 burst_size = 0; + int ret; + u8 tpl; + + if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) { + tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, bchan->id); + + burst_size = ud->match_data->burst_size[tpl]; + } req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN; req_tx.index = bchan->id; + if (burst_size) { + req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; + req_tx.tx_burst_size = burst_size; + } ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) @@ -1877,7 +1923,7 @@ static int udma_tisci_tx_channel_config(struct udma_chan *uc) int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; u32 mode, fetch_size; - int ret = 0; + int ret; if (uc->config.pkt_mode) { mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; @@ -1918,7 +1964,7 @@ static int bcdma_tisci_tx_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct udma_tchan *tchan = uc->tchan; struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; - int ret = 0; + int ret; req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; @@ -1951,7 +1997,7 @@ static int udma_tisci_rx_channel_config(struct udma_chan *uc) struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; u32 mode, fetch_size; - int ret = 0; + int ret; if (uc->config.pkt_mode) { mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; @@ -2028,7 +2074,7 @@ static int bcdma_tisci_rx_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct udma_rchan *rchan = uc->rchan; struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; - int ret = 0; + int ret; req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; req_rx.nav_id = tisci_rm->tisci_dev_id; @@ -2048,7 +2094,7 @@ static int pktdma_tisci_rx_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; - int ret = 0; + int ret; req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; req_rx.nav_id = tisci_rm->tisci_dev_id; @@ -4168,6 +4214,11 @@ static struct udma_match_data am654_main_data = { .psil_base = 0x1000, .enable_memcpy_support = true, .statictr_z_mask = GENMASK(11, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data am654_mcu_data = { @@ -4175,38 +4226,63 @@ static struct udma_match_data am654_mcu_data = { .psil_base = 0x6000, .enable_memcpy_support = false, .statictr_z_mask = GENMASK(11, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data j721e_main_data = { .type = DMA_TYPE_UDMA, .psil_base = 0x1000, .enable_memcpy_support = true, - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* H Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* UH Channels */ + }, }; static struct udma_match_data j721e_mcu_data = { .type = DMA_TYPE_UDMA, .psil_base = 0x6000, .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES, /* H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data am64_bcdma_data = { .type = DMA_TYPE_BCDMA, .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */ .enable_memcpy_support = true, /* Supported via bchan */ - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data am64_pktdma_data = { .type = DMA_TYPE_PKTDMA, .psil_base = 0x1000, .enable_memcpy_support = false, /* PKTDMA does not support MEM_TO_MEM */ - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, }; static const struct of_device_id udma_of_match[] = { @@ -4306,6 +4382,7 @@ static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2); ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2); ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2); + ud->rflow_cnt = ud->rchan_cnt; break; case DMA_TYPE_PKTDMA: cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30); @@ -5046,6 +5123,34 @@ static void udma_dbg_summary_show(struct seq_file *s, } #endif /* CONFIG_DEBUG_FS */ +static enum dmaengine_alignment udma_get_copy_align(struct udma_dev *ud) +{ + const struct udma_match_data *match_data = ud->match_data; + u8 tpl; + + if (!match_data->enable_memcpy_support) + return DMAENGINE_ALIGN_8_BYTES; + + /* Get the highest TPL level the device supports for memcpy */ + if (ud->bchan_cnt) + tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, 0); + else if (ud->tchan_cnt) + tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, 0); + else + return DMAENGINE_ALIGN_8_BYTES; + + switch (match_data->burst_size[tpl]) { + case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES: + return DMAENGINE_ALIGN_256_BYTES; + case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES: + return DMAENGINE_ALIGN_128_BYTES; + case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES: + fallthrough; + default: + return DMAENGINE_ALIGN_64_BYTES; + } +} + #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ @@ -5202,7 +5307,6 @@ static int udma_probe(struct platform_device *pdev) ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; - ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES; ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | DESC_METADATA_ENGINE; if (ud->match_data->enable_memcpy_support && @@ -5284,6 +5388,9 @@ static int udma_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion); } + /* Configure the copy_align to the maximum burst size the device supports */ + ud->ddev.copy_align = udma_get_copy_align(ud); + ret = dma_async_device_register(&ud->ddev); if (ret) { dev_err(dev, "failed to register slave DMA engine: %d\n", ret); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 79777550a6ff..3aded7861fef 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -800,7 +800,7 @@ xilinx_dma_alloc_tx_descriptor(struct xilinx_dma_chan *chan) { struct xilinx_dma_tx_descriptor *desc; - desc = kzalloc(sizeof(*desc), GFP_KERNEL); + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); if (!desc) return NULL; diff --git a/drivers/dma/zx_dma.c b/drivers/dma/zx_dma.c deleted file mode 100644 index b057582b2fac..000000000000 --- a/drivers/dma/zx_dma.c +++ /dev/null @@ -1,941 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2015 Linaro. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "virt-dma.h" - -#define DRIVER_NAME "zx-dma" -#define DMA_ALIGN 4 -#define DMA_MAX_SIZE (0x10000 - 512) -#define LLI_BLOCK_SIZE (4 * PAGE_SIZE) - -#define REG_ZX_SRC_ADDR 0x00 -#define REG_ZX_DST_ADDR 0x04 -#define REG_ZX_TX_X_COUNT 0x08 -#define REG_ZX_TX_ZY_COUNT 0x0c -#define REG_ZX_SRC_ZY_STEP 0x10 -#define REG_ZX_DST_ZY_STEP 0x14 -#define REG_ZX_LLI_ADDR 0x1c -#define REG_ZX_CTRL 0x20 -#define REG_ZX_TC_IRQ 0x800 -#define REG_ZX_SRC_ERR_IRQ 0x804 -#define REG_ZX_DST_ERR_IRQ 0x808 -#define REG_ZX_CFG_ERR_IRQ 0x80c -#define REG_ZX_TC_IRQ_RAW 0x810 -#define REG_ZX_SRC_ERR_IRQ_RAW 0x814 -#define REG_ZX_DST_ERR_IRQ_RAW 0x818 -#define REG_ZX_CFG_ERR_IRQ_RAW 0x81c -#define REG_ZX_STATUS 0x820 -#define REG_ZX_DMA_GRP_PRIO 0x824 -#define REG_ZX_DMA_ARB 0x828 - -#define ZX_FORCE_CLOSE BIT(31) -#define ZX_DST_BURST_WIDTH(x) (((x) & 0x7) << 13) -#define ZX_MAX_BURST_LEN 16 -#define ZX_SRC_BURST_LEN(x) (((x) & 0xf) << 9) -#define ZX_SRC_BURST_WIDTH(x) (((x) & 0x7) << 6) -#define ZX_IRQ_ENABLE_ALL (3 << 4) -#define ZX_DST_FIFO_MODE BIT(3) -#define ZX_SRC_FIFO_MODE BIT(2) -#define ZX_SOFT_REQ BIT(1) -#define ZX_CH_ENABLE BIT(0) - -#define ZX_DMA_BUSWIDTHS \ - (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ - BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ - BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) - -enum zx_dma_burst_width { - ZX_DMA_WIDTH_8BIT = 0, - ZX_DMA_WIDTH_16BIT = 1, - ZX_DMA_WIDTH_32BIT = 2, - ZX_DMA_WIDTH_64BIT = 3, -}; - -struct zx_desc_hw { - u32 saddr; - u32 daddr; - u32 src_x; - u32 src_zy; - u32 src_zy_step; - u32 dst_zy_step; - u32 reserved1; - u32 lli; - u32 ctr; - u32 reserved[7]; /* pack as hardware registers region size */ -} __aligned(32); - -struct zx_dma_desc_sw { - struct virt_dma_desc vd; - dma_addr_t desc_hw_lli; - size_t desc_num; - size_t size; - struct zx_desc_hw *desc_hw; -}; - -struct zx_dma_phy; - -struct zx_dma_chan { - struct dma_slave_config slave_cfg; - int id; /* Request phy chan id */ - u32 ccfg; - u32 cyclic; - struct virt_dma_chan vc; - struct zx_dma_phy *phy; - struct list_head node; - dma_addr_t dev_addr; - enum dma_status status; -}; - -struct zx_dma_phy { - u32 idx; - void __iomem *base; - struct zx_dma_chan *vchan; - struct zx_dma_desc_sw *ds_run; - struct zx_dma_desc_sw *ds_done; -}; - -struct zx_dma_dev { - struct dma_device slave; - void __iomem *base; - spinlock_t lock; /* lock for ch and phy */ - struct list_head chan_pending; - struct zx_dma_phy *phy; - struct zx_dma_chan *chans; - struct clk *clk; - struct dma_pool *pool; - u32 dma_channels; - u32 dma_requests; - int irq; -}; - -#define to_zx_dma(dmadev) container_of(dmadev, struct zx_dma_dev, slave) - -static struct zx_dma_chan *to_zx_chan(struct dma_chan *chan) -{ - return container_of(chan, struct zx_dma_chan, vc.chan); -} - -static void zx_dma_terminate_chan(struct zx_dma_phy *phy, struct zx_dma_dev *d) -{ - u32 val = 0; - - val = readl_relaxed(phy->base + REG_ZX_CTRL); - val &= ~ZX_CH_ENABLE; - val |= ZX_FORCE_CLOSE; - writel_relaxed(val, phy->base + REG_ZX_CTRL); - - val = 0x1 << phy->idx; - writel_relaxed(val, d->base + REG_ZX_TC_IRQ_RAW); - writel_relaxed(val, d->base + REG_ZX_SRC_ERR_IRQ_RAW); - writel_relaxed(val, d->base + REG_ZX_DST_ERR_IRQ_RAW); - writel_relaxed(val, d->base + REG_ZX_CFG_ERR_IRQ_RAW); -} - -static void zx_dma_set_desc(struct zx_dma_phy *phy, struct zx_desc_hw *hw) -{ - writel_relaxed(hw->saddr, phy->base + REG_ZX_SRC_ADDR); - writel_relaxed(hw->daddr, phy->base + REG_ZX_DST_ADDR); - writel_relaxed(hw->src_x, phy->base + REG_ZX_TX_X_COUNT); - writel_relaxed(0, phy->base + REG_ZX_TX_ZY_COUNT); - writel_relaxed(0, phy->base + REG_ZX_SRC_ZY_STEP); - writel_relaxed(0, phy->base + REG_ZX_DST_ZY_STEP); - writel_relaxed(hw->lli, phy->base + REG_ZX_LLI_ADDR); - writel_relaxed(hw->ctr, phy->base + REG_ZX_CTRL); -} - -static u32 zx_dma_get_curr_lli(struct zx_dma_phy *phy) -{ - return readl_relaxed(phy->base + REG_ZX_LLI_ADDR); -} - -static u32 zx_dma_get_chan_stat(struct zx_dma_dev *d) -{ - return readl_relaxed(d->base + REG_ZX_STATUS); -} - -static void zx_dma_init_state(struct zx_dma_dev *d) -{ - /* set same priority */ - writel_relaxed(0x0, d->base + REG_ZX_DMA_ARB); - /* clear all irq */ - writel_relaxed(0xffffffff, d->base + REG_ZX_TC_IRQ_RAW); - writel_relaxed(0xffffffff, d->base + REG_ZX_SRC_ERR_IRQ_RAW); - writel_relaxed(0xffffffff, d->base + REG_ZX_DST_ERR_IRQ_RAW); - writel_relaxed(0xffffffff, d->base + REG_ZX_CFG_ERR_IRQ_RAW); -} - -static int zx_dma_start_txd(struct zx_dma_chan *c) -{ - struct zx_dma_dev *d = to_zx_dma(c->vc.chan.device); - struct virt_dma_desc *vd = vchan_next_desc(&c->vc); - - if (!c->phy) - return -EAGAIN; - - if (BIT(c->phy->idx) & zx_dma_get_chan_stat(d)) - return -EAGAIN; - - if (vd) { - struct zx_dma_desc_sw *ds = - container_of(vd, struct zx_dma_desc_sw, vd); - /* - * fetch and remove request from vc->desc_issued - * so vc->desc_issued only contains desc pending - */ - list_del(&ds->vd.node); - c->phy->ds_run = ds; - c->phy->ds_done = NULL; - /* start dma */ - zx_dma_set_desc(c->phy, ds->desc_hw); - return 0; - } - c->phy->ds_done = NULL; - c->phy->ds_run = NULL; - return -EAGAIN; -} - -static void zx_dma_task(struct zx_dma_dev *d) -{ - struct zx_dma_phy *p; - struct zx_dma_chan *c, *cn; - unsigned pch, pch_alloc = 0; - unsigned long flags; - - /* check new dma request of running channel in vc->desc_issued */ - list_for_each_entry_safe(c, cn, &d->slave.channels, - vc.chan.device_node) { - spin_lock_irqsave(&c->vc.lock, flags); - p = c->phy; - if (p && p->ds_done && zx_dma_start_txd(c)) { - /* No current txd associated with this channel */ - dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx); - /* Mark this channel free */ - c->phy = NULL; - p->vchan = NULL; - } - spin_unlock_irqrestore(&c->vc.lock, flags); - } - - /* check new channel request in d->chan_pending */ - spin_lock_irqsave(&d->lock, flags); - while (!list_empty(&d->chan_pending)) { - c = list_first_entry(&d->chan_pending, - struct zx_dma_chan, node); - p = &d->phy[c->id]; - if (!p->vchan) { - /* remove from d->chan_pending */ - list_del_init(&c->node); - pch_alloc |= 1 << c->id; - /* Mark this channel allocated */ - p->vchan = c; - c->phy = p; - } else { - dev_dbg(d->slave.dev, "pchan %u: busy!\n", c->id); - } - } - spin_unlock_irqrestore(&d->lock, flags); - - for (pch = 0; pch < d->dma_channels; pch++) { - if (pch_alloc & (1 << pch)) { - p = &d->phy[pch]; - c = p->vchan; - if (c) { - spin_lock_irqsave(&c->vc.lock, flags); - zx_dma_start_txd(c); - spin_unlock_irqrestore(&c->vc.lock, flags); - } - } - } -} - -static irqreturn_t zx_dma_int_handler(int irq, void *dev_id) -{ - struct zx_dma_dev *d = (struct zx_dma_dev *)dev_id; - struct zx_dma_phy *p; - struct zx_dma_chan *c; - u32 tc = readl_relaxed(d->base + REG_ZX_TC_IRQ); - u32 serr = readl_relaxed(d->base + REG_ZX_SRC_ERR_IRQ); - u32 derr = readl_relaxed(d->base + REG_ZX_DST_ERR_IRQ); - u32 cfg = readl_relaxed(d->base + REG_ZX_CFG_ERR_IRQ); - u32 i, irq_chan = 0, task = 0; - - while (tc) { - i = __ffs(tc); - tc &= ~BIT(i); - p = &d->phy[i]; - c = p->vchan; - if (c) { - spin_lock(&c->vc.lock); - if (c->cyclic) { - vchan_cyclic_callback(&p->ds_run->vd); - } else { - vchan_cookie_complete(&p->ds_run->vd); - p->ds_done = p->ds_run; - task = 1; - } - spin_unlock(&c->vc.lock); - irq_chan |= BIT(i); - } - } - - if (serr || derr || cfg) - dev_warn(d->slave.dev, "DMA ERR src 0x%x, dst 0x%x, cfg 0x%x\n", - serr, derr, cfg); - - writel_relaxed(irq_chan, d->base + REG_ZX_TC_IRQ_RAW); - writel_relaxed(serr, d->base + REG_ZX_SRC_ERR_IRQ_RAW); - writel_relaxed(derr, d->base + REG_ZX_DST_ERR_IRQ_RAW); - writel_relaxed(cfg, d->base + REG_ZX_CFG_ERR_IRQ_RAW); - - if (task) - zx_dma_task(d); - return IRQ_HANDLED; -} - -static void zx_dma_free_chan_resources(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_dev *d = to_zx_dma(chan->device); - unsigned long flags; - - spin_lock_irqsave(&d->lock, flags); - list_del_init(&c->node); - spin_unlock_irqrestore(&d->lock, flags); - - vchan_free_chan_resources(&c->vc); - c->ccfg = 0; -} - -static enum dma_status zx_dma_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, - struct dma_tx_state *state) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_phy *p; - struct virt_dma_desc *vd; - unsigned long flags; - enum dma_status ret; - size_t bytes = 0; - - ret = dma_cookie_status(&c->vc.chan, cookie, state); - if (ret == DMA_COMPLETE || !state) - return ret; - - spin_lock_irqsave(&c->vc.lock, flags); - p = c->phy; - ret = c->status; - - /* - * If the cookie is on our issue queue, then the residue is - * its total size. - */ - vd = vchan_find_desc(&c->vc, cookie); - if (vd) { - bytes = container_of(vd, struct zx_dma_desc_sw, vd)->size; - } else if ((!p) || (!p->ds_run)) { - bytes = 0; - } else { - struct zx_dma_desc_sw *ds = p->ds_run; - u32 clli = 0, index = 0; - - bytes = 0; - clli = zx_dma_get_curr_lli(p); - index = (clli - ds->desc_hw_lli) / - sizeof(struct zx_desc_hw) + 1; - for (; index < ds->desc_num; index++) { - bytes += ds->desc_hw[index].src_x; - /* end of lli */ - if (!ds->desc_hw[index].lli) - break; - } - } - spin_unlock_irqrestore(&c->vc.lock, flags); - dma_set_residue(state, bytes); - return ret; -} - -static void zx_dma_issue_pending(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_dev *d = to_zx_dma(chan->device); - unsigned long flags; - int issue = 0; - - spin_lock_irqsave(&c->vc.lock, flags); - /* add request to vc->desc_issued */ - if (vchan_issue_pending(&c->vc)) { - spin_lock(&d->lock); - if (!c->phy && list_empty(&c->node)) { - /* if new channel, add chan_pending */ - list_add_tail(&c->node, &d->chan_pending); - issue = 1; - dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc); - } - spin_unlock(&d->lock); - } else { - dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc); - } - spin_unlock_irqrestore(&c->vc.lock, flags); - - if (issue) - zx_dma_task(d); -} - -static void zx_dma_fill_desc(struct zx_dma_desc_sw *ds, dma_addr_t dst, - dma_addr_t src, size_t len, u32 num, u32 ccfg) -{ - if ((num + 1) < ds->desc_num) - ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) * - sizeof(struct zx_desc_hw); - ds->desc_hw[num].saddr = src; - ds->desc_hw[num].daddr = dst; - ds->desc_hw[num].src_x = len; - ds->desc_hw[num].ctr = ccfg; -} - -static struct zx_dma_desc_sw *zx_alloc_desc_resource(int num, - struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - struct zx_dma_dev *d = to_zx_dma(chan->device); - int lli_limit = LLI_BLOCK_SIZE / sizeof(struct zx_desc_hw); - - if (num > lli_limit) { - dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n", - &c->vc, num, lli_limit); - return NULL; - } - - ds = kzalloc(sizeof(*ds), GFP_ATOMIC); - if (!ds) - return NULL; - - ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli); - if (!ds->desc_hw) { - dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc); - kfree(ds); - return NULL; - } - ds->desc_num = num; - return ds; -} - -static enum zx_dma_burst_width zx_dma_burst_width(enum dma_slave_buswidth width) -{ - switch (width) { - case DMA_SLAVE_BUSWIDTH_1_BYTE: - case DMA_SLAVE_BUSWIDTH_2_BYTES: - case DMA_SLAVE_BUSWIDTH_4_BYTES: - case DMA_SLAVE_BUSWIDTH_8_BYTES: - return ffs(width) - 1; - default: - return ZX_DMA_WIDTH_32BIT; - } -} - -static int zx_pre_config(struct zx_dma_chan *c, enum dma_transfer_direction dir) -{ - struct dma_slave_config *cfg = &c->slave_cfg; - enum zx_dma_burst_width src_width; - enum zx_dma_burst_width dst_width; - u32 maxburst = 0; - - switch (dir) { - case DMA_MEM_TO_MEM: - c->ccfg = ZX_CH_ENABLE | ZX_SOFT_REQ - | ZX_SRC_BURST_LEN(ZX_MAX_BURST_LEN - 1) - | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_32BIT) - | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_32BIT); - break; - case DMA_MEM_TO_DEV: - c->dev_addr = cfg->dst_addr; - /* dst len is calculated from src width, len and dst width. - * We need make sure dst len not exceed MAX LEN. - * Trailing single transaction that does not fill a full - * burst also require identical src/dst data width. - */ - dst_width = zx_dma_burst_width(cfg->dst_addr_width); - maxburst = cfg->dst_maxburst; - maxburst = maxburst < ZX_MAX_BURST_LEN ? - maxburst : ZX_MAX_BURST_LEN; - c->ccfg = ZX_DST_FIFO_MODE | ZX_CH_ENABLE - | ZX_SRC_BURST_LEN(maxburst - 1) - | ZX_SRC_BURST_WIDTH(dst_width) - | ZX_DST_BURST_WIDTH(dst_width); - break; - case DMA_DEV_TO_MEM: - c->dev_addr = cfg->src_addr; - src_width = zx_dma_burst_width(cfg->src_addr_width); - maxburst = cfg->src_maxburst; - maxburst = maxburst < ZX_MAX_BURST_LEN ? - maxburst : ZX_MAX_BURST_LEN; - c->ccfg = ZX_SRC_FIFO_MODE | ZX_CH_ENABLE - | ZX_SRC_BURST_LEN(maxburst - 1) - | ZX_SRC_BURST_WIDTH(src_width) - | ZX_DST_BURST_WIDTH(src_width); - break; - default: - return -EINVAL; - } - return 0; -} - -static struct dma_async_tx_descriptor *zx_dma_prep_memcpy( - struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, - size_t len, unsigned long flags) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - size_t copy = 0; - int num = 0; - - if (!len) - return NULL; - - if (zx_pre_config(c, DMA_MEM_TO_MEM)) - return NULL; - - num = DIV_ROUND_UP(len, DMA_MAX_SIZE); - - ds = zx_alloc_desc_resource(num, chan); - if (!ds) - return NULL; - - ds->size = len; - num = 0; - - do { - copy = min_t(size_t, len, DMA_MAX_SIZE); - zx_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg); - - src += copy; - dst += copy; - len -= copy; - } while (len); - - c->cyclic = 0; - ds->desc_hw[num - 1].lli = 0; /* end of link */ - ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL; - return vchan_tx_prep(&c->vc, &ds->vd, flags); -} - -static struct dma_async_tx_descriptor *zx_dma_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen, - enum dma_transfer_direction dir, unsigned long flags, void *context) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - size_t len, avail, total = 0; - struct scatterlist *sg; - dma_addr_t addr, src = 0, dst = 0; - int num = sglen, i; - - if (!sgl) - return NULL; - - if (zx_pre_config(c, dir)) - return NULL; - - for_each_sg(sgl, sg, sglen, i) { - avail = sg_dma_len(sg); - if (avail > DMA_MAX_SIZE) - num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; - } - - ds = zx_alloc_desc_resource(num, chan); - if (!ds) - return NULL; - - c->cyclic = 0; - num = 0; - for_each_sg(sgl, sg, sglen, i) { - addr = sg_dma_address(sg); - avail = sg_dma_len(sg); - total += avail; - - do { - len = min_t(size_t, avail, DMA_MAX_SIZE); - - if (dir == DMA_MEM_TO_DEV) { - src = addr; - dst = c->dev_addr; - } else if (dir == DMA_DEV_TO_MEM) { - src = c->dev_addr; - dst = addr; - } - - zx_dma_fill_desc(ds, dst, src, len, num++, c->ccfg); - - addr += len; - avail -= len; - } while (avail); - } - - ds->desc_hw[num - 1].lli = 0; /* end of link */ - ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL; - ds->size = total; - return vchan_tx_prep(&c->vc, &ds->vd, flags); -} - -static struct dma_async_tx_descriptor *zx_dma_prep_dma_cyclic( - struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, - size_t period_len, enum dma_transfer_direction dir, - unsigned long flags) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - dma_addr_t src = 0, dst = 0; - int num_periods = buf_len / period_len; - int buf = 0, num = 0; - - if (period_len > DMA_MAX_SIZE) { - dev_err(chan->device->dev, "maximum period size exceeded\n"); - return NULL; - } - - if (zx_pre_config(c, dir)) - return NULL; - - ds = zx_alloc_desc_resource(num_periods, chan); - if (!ds) - return NULL; - c->cyclic = 1; - - while (buf < buf_len) { - if (dir == DMA_MEM_TO_DEV) { - src = dma_addr; - dst = c->dev_addr; - } else if (dir == DMA_DEV_TO_MEM) { - src = c->dev_addr; - dst = dma_addr; - } - zx_dma_fill_desc(ds, dst, src, period_len, num++, - c->ccfg | ZX_IRQ_ENABLE_ALL); - dma_addr += period_len; - buf += period_len; - } - - ds->desc_hw[num - 1].lli = ds->desc_hw_lli; - ds->size = buf_len; - return vchan_tx_prep(&c->vc, &ds->vd, flags); -} - -static int zx_dma_config(struct dma_chan *chan, - struct dma_slave_config *cfg) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - - if (!cfg) - return -EINVAL; - - memcpy(&c->slave_cfg, cfg, sizeof(*cfg)); - - return 0; -} - -static int zx_dma_terminate_all(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_dev *d = to_zx_dma(chan->device); - struct zx_dma_phy *p = c->phy; - unsigned long flags; - LIST_HEAD(head); - - dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc); - - /* Prevent this channel being scheduled */ - spin_lock(&d->lock); - list_del_init(&c->node); - spin_unlock(&d->lock); - - /* Clear the tx descriptor lists */ - spin_lock_irqsave(&c->vc.lock, flags); - vchan_get_all_descriptors(&c->vc, &head); - if (p) { - /* vchan is assigned to a pchan - stop the channel */ - zx_dma_terminate_chan(p, d); - c->phy = NULL; - p->vchan = NULL; - p->ds_run = NULL; - p->ds_done = NULL; - } - spin_unlock_irqrestore(&c->vc.lock, flags); - vchan_dma_desc_free_list(&c->vc, &head); - - return 0; -} - -static int zx_dma_transfer_pause(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - u32 val = 0; - - val = readl_relaxed(c->phy->base + REG_ZX_CTRL); - val &= ~ZX_CH_ENABLE; - writel_relaxed(val, c->phy->base + REG_ZX_CTRL); - - return 0; -} - -static int zx_dma_transfer_resume(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - u32 val = 0; - - val = readl_relaxed(c->phy->base + REG_ZX_CTRL); - val |= ZX_CH_ENABLE; - writel_relaxed(val, c->phy->base + REG_ZX_CTRL); - - return 0; -} - -static void zx_dma_free_desc(struct virt_dma_desc *vd) -{ - struct zx_dma_desc_sw *ds = - container_of(vd, struct zx_dma_desc_sw, vd); - struct zx_dma_dev *d = to_zx_dma(vd->tx.chan->device); - - dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli); - kfree(ds); -} - -static const struct of_device_id zx6702_dma_dt_ids[] = { - { .compatible = "zte,zx296702-dma", }, - {} -}; -MODULE_DEVICE_TABLE(of, zx6702_dma_dt_ids); - -static struct dma_chan *zx_of_dma_simple_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct zx_dma_dev *d = ofdma->of_dma_data; - unsigned int request = dma_spec->args[0]; - struct dma_chan *chan; - struct zx_dma_chan *c; - - if (request >= d->dma_requests) - return NULL; - - chan = dma_get_any_slave_channel(&d->slave); - if (!chan) { - dev_err(d->slave.dev, "get channel fail in %s.\n", __func__); - return NULL; - } - c = to_zx_chan(chan); - c->id = request; - dev_info(d->slave.dev, "zx_dma: pchan %u: alloc vchan %p\n", - c->id, &c->vc); - return chan; -} - -static int zx_dma_probe(struct platform_device *op) -{ - struct zx_dma_dev *d; - int i, ret = 0; - - d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL); - if (!d) - return -ENOMEM; - - d->base = devm_platform_ioremap_resource(op, 0); - if (IS_ERR(d->base)) - return PTR_ERR(d->base); - - of_property_read_u32((&op->dev)->of_node, - "dma-channels", &d->dma_channels); - of_property_read_u32((&op->dev)->of_node, - "dma-requests", &d->dma_requests); - if (!d->dma_requests || !d->dma_channels) - return -EINVAL; - - d->clk = devm_clk_get(&op->dev, NULL); - if (IS_ERR(d->clk)) { - dev_err(&op->dev, "no dma clk\n"); - return PTR_ERR(d->clk); - } - - d->irq = platform_get_irq(op, 0); - ret = devm_request_irq(&op->dev, d->irq, zx_dma_int_handler, - 0, DRIVER_NAME, d); - if (ret) - return ret; - - /* A DMA memory pool for LLIs, align on 32-byte boundary */ - d->pool = dmam_pool_create(DRIVER_NAME, &op->dev, - LLI_BLOCK_SIZE, 32, 0); - if (!d->pool) - return -ENOMEM; - - /* init phy channel */ - d->phy = devm_kcalloc(&op->dev, - d->dma_channels, sizeof(struct zx_dma_phy), GFP_KERNEL); - if (!d->phy) - return -ENOMEM; - - for (i = 0; i < d->dma_channels; i++) { - struct zx_dma_phy *p = &d->phy[i]; - - p->idx = i; - p->base = d->base + i * 0x40; - } - - INIT_LIST_HEAD(&d->slave.channels); - dma_cap_set(DMA_SLAVE, d->slave.cap_mask); - dma_cap_set(DMA_MEMCPY, d->slave.cap_mask); - dma_cap_set(DMA_CYCLIC, d->slave.cap_mask); - dma_cap_set(DMA_PRIVATE, d->slave.cap_mask); - d->slave.dev = &op->dev; - d->slave.device_free_chan_resources = zx_dma_free_chan_resources; - d->slave.device_tx_status = zx_dma_tx_status; - d->slave.device_prep_dma_memcpy = zx_dma_prep_memcpy; - d->slave.device_prep_slave_sg = zx_dma_prep_slave_sg; - d->slave.device_prep_dma_cyclic = zx_dma_prep_dma_cyclic; - d->slave.device_issue_pending = zx_dma_issue_pending; - d->slave.device_config = zx_dma_config; - d->slave.device_terminate_all = zx_dma_terminate_all; - d->slave.device_pause = zx_dma_transfer_pause; - d->slave.device_resume = zx_dma_transfer_resume; - d->slave.copy_align = DMA_ALIGN; - d->slave.src_addr_widths = ZX_DMA_BUSWIDTHS; - d->slave.dst_addr_widths = ZX_DMA_BUSWIDTHS; - d->slave.directions = BIT(DMA_MEM_TO_MEM) | BIT(DMA_MEM_TO_DEV) - | BIT(DMA_DEV_TO_MEM); - d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; - - /* init virtual channel */ - d->chans = devm_kcalloc(&op->dev, - d->dma_requests, sizeof(struct zx_dma_chan), GFP_KERNEL); - if (!d->chans) - return -ENOMEM; - - for (i = 0; i < d->dma_requests; i++) { - struct zx_dma_chan *c = &d->chans[i]; - - c->status = DMA_IN_PROGRESS; - INIT_LIST_HEAD(&c->node); - c->vc.desc_free = zx_dma_free_desc; - vchan_init(&c->vc, &d->slave); - } - - /* Enable clock before accessing registers */ - ret = clk_prepare_enable(d->clk); - if (ret < 0) { - dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret); - goto zx_dma_out; - } - - zx_dma_init_state(d); - - spin_lock_init(&d->lock); - INIT_LIST_HEAD(&d->chan_pending); - platform_set_drvdata(op, d); - - ret = dma_async_device_register(&d->slave); - if (ret) - goto clk_dis; - - ret = of_dma_controller_register((&op->dev)->of_node, - zx_of_dma_simple_xlate, d); - if (ret) - goto of_dma_register_fail; - - dev_info(&op->dev, "initialized\n"); - return 0; - -of_dma_register_fail: - dma_async_device_unregister(&d->slave); -clk_dis: - clk_disable_unprepare(d->clk); -zx_dma_out: - return ret; -} - -static int zx_dma_remove(struct platform_device *op) -{ - struct zx_dma_chan *c, *cn; - struct zx_dma_dev *d = platform_get_drvdata(op); - - /* explictly free the irq */ - devm_free_irq(&op->dev, d->irq, d); - - dma_async_device_unregister(&d->slave); - of_dma_controller_free((&op->dev)->of_node); - - list_for_each_entry_safe(c, cn, &d->slave.channels, - vc.chan.device_node) { - list_del(&c->vc.chan.device_node); - } - clk_disable_unprepare(d->clk); - - return 0; -} - -#ifdef CONFIG_PM_SLEEP -static int zx_dma_suspend_dev(struct device *dev) -{ - struct zx_dma_dev *d = dev_get_drvdata(dev); - u32 stat = 0; - - stat = zx_dma_get_chan_stat(d); - if (stat) { - dev_warn(d->slave.dev, - "chan %d is running fail to suspend\n", stat); - return -1; - } - clk_disable_unprepare(d->clk); - return 0; -} - -static int zx_dma_resume_dev(struct device *dev) -{ - struct zx_dma_dev *d = dev_get_drvdata(dev); - int ret = 0; - - ret = clk_prepare_enable(d->clk); - if (ret < 0) { - dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret); - return ret; - } - zx_dma_init_state(d); - return 0; -} -#endif - -static SIMPLE_DEV_PM_OPS(zx_dma_pmops, zx_dma_suspend_dev, zx_dma_resume_dev); - -static struct platform_driver zx_pdma_driver = { - .driver = { - .name = DRIVER_NAME, - .pm = &zx_dma_pmops, - .of_match_table = zx6702_dma_dt_ids, - }, - .probe = zx_dma_probe, - .remove = zx_dma_remove, -}; - -module_platform_driver(zx_pdma_driver); - -MODULE_DESCRIPTION("ZTE ZX296702 DMA Driver"); -MODULE_AUTHOR("Jun Nie jun.nie@linaro.org"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/firewire/core-device.c b/drivers/firewire/core-device.c index 80db43a22069..68216988391f 100644 --- a/drivers/firewire/core-device.c +++ b/drivers/firewire/core-device.c @@ -192,7 +192,9 @@ static int fw_unit_remove(struct device *dev) struct fw_driver *driver = container_of(dev->driver, struct fw_driver, driver); - return driver->remove(fw_unit(dev)), 0; + driver->remove(fw_unit(dev)); + + return 0; } static int get_modalias(struct fw_unit *unit, char *buffer, size_t buffer_size) diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 8a94388e38b3..c23466e05e60 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -38,6 +38,8 @@ KBUILD_CFLAGS := $(cflags-y) -Os -DDISABLE_BRANCH_PROFILING \ # remove SCS flags from all objects in this directory KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) +# disable LTO +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS)) GCOV_PROFILE := n # Sanitizer runtimes are unavailable and cannot be linked here. diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index 0205987a4fd4..dc83ea118c67 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -46,14 +46,13 @@ static int coreboot_bus_probe(struct device *dev) static int coreboot_bus_remove(struct device *dev) { - int ret = 0; struct coreboot_device *device = CB_DEV(dev); struct coreboot_driver *driver = CB_DRV(dev->driver); if (driver->remove) - ret = driver->remove(device); + driver->remove(device); - return ret; + return 0; } static struct bus_type coreboot_bus_type = { diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/google/coreboot_table.h index 7b7b4a6eedda..beb778674acd 100644 --- a/drivers/firmware/google/coreboot_table.h +++ b/drivers/firmware/google/coreboot_table.h @@ -72,7 +72,7 @@ struct coreboot_device { /* A driver for handling devices described in coreboot tables. */ struct coreboot_driver { int (*probe)(struct coreboot_device *); - int (*remove)(struct coreboot_device *); + void (*remove)(struct coreboot_device *); struct device_driver drv; u32 tag; }; diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c index 916f26adc595..c6dcc1ef93ac 100644 --- a/drivers/firmware/google/framebuffer-coreboot.c +++ b/drivers/firmware/google/framebuffer-coreboot.c @@ -72,13 +72,11 @@ static int framebuffer_probe(struct coreboot_device *dev) return PTR_ERR_OR_ZERO(pdev); } -static int framebuffer_remove(struct coreboot_device *dev) +static void framebuffer_remove(struct coreboot_device *dev) { struct platform_device *pdev = dev_get_drvdata(&dev->dev); platform_device_unregister(pdev); - - return 0; } static struct coreboot_driver framebuffer_driver = { diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/google/memconsole-coreboot.c index d17e4d6ac9bc..74b5286518ee 100644 --- a/drivers/firmware/google/memconsole-coreboot.c +++ b/drivers/firmware/google/memconsole-coreboot.c @@ -91,11 +91,9 @@ static int memconsole_probe(struct coreboot_device *dev) return memconsole_sysfs_init(); } -static int memconsole_remove(struct coreboot_device *dev) +static void memconsole_remove(struct coreboot_device *dev) { memconsole_exit(); - - return 0; } static struct coreboot_driver memconsole_driver = { diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c index d23c5c69ab52..ee6e08c0592b 100644 --- a/drivers/firmware/google/vpd.c +++ b/drivers/firmware/google/vpd.c @@ -298,14 +298,12 @@ static int vpd_probe(struct coreboot_device *dev) return 0; } -static int vpd_remove(struct coreboot_device *dev) +static void vpd_remove(struct coreboot_device *dev) { vpd_section_destroy(&ro_vpd); vpd_section_destroy(&rw_vpd); kobject_put(vpd_kobj); - - return 0; } static struct coreboot_driver vpd_driver = { diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c index 7be48c1bec96..f57779fc7ee9 100644 --- a/drivers/firmware/qcom_scm.c +++ b/drivers/firmware/qcom_scm.c @@ -965,8 +965,11 @@ EXPORT_SYMBOL(qcom_scm_ice_available); * qcom_scm_ice_invalidate_key() - Invalidate an inline encryption key * @index: the keyslot to invalidate * - * The UFSHCI standard defines a standard way to do this, but it doesn't work on - * these SoCs; only this SCM call does. + * The UFSHCI and eMMC standards define a standard way to do this, but it + * doesn't work on these SoCs; only this SCM call does. + * + * It is assumed that the SoC has only one ICE instance being used, as this SCM + * call doesn't specify which ICE instance the keyslot belongs to. * * Return: 0 on success; -errno on failure. */ @@ -995,10 +998,13 @@ EXPORT_SYMBOL(qcom_scm_ice_invalidate_key); * units, e.g. 1 = 512 bytes, 8 = 4096 bytes, etc. * * Program a key into a keyslot of Qualcomm ICE (Inline Crypto Engine), where it - * can then be used to encrypt/decrypt UFS I/O requests inline. + * can then be used to encrypt/decrypt UFS or eMMC I/O requests inline. * - * The UFSHCI standard defines a standard way to do this, but it doesn't work on - * these SoCs; only this SCM call does. + * The UFSHCI and eMMC standards define a standard way to do this, but it + * doesn't work on these SoCs; only this SCM call does. + * + * It is assumed that the SoC has only one ICE instance being used, as this SCM + * call doesn't specify which ICE instance the keyslot belongs to. * * Return: 0 on success; -errno on failure. */ diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig index 5645226ca3ce..5ff9438b7b46 100644 --- a/drivers/fpga/Kconfig +++ b/drivers/fpga/Kconfig @@ -192,6 +192,17 @@ config FPGA_DFL_AFU to the FPGA infrastructure via a Port. There may be more than one Port/AFU per DFL based FPGA device. +config FPGA_DFL_NIOS_INTEL_PAC_N3000 + tristate "FPGA DFL NIOS Driver for Intel PAC N3000" + depends on FPGA_DFL + select REGMAP + help + This is the driver for the N3000 Nios private feature on Intel + PAC (Programmable Acceleration Card) N3000. It communicates + with the embedded Nios processor to configure the retimers on + the card. It also instantiates the SPI master (spi-altera) for + the card's BMC (Board Management Controller). + config FPGA_DFL_PCI tristate "FPGA DFL PCIe Device Driver" depends on PCI && FPGA_DFL diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile index d8e21dfc6778..18dc9885883a 100644 --- a/drivers/fpga/Makefile +++ b/drivers/fpga/Makefile @@ -44,5 +44,7 @@ dfl-fme-objs += dfl-fme-perf.o dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o dfl-afu-objs += dfl-afu-error.o +obj-$(CONFIG_FPGA_DFL_NIOS_INTEL_PAC_N3000) += dfl-n3000-nios.o + # Drivers for FPGAs which implement DFL obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o diff --git a/drivers/fpga/dfl-fme-perf.c b/drivers/fpga/dfl-fme-perf.c index 531266287eee..4299145ef347 100644 --- a/drivers/fpga/dfl-fme-perf.c +++ b/drivers/fpga/dfl-fme-perf.c @@ -192,7 +192,7 @@ static struct attribute *fme_perf_cpumask_attrs[] = { NULL, }; -static struct attribute_group fme_perf_cpumask_group = { +static const struct attribute_group fme_perf_cpumask_group = { .attrs = fme_perf_cpumask_attrs, }; @@ -225,7 +225,7 @@ static struct attribute *fme_perf_format_attrs[] = { NULL, }; -static struct attribute_group fme_perf_format_group = { +static const struct attribute_group fme_perf_format_group = { .name = "format", .attrs = fme_perf_format_attrs, }; @@ -239,7 +239,7 @@ static struct attribute *fme_perf_events_attrs_empty[] = { NULL, }; -static struct attribute_group fme_perf_events_group = { +static const struct attribute_group fme_perf_events_group = { .name = "events", .attrs = fme_perf_events_attrs_empty, }; diff --git a/drivers/fpga/dfl-n3000-nios.c b/drivers/fpga/dfl-n3000-nios.c new file mode 100644 index 000000000000..7a95366f6516 --- /dev/null +++ b/drivers/fpga/dfl-n3000-nios.c @@ -0,0 +1,588 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DFL device driver for Nios private feature on Intel PAC (Programmable + * Acceleration Card) N3000 + * + * Copyright (C) 2019-2020 Intel Corporation, Inc. + * + * Authors: + * Wu Hao + * Xu Yilun + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * N3000 Nios private feature registers, named as NIOS_SPI_XX on spec. + * NS is the abbreviation of NIOS_SPI. + */ +#define N3000_NS_PARAM 0x8 +#define N3000_NS_PARAM_SHIFT_MODE_MSK BIT_ULL(1) +#define N3000_NS_PARAM_SHIFT_MODE_MSB 0 +#define N3000_NS_PARAM_SHIFT_MODE_LSB 1 +#define N3000_NS_PARAM_DATA_WIDTH GENMASK_ULL(7, 2) +#define N3000_NS_PARAM_NUM_CS GENMASK_ULL(13, 8) +#define N3000_NS_PARAM_CLK_POL BIT_ULL(14) +#define N3000_NS_PARAM_CLK_PHASE BIT_ULL(15) +#define N3000_NS_PARAM_PERIPHERAL_ID GENMASK_ULL(47, 32) + +#define N3000_NS_CTRL 0x10 +#define N3000_NS_CTRL_WR_DATA GENMASK_ULL(31, 0) +#define N3000_NS_CTRL_ADDR GENMASK_ULL(44, 32) +#define N3000_NS_CTRL_CMD_MSK GENMASK_ULL(63, 62) +#define N3000_NS_CTRL_CMD_NOP 0 +#define N3000_NS_CTRL_CMD_RD 1 +#define N3000_NS_CTRL_CMD_WR 2 + +#define N3000_NS_STAT 0x18 +#define N3000_NS_STAT_RD_DATA GENMASK_ULL(31, 0) +#define N3000_NS_STAT_RW_VAL BIT_ULL(32) + +/* Nios handshake registers, indirect access */ +#define N3000_NIOS_INIT 0x1000 +#define N3000_NIOS_INIT_DONE BIT(0) +#define N3000_NIOS_INIT_START BIT(1) +/* Mode for retimer A, link 0, the same below */ +#define N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK GENMASK(9, 8) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK GENMASK(11, 10) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK GENMASK(13, 12) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK GENMASK(15, 14) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK GENMASK(17, 16) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK GENMASK(19, 18) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK GENMASK(21, 20) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK GENMASK(23, 22) +#define N3000_NIOS_INIT_REQ_FEC_MODE_NO 0x0 +#define N3000_NIOS_INIT_REQ_FEC_MODE_KR 0x1 +#define N3000_NIOS_INIT_REQ_FEC_MODE_RS 0x2 + +#define N3000_NIOS_FW_VERSION 0x1004 +#define N3000_NIOS_FW_VERSION_PATCH GENMASK(23, 20) +#define N3000_NIOS_FW_VERSION_MINOR GENMASK(27, 24) +#define N3000_NIOS_FW_VERSION_MAJOR GENMASK(31, 28) + +/* The retimers we use on Intel PAC N3000 is Parkvale, abbreviated to PKVL */ +#define N3000_NIOS_PKVL_A_MODE_STS 0x1020 +#define N3000_NIOS_PKVL_B_MODE_STS 0x1024 +#define N3000_NIOS_PKVL_MODE_STS_GROUP_MSK GENMASK(15, 8) +#define N3000_NIOS_PKVL_MODE_STS_GROUP_OK 0x0 +#define N3000_NIOS_PKVL_MODE_STS_ID_MSK GENMASK(7, 0) +/* When GROUP MASK field == GROUP_OK */ +#define N3000_NIOS_PKVL_MODE_ID_RESET 0x0 +#define N3000_NIOS_PKVL_MODE_ID_4X10G 0x1 +#define N3000_NIOS_PKVL_MODE_ID_4X25G 0x2 +#define N3000_NIOS_PKVL_MODE_ID_2X25G 0x3 +#define N3000_NIOS_PKVL_MODE_ID_2X25G_2X10G 0x4 +#define N3000_NIOS_PKVL_MODE_ID_1X25G 0x5 + +#define N3000_NIOS_REGBUS_RETRY_COUNT 10000 /* loop count */ + +#define N3000_NIOS_INIT_TIMEOUT 10000000 /* usec */ +#define N3000_NIOS_INIT_TIME_INTV 100000 /* usec */ + +#define N3000_NIOS_INIT_REQ_FEC_MODE_MSK_ALL \ + (N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_NO_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO)) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_KR_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR)) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS)) + +struct n3000_nios { + void __iomem *base; + struct regmap *regmap; + struct device *dev; + struct platform_device *altera_spi; +}; + +static ssize_t nios_fw_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int val; + int ret; + + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + return sysfs_emit(buf, "%x.%x.%x\n", + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val), + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_MINOR, val), + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_PATCH, val)); +} +static DEVICE_ATTR_RO(nios_fw_version); + +#define IS_MODE_STATUS_OK(mode_stat) \ + (FIELD_GET(N3000_NIOS_PKVL_MODE_STS_GROUP_MSK, (mode_stat)) == \ + N3000_NIOS_PKVL_MODE_STS_GROUP_OK) + +#define IS_RETIMER_FEC_SUPPORTED(retimer_mode) \ + ((retimer_mode) != N3000_NIOS_PKVL_MODE_ID_RESET && \ + (retimer_mode) != N3000_NIOS_PKVL_MODE_ID_4X10G) + +static int get_retimer_mode(struct n3000_nios *nn, unsigned int mode_stat_reg, + unsigned int *retimer_mode) +{ + unsigned int val; + int ret; + + ret = regmap_read(nn->regmap, mode_stat_reg, &val); + if (ret) + return ret; + + if (!IS_MODE_STATUS_OK(val)) + return -EFAULT; + + *retimer_mode = FIELD_GET(N3000_NIOS_PKVL_MODE_STS_ID_MSK, val); + + return 0; +} + +static ssize_t retimer_A_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int mode; + int ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_A_MODE_STS, &mode); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%x\n", mode); +} +static DEVICE_ATTR_RO(retimer_A_mode); + +static ssize_t retimer_B_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int mode; + int ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_B_MODE_STS, &mode); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%x\n", mode); +} +static DEVICE_ATTR_RO(retimer_B_mode); + +static ssize_t fec_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned int val, retimer_a_mode, retimer_b_mode, fec_modes; + struct n3000_nios *nn = dev_get_drvdata(dev); + int ret; + + /* FEC mode setting is not supported in early FW versions */ + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + if (FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val) < 3) + return sysfs_emit(buf, "not supported\n"); + + /* If no 25G links, FEC mode setting is not supported either */ + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_A_MODE_STS, &retimer_a_mode); + if (ret) + return ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_B_MODE_STS, &retimer_b_mode); + if (ret) + return ret; + + if (!IS_RETIMER_FEC_SUPPORTED(retimer_a_mode) && + !IS_RETIMER_FEC_SUPPORTED(retimer_b_mode)) + return sysfs_emit(buf, "not supported\n"); + + /* get the valid FEC mode for 25G links */ + ret = regmap_read(nn->regmap, N3000_NIOS_INIT, &val); + if (ret) + return ret; + + /* + * FEC mode should always be the same for all links, as we set them + * in this way. + */ + fec_modes = (val & N3000_NIOS_INIT_REQ_FEC_MODE_MSK_ALL); + if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_NO_ALL) + return sysfs_emit(buf, "no\n"); + else if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_KR_ALL) + return sysfs_emit(buf, "kr\n"); + else if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL) + return sysfs_emit(buf, "rs\n"); + + return -EFAULT; +} +static DEVICE_ATTR_RO(fec_mode); + +static struct attribute *n3000_nios_attrs[] = { + &dev_attr_nios_fw_version.attr, + &dev_attr_retimer_A_mode.attr, + &dev_attr_retimer_B_mode.attr, + &dev_attr_fec_mode.attr, + NULL, +}; +ATTRIBUTE_GROUPS(n3000_nios); + +static int n3000_nios_init_done_check(struct n3000_nios *nn) +{ + unsigned int val, state_a, state_b; + struct device *dev = nn->dev; + int ret, ret2; + + /* + * The SPI is shared by the Nios core inside the FPGA, Nios will use + * this SPI master to do some one time initialization after power up, + * and then release the control to OS. The driver needs to poll on + * INIT_DONE to see when driver could take the control. + * + * Please note that after Nios firmware version 3.0.0, INIT_START is + * introduced, so driver needs to trigger START firstly and then check + * INIT_DONE. + */ + + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + /* + * If Nios version register is totally uninitialized(== 0x0), then the + * Nios firmware is missing. So host could take control of SPI master + * safely, but initialization work for Nios is not done. To restore the + * card, we need to reprogram a new Nios firmware via the BMC chip on + * SPI bus. So the driver doesn't error out, it continues to create the + * spi controller device and spi_board_info for BMC. + */ + if (val == 0) { + dev_err(dev, "Nios version reg = 0x%x, skip INIT_DONE check, but the retimer may be uninitialized\n", + val); + return 0; + } + + if (FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val) >= 3) { + /* read NIOS_INIT to check if retimer initialization is done */ + ret = regmap_read(nn->regmap, N3000_NIOS_INIT, &val); + if (ret) + return ret; + + /* check if retimers are initialized already */ + if (val & (N3000_NIOS_INIT_DONE | N3000_NIOS_INIT_START)) + goto nios_init_done; + + /* configure FEC mode per module param */ + val = N3000_NIOS_INIT_START; + + /* + * When the retimer is to be set to 10G mode, there is no FEC + * mode setting, so the REQ_FEC_MODE field will be ignored by + * Nios firmware in this case. But we should still fill the FEC + * mode field cause host could not get the retimer working mode + * until the Nios init is done. + * + * For now the driver doesn't support the retimer FEC mode + * switching per user's request. It is always set to Reed + * Solomon FEC. + * + * The driver will set the same FEC mode for all links. + */ + val |= N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL; + + ret = regmap_write(nn->regmap, N3000_NIOS_INIT, val); + if (ret) + return ret; + } + +nios_init_done: + /* polls on NIOS_INIT_DONE */ + ret = regmap_read_poll_timeout(nn->regmap, N3000_NIOS_INIT, val, + val & N3000_NIOS_INIT_DONE, + N3000_NIOS_INIT_TIME_INTV, + N3000_NIOS_INIT_TIMEOUT); + if (ret) + dev_err(dev, "NIOS_INIT_DONE %s\n", + (ret == -ETIMEDOUT) ? "timed out" : "check error"); + + ret2 = regmap_read(nn->regmap, N3000_NIOS_PKVL_A_MODE_STS, &state_a); + if (ret2) + return ret2; + + ret2 = regmap_read(nn->regmap, N3000_NIOS_PKVL_B_MODE_STS, &state_b); + if (ret2) + return ret2; + + if (!ret) { + /* + * After INIT_DONE is detected, it still needs to check if the + * Nios firmware reports any error during the retimer + * configuration. + */ + if (IS_MODE_STATUS_OK(state_a) && IS_MODE_STATUS_OK(state_b)) + return 0; + + /* + * If the retimer configuration is failed, the Nios firmware + * will still release the spi controller for host to + * communicate with the BMC. It makes possible for people to + * reprogram a new Nios firmware and restore the card. So the + * driver doesn't error out, it continues to create the spi + * controller device and spi_board_info for BMC. + */ + dev_err(dev, "NIOS_INIT_DONE OK, but err on retimer init\n"); + } + + dev_err(nn->dev, "PKVL_A_MODE_STS 0x%x\n", state_a); + dev_err(nn->dev, "PKVL_B_MODE_STS 0x%x\n", state_b); + + return ret; +} + +static struct spi_board_info m10_n3000_info = { + .modalias = "m10-n3000", + .max_speed_hz = 12500000, + .bus_num = 0, + .chip_select = 0, +}; + +static int create_altera_spi_controller(struct n3000_nios *nn) +{ + struct altera_spi_platform_data pdata = { 0 }; + struct platform_device_info pdevinfo = { 0 }; + void __iomem *base = nn->base; + u64 v; + + v = readq(base + N3000_NS_PARAM); + + pdata.mode_bits = SPI_CS_HIGH; + if (FIELD_GET(N3000_NS_PARAM_CLK_POL, v)) + pdata.mode_bits |= SPI_CPOL; + if (FIELD_GET(N3000_NS_PARAM_CLK_PHASE, v)) + pdata.mode_bits |= SPI_CPHA; + + pdata.num_chipselect = FIELD_GET(N3000_NS_PARAM_NUM_CS, v); + pdata.bits_per_word_mask = + SPI_BPW_RANGE_MASK(1, FIELD_GET(N3000_NS_PARAM_DATA_WIDTH, v)); + + pdata.num_devices = 1; + pdata.devices = &m10_n3000_info; + + dev_dbg(nn->dev, "%s cs %u bpm 0x%x mode 0x%x\n", __func__, + pdata.num_chipselect, pdata.bits_per_word_mask, + pdata.mode_bits); + + pdevinfo.name = "subdev_spi_altera"; + pdevinfo.id = PLATFORM_DEVID_AUTO; + pdevinfo.parent = nn->dev; + pdevinfo.data = &pdata; + pdevinfo.size_data = sizeof(pdata); + + nn->altera_spi = platform_device_register_full(&pdevinfo); + return PTR_ERR_OR_ZERO(nn->altera_spi); +} + +static void destroy_altera_spi_controller(struct n3000_nios *nn) +{ + platform_device_unregister(nn->altera_spi); +} + +static int n3000_nios_poll_stat_timeout(void __iomem *base, u64 *v) +{ + int loops; + + /* + * We don't use the time based timeout here for performance. + * + * The regbus read/write is on the critical path of Intel PAC N3000 + * image programing. The time based timeout checking will add too much + * overhead on it. Usually the state changes in 1 or 2 loops on the + * test server, and we set 10000 times loop here for safety. + */ + for (loops = N3000_NIOS_REGBUS_RETRY_COUNT; loops > 0 ; loops--) { + *v = readq(base + N3000_NS_STAT); + if (*v & N3000_NS_STAT_RW_VAL) + break; + cpu_relax(); + } + + return (loops > 0) ? 0 : -ETIMEDOUT; +} + +static int n3000_nios_reg_write(void *context, unsigned int reg, unsigned int val) +{ + struct n3000_nios *nn = context; + u64 v; + int ret; + + v = FIELD_PREP(N3000_NS_CTRL_CMD_MSK, N3000_NS_CTRL_CMD_WR) | + FIELD_PREP(N3000_NS_CTRL_ADDR, reg) | + FIELD_PREP(N3000_NS_CTRL_WR_DATA, val); + writeq(v, nn->base + N3000_NS_CTRL); + + ret = n3000_nios_poll_stat_timeout(nn->base, &v); + if (ret) + dev_err(nn->dev, "fail to write reg 0x%x val 0x%x: %d\n", + reg, val, ret); + + return ret; +} + +static int n3000_nios_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + struct n3000_nios *nn = context; + u64 v; + int ret; + + v = FIELD_PREP(N3000_NS_CTRL_CMD_MSK, N3000_NS_CTRL_CMD_RD) | + FIELD_PREP(N3000_NS_CTRL_ADDR, reg); + writeq(v, nn->base + N3000_NS_CTRL); + + ret = n3000_nios_poll_stat_timeout(nn->base, &v); + if (ret) + dev_err(nn->dev, "fail to read reg 0x%x: %d\n", reg, ret); + else + *val = FIELD_GET(N3000_NS_STAT_RD_DATA, v); + + return ret; +} + +static const struct regmap_config n3000_nios_regbus_cfg = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .fast_io = true, + + .reg_write = n3000_nios_reg_write, + .reg_read = n3000_nios_reg_read, +}; + +static int n3000_nios_probe(struct dfl_device *ddev) +{ + struct device *dev = &ddev->dev; + struct n3000_nios *nn; + int ret; + + nn = devm_kzalloc(dev, sizeof(*nn), GFP_KERNEL); + if (!nn) + return -ENOMEM; + + dev_set_drvdata(&ddev->dev, nn); + + nn->dev = dev; + + nn->base = devm_ioremap_resource(&ddev->dev, &ddev->mmio_res); + if (IS_ERR(nn->base)) + return PTR_ERR(nn->base); + + nn->regmap = devm_regmap_init(dev, NULL, nn, &n3000_nios_regbus_cfg); + if (IS_ERR(nn->regmap)) + return PTR_ERR(nn->regmap); + + ret = n3000_nios_init_done_check(nn); + if (ret) + return ret; + + ret = create_altera_spi_controller(nn); + if (ret) + dev_err(dev, "altera spi controller create failed: %d\n", ret); + + return ret; +} + +static void n3000_nios_remove(struct dfl_device *ddev) +{ + struct n3000_nios *nn = dev_get_drvdata(&ddev->dev); + + destroy_altera_spi_controller(nn); +} + +#define FME_FEATURE_ID_N3000_NIOS 0xd + +static const struct dfl_device_id n3000_nios_ids[] = { + { FME_ID, FME_FEATURE_ID_N3000_NIOS }, + { } +}; +MODULE_DEVICE_TABLE(dfl, n3000_nios_ids); + +static struct dfl_driver n3000_nios_driver = { + .drv = { + .name = "dfl-n3000-nios", + .dev_groups = n3000_nios_groups, + }, + .id_table = n3000_nios_ids, + .probe = n3000_nios_probe, + .remove = n3000_nios_remove, +}; + +module_dfl_driver(n3000_nios_driver); + +MODULE_DESCRIPTION("Driver for Nios private feature on Intel PAC N3000"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c index a2203d03c9e2..04e47e266f26 100644 --- a/drivers/fpga/dfl-pci.c +++ b/drivers/fpga/dfl-pci.c @@ -27,6 +27,14 @@ #define DRV_VERSION "0.8" #define DRV_NAME "dfl-pci" +#define PCI_VSEC_ID_INTEL_DFLS 0x43 + +#define PCI_VNDR_DFLS_CNT 0x8 +#define PCI_VNDR_DFLS_RES 0xc + +#define PCI_VNDR_DFLS_RES_BAR_MASK GENMASK(2, 0) +#define PCI_VNDR_DFLS_RES_OFF_MASK GENMASK(31, 3) + struct cci_drvdata { struct dfl_fpga_cdev *cdev; /* container device */ }; @@ -119,49 +127,94 @@ static int *cci_pci_create_irq_table(struct pci_dev *pcidev, unsigned int nvec) return table; } -/* enumerate feature devices under pci device */ -static int cci_enumerate_feature_devs(struct pci_dev *pcidev) +static int find_dfls_by_vsec(struct pci_dev *pcidev, struct dfl_fpga_enum_info *info) { - struct cci_drvdata *drvdata = pci_get_drvdata(pcidev); - int port_num, bar, i, nvec, ret = 0; - struct dfl_fpga_enum_info *info; - struct dfl_fpga_cdev *cdev; + u32 bir, offset, vndr_hdr, dfl_cnt, dfl_res; + int dfl_res_off, i, bars, voff = 0; + resource_size_t start, len; + + while ((voff = pci_find_next_ext_capability(pcidev, voff, PCI_EXT_CAP_ID_VNDR))) { + vndr_hdr = 0; + pci_read_config_dword(pcidev, voff + PCI_VNDR_HEADER, &vndr_hdr); + + if (PCI_VNDR_HEADER_ID(vndr_hdr) == PCI_VSEC_ID_INTEL_DFLS && + pcidev->vendor == PCI_VENDOR_ID_INTEL) + break; + } + + if (!voff) { + dev_dbg(&pcidev->dev, "%s no DFL VSEC found\n", __func__); + return -ENODEV; + } + + dfl_cnt = 0; + pci_read_config_dword(pcidev, voff + PCI_VNDR_DFLS_CNT, &dfl_cnt); + if (dfl_cnt > PCI_STD_NUM_BARS) { + dev_err(&pcidev->dev, "%s too many DFLs %d > %d\n", + __func__, dfl_cnt, PCI_STD_NUM_BARS); + return -EINVAL; + } + + dfl_res_off = voff + PCI_VNDR_DFLS_RES; + if (dfl_res_off + (dfl_cnt * sizeof(u32)) > PCI_CFG_SPACE_EXP_SIZE) { + dev_err(&pcidev->dev, "%s DFL VSEC too big for PCIe config space\n", + __func__); + return -EINVAL; + } + + for (i = 0, bars = 0; i < dfl_cnt; i++, dfl_res_off += sizeof(u32)) { + dfl_res = GENMASK(31, 0); + pci_read_config_dword(pcidev, dfl_res_off, &dfl_res); + + bir = dfl_res & PCI_VNDR_DFLS_RES_BAR_MASK; + if (bir >= PCI_STD_NUM_BARS) { + dev_err(&pcidev->dev, "%s bad bir number %d\n", + __func__, bir); + return -EINVAL; + } + + if (bars & BIT(bir)) { + dev_err(&pcidev->dev, "%s DFL for BAR %d already specified\n", + __func__, bir); + return -EINVAL; + } + + bars |= BIT(bir); + + len = pci_resource_len(pcidev, bir); + offset = dfl_res & PCI_VNDR_DFLS_RES_OFF_MASK; + if (offset >= len) { + dev_err(&pcidev->dev, "%s bad offset %u >= %pa\n", + __func__, offset, &len); + return -EINVAL; + } + + dev_dbg(&pcidev->dev, "%s BAR %d offset 0x%x\n", __func__, bir, offset); + + len -= offset; + + start = pci_resource_start(pcidev, bir) + offset; + + dfl_fpga_enum_info_add_dfl(info, start, len); + } + + return 0; +} + +/* default method of finding dfls starting at offset 0 of bar 0 */ +static int find_dfls_by_default(struct pci_dev *pcidev, + struct dfl_fpga_enum_info *info) +{ + int port_num, bar, i, ret = 0; resource_size_t start, len; void __iomem *base; - int *irq_table; u32 offset; u64 v; - /* allocate enumeration info via pci_dev */ - info = dfl_fpga_enum_info_alloc(&pcidev->dev); - if (!info) - return -ENOMEM; - - /* add irq info for enumeration if the device support irq */ - nvec = cci_pci_alloc_irq(pcidev); - if (nvec < 0) { - dev_err(&pcidev->dev, "Fail to alloc irq %d.\n", nvec); - ret = nvec; - goto enum_info_free_exit; - } else if (nvec) { - irq_table = cci_pci_create_irq_table(pcidev, nvec); - if (!irq_table) { - ret = -ENOMEM; - goto irq_free_exit; - } - - ret = dfl_fpga_enum_info_add_irq(info, nvec, irq_table); - kfree(irq_table); - if (ret) - goto irq_free_exit; - } - - /* start to find Device Feature List in Bar 0 */ + /* start to find Device Feature List from Bar 0 */ base = cci_pci_ioremap_bar0(pcidev); - if (!base) { - ret = -ENOMEM; - goto irq_free_exit; - } + if (!base) + return -ENOMEM; /* * PF device has FME and Ports/AFUs, and VF device only has one @@ -208,12 +261,54 @@ static int cci_enumerate_feature_devs(struct pci_dev *pcidev) dfl_fpga_enum_info_add_dfl(info, start, len); } else { ret = -ENODEV; - goto irq_free_exit; } /* release I/O mappings for next step enumeration */ pcim_iounmap_regions(pcidev, BIT(0)); + return ret; +} + +/* enumerate feature devices under pci device */ +static int cci_enumerate_feature_devs(struct pci_dev *pcidev) +{ + struct cci_drvdata *drvdata = pci_get_drvdata(pcidev); + struct dfl_fpga_enum_info *info; + struct dfl_fpga_cdev *cdev; + int nvec, ret = 0; + int *irq_table; + + /* allocate enumeration info via pci_dev */ + info = dfl_fpga_enum_info_alloc(&pcidev->dev); + if (!info) + return -ENOMEM; + + /* add irq info for enumeration if the device support irq */ + nvec = cci_pci_alloc_irq(pcidev); + if (nvec < 0) { + dev_err(&pcidev->dev, "Fail to alloc irq %d.\n", nvec); + ret = nvec; + goto enum_info_free_exit; + } else if (nvec) { + irq_table = cci_pci_create_irq_table(pcidev, nvec); + if (!irq_table) { + ret = -ENOMEM; + goto irq_free_exit; + } + + ret = dfl_fpga_enum_info_add_irq(info, nvec, irq_table); + kfree(irq_table); + if (ret) + goto irq_free_exit; + } + + ret = find_dfls_by_vsec(pcidev, info); + if (ret == -ENODEV) + ret = find_dfls_by_default(pcidev, info); + + if (ret) + goto irq_free_exit; + /* start enumeration with prepared enumeration information */ cdev = dfl_fpga_feature_devs_enumerate(info); if (IS_ERR(cdev)) { diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index b450870b75ed..511b20ff35a3 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -10,6 +10,7 @@ * Wu Hao * Xiao Guangrong */ +#include #include #include #include @@ -298,8 +299,7 @@ static int dfl_bus_uevent(struct device *dev, struct kobj_uevent_env *env) { struct dfl_device *ddev = to_dfl_dev(dev); - /* The type has 4 valid bits and feature_id has 12 valid bits */ - return add_uevent_var(env, "MODALIAS=dfl:t%01Xf%03X", + return add_uevent_var(env, "MODALIAS=dfl:t%04Xf%04X", ddev->type, ddev->feature_id); } diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h index 5dc758f655b7..2b82c96ba56c 100644 --- a/drivers/fpga/dfl.h +++ b/drivers/fpga/dfl.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -516,88 +517,4 @@ long dfl_feature_ioctl_set_irq(struct platform_device *pdev, struct dfl_feature *feature, unsigned long arg); -/** - * enum dfl_id_type - define the DFL FIU types - */ -enum dfl_id_type { - FME_ID, - PORT_ID, - DFL_ID_MAX, -}; - -/** - * struct dfl_device_id - dfl device identifier - * @type: contains 4 bits DFL FIU type of the device. See enum dfl_id_type. - * @feature_id: contains 12 bits feature identifier local to its DFL FIU type. - * @driver_data: driver specific data. - */ -struct dfl_device_id { - u8 type; - u16 feature_id; - unsigned long driver_data; -}; - -/** - * struct dfl_device - represent an dfl device on dfl bus - * - * @dev: generic device interface. - * @id: id of the dfl device. - * @type: type of DFL FIU of the device. See enum dfl_id_type. - * @feature_id: 16 bits feature identifier local to its DFL FIU type. - * @mmio_res: mmio resource of this dfl device. - * @irqs: list of Linux IRQ numbers of this dfl device. - * @num_irqs: number of IRQs supported by this dfl device. - * @cdev: pointer to DFL FPGA container device this dfl device belongs to. - * @id_entry: matched id entry in dfl driver's id table. - */ -struct dfl_device { - struct device dev; - int id; - u8 type; - u16 feature_id; - struct resource mmio_res; - int *irqs; - unsigned int num_irqs; - struct dfl_fpga_cdev *cdev; - const struct dfl_device_id *id_entry; -}; - -/** - * struct dfl_driver - represent an dfl device driver - * - * @drv: driver model structure. - * @id_table: pointer to table of device IDs the driver is interested in. - * { } member terminated. - * @probe: mandatory callback for device binding. - * @remove: callback for device unbinding. - */ -struct dfl_driver { - struct device_driver drv; - const struct dfl_device_id *id_table; - - int (*probe)(struct dfl_device *dfl_dev); - void (*remove)(struct dfl_device *dfl_dev); -}; - -#define to_dfl_dev(d) container_of(d, struct dfl_device, dev) -#define to_dfl_drv(d) container_of(d, struct dfl_driver, drv) - -/* - * use a macro to avoid include chaining to get THIS_MODULE. - */ -#define dfl_driver_register(drv) \ - __dfl_driver_register(drv, THIS_MODULE) -int __dfl_driver_register(struct dfl_driver *dfl_drv, struct module *owner); -void dfl_driver_unregister(struct dfl_driver *dfl_drv); - -/* - * module_dfl_driver() - Helper macro for drivers that don't do - * anything special in module init/exit. This eliminates a lot of - * boilerplate. Each module may only use this macro once, and - * calling it replaces module_init() and module_exit(). - */ -#define module_dfl_driver(__dfl_driver) \ - module_driver(__dfl_driver, dfl_driver_register, \ - dfl_driver_unregister) - #endif /* __FPGA_DFL_H */ diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c index 2deccacc3aa7..e9266b2a357f 100644 --- a/drivers/fpga/fpga-bridge.c +++ b/drivers/fpga/fpga-bridge.c @@ -17,7 +17,7 @@ static DEFINE_IDA(fpga_bridge_ida); static struct class *fpga_bridge_class; /* Lock for adding/removing bridges to linked lists*/ -static spinlock_t bridge_list_lock; +static DEFINE_SPINLOCK(bridge_list_lock); /** * fpga_bridge_enable - Enable transactions on the bridge @@ -479,8 +479,6 @@ static void fpga_bridge_dev_release(struct device *dev) static int __init fpga_bridge_dev_init(void) { - spin_lock_init(&bridge_list_lock); - fpga_bridge_class = class_create(THIS_MODULE, "fpga_bridge"); if (IS_ERR(fpga_bridge_class)) return PTR_ERR(fpga_bridge_class); diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index fa225175e68d..e3607ec4c2e8 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -487,11 +487,11 @@ config GPIO_PXA Say yes here to support the PXA GPIO device config GPIO_RCAR - tristate "Renesas R-Car GPIO" + tristate "Renesas R-Car and RZ/G GPIO support" depends on ARCH_RENESAS || COMPILE_TEST select GPIOLIB_IRQCHIP help - Say yes here to support GPIO on Renesas R-Car SoCs. + Say yes here to support GPIO on Renesas R-Car or RZ/G SoCs. config GPIO_RDA bool "RDA Micro GPIO controller support" @@ -595,7 +595,7 @@ config GPIO_TB10X select OF_GPIO config GPIO_TEGRA - bool "NVIDIA Tegra GPIO support" + tristate "NVIDIA Tegra GPIO support" default ARCH_TEGRA depends on ARCH_TEGRA || COMPILE_TEST depends on OF_GPIO @@ -648,6 +648,16 @@ config GPIO_VF610 help Say yes here to support Vybrid vf610 GPIOs. +config GPIO_VISCONTI + tristate "Toshiba Visconti GPIO support" + depends on ARCH_VISCONTI || COMPILE_TEST + depends on OF_GPIO + select GPIOLIB_IRQCHIP + select GPIO_GENERIC + select IRQ_DOMAIN_HIERARCHY + help + Say yes here to support GPIO on Tohisba Visconti. + config GPIO_VR41XX tristate "NEC VR4100 series General-purpose I/O Uint support" depends on CPU_VR41XX @@ -670,7 +680,7 @@ config GPIO_WCD934X tristate "Qualcomm Technologies Inc WCD9340/WCD9341 gpio controller driver" depends on MFD_WCD934X && OF_GPIO help - This driver is to supprot GPIO block found on the Qualcomm Technologies + This driver is to support GPIO block found on the Qualcomm Technologies Inc WCD9340/WCD9341 Audio Codec. config GPIO_XGENE @@ -694,6 +704,8 @@ config GPIO_XGENE_SB config GPIO_XILINX tristate "Xilinx GPIO support" + select GPIOLIB_IRQCHIP + depends on OF_GPIO help Say yes here to support the Xilinx FPGA GPIO device @@ -731,13 +743,6 @@ config GPIO_ZYNQ help Say yes here to support Xilinx Zynq GPIO controller. -config GPIO_ZX - bool "ZTE ZX GPIO support" - depends on ARCH_ZX || COMPILE_TEST - select GPIOLIB_IRQCHIP - help - Say yes here to support the GPIO device on ZTE ZX SoCs. - config GPIO_LOONGSON1 tristate "Loongson1 GPIO support" depends on MACH_LOONGSON32 @@ -1253,13 +1258,6 @@ config GPIO_MAX77650 GPIO driver for MAX77650/77651 PMIC from Maxim Semiconductor. These chips have a single pin that can be configured as GPIO. -config GPIO_MSIC - bool "Intel MSIC mixed signal gpio support" - depends on (X86 || COMPILE_TEST) && MFD_INTEL_MSIC - help - Enable support for GPIO on intel MSIC controllers found in - intel MID devices - config GPIO_PALMAS bool "TI PALMAS series PMICs GPIO" depends on MFD_PALMAS @@ -1455,13 +1453,6 @@ config GPIO_BT8XX If unsure, say N. -config GPIO_INTEL_MID - bool "Intel MID GPIO support" - depends on X86_INTEL_MID - select GPIOLIB_IRQCHIP - help - Say Y here to support Intel MID GPIO. - config GPIO_MERRIFIELD tristate "Intel Merrifield GPIO support" depends on X86_INTEL_MID @@ -1637,8 +1628,7 @@ config GPIO_MOCKUP select IRQ_SIM help This enables GPIO Testing driver, which provides a way to test GPIO - subsystem through sysfs(or char device) and debugfs. GPIO_SYSFS - must be selected for this test. + subsystem through sysfs (or char device) and debugfs. User could use it through the script in tools/testing/selftests/gpio/gpio-mockup.sh. Reference the usage in it. diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index 35e3b6026665..c58a90a3c3b1 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -67,7 +67,6 @@ obj-$(CONFIG_GPIO_HISI) += gpio-hisi.o obj-$(CONFIG_GPIO_HLWD) += gpio-hlwd.o obj-$(CONFIG_HTC_EGPIO) += gpio-htc-egpio.o obj-$(CONFIG_GPIO_ICH) += gpio-ich.o -obj-$(CONFIG_GPIO_INTEL_MID) += gpio-intel-mid.o obj-$(CONFIG_GPIO_IOP) += gpio-iop.o obj-$(CONFIG_GPIO_IT87) += gpio-it87.o obj-$(CONFIG_GPIO_IXP4XX) += gpio-ixp4xx.o @@ -103,7 +102,6 @@ obj-$(CONFIG_GPIO_MOXTET) += gpio-moxtet.o obj-$(CONFIG_GPIO_MPC5200) += gpio-mpc5200.o obj-$(CONFIG_GPIO_MPC8XXX) += gpio-mpc8xxx.o obj-$(CONFIG_GPIO_MSC313) += gpio-msc313.o -obj-$(CONFIG_GPIO_MSIC) += gpio-msic.o obj-$(CONFIG_GPIO_MT7621) += gpio-mt7621.o obj-$(CONFIG_GPIO_MVEBU) += gpio-mvebu.o obj-$(CONFIG_GPIO_MXC) += gpio-mxc.o @@ -164,6 +162,7 @@ obj-$(CONFIG_GPIO_UCB1400) += gpio-ucb1400.o obj-$(CONFIG_GPIO_UNIPHIER) += gpio-uniphier.o obj-$(CONFIG_GPIO_VF610) += gpio-vf610.o obj-$(CONFIG_GPIO_VIPERBOARD) += gpio-viperboard.o +obj-$(CONFIG_GPIO_VISCONTI) += gpio-visconti.o obj-$(CONFIG_GPIO_VR41XX) += gpio-vr41xx.o obj-$(CONFIG_GPIO_VX855) += gpio-vx855.o obj-$(CONFIG_GPIO_WCD934X) += gpio-wcd934x.o @@ -180,5 +179,4 @@ obj-$(CONFIG_GPIO_XLP) += gpio-xlp.o obj-$(CONFIG_GPIO_XRA1403) += gpio-xra1403.o obj-$(CONFIG_GPIO_XTENSA) += gpio-xtensa.o obj-$(CONFIG_GPIO_ZEVIO) += gpio-zevio.o -obj-$(CONFIG_GPIO_ZX) += gpio-zx.o obj-$(CONFIG_GPIO_ZYNQ) += gpio-zynq.o diff --git a/drivers/gpio/TODO b/drivers/gpio/TODO index 0229fa79499e..b8b1473a5b1e 100644 --- a/drivers/gpio/TODO +++ b/drivers/gpio/TODO @@ -101,7 +101,7 @@ for a few GPIOs. Those should stay where they are. At the same time it makes sense to get rid of code duplication in existing or new coming drivers. For example, gpio-ml-ioh should be incorporated into -gpio-pch. In similar way gpio-intel-mid into gpio-pxa. +gpio-pch. Generic MMIO GPIO diff --git a/drivers/gpio/gpio-aggregator.c b/drivers/gpio/gpio-aggregator.c index dfd8a4876a27..08171431bb8f 100644 --- a/drivers/gpio/gpio-aggregator.c +++ b/drivers/gpio/gpio-aggregator.c @@ -62,34 +62,6 @@ static char *get_arg(char **args) return start; } -static bool isrange(const char *s) -{ - size_t n; - - if (IS_ERR_OR_NULL(s)) - return false; - - while (1) { - n = strspn(s, "0123456789"); - if (!n) - return false; - - s += n; - - switch (*s++) { - case '\0': - return true; - - case '-': - case ',': - break; - - default: - return false; - } - } -} - static int aggr_add_gpio(struct gpio_aggregator *aggr, const char *key, int hwnum, unsigned int *n) { @@ -100,8 +72,7 @@ static int aggr_add_gpio(struct gpio_aggregator *aggr, const char *key, if (!lookups) return -ENOMEM; - lookups->table[*n] = - (struct gpiod_lookup)GPIO_LOOKUP_IDX(key, hwnum, NULL, *n, 0); + lookups->table[*n] = GPIO_LOOKUP_IDX(key, hwnum, NULL, *n, 0); (*n)++; memset(&lookups->table[*n], 0, sizeof(lookups->table[*n])); @@ -112,10 +83,10 @@ static int aggr_add_gpio(struct gpio_aggregator *aggr, const char *key, static int aggr_parse(struct gpio_aggregator *aggr) { + char *name, *offsets, *p; char *args = aggr->args; unsigned long *bitmap; unsigned int i, n = 0; - char *name, *offsets; int error = 0; bitmap = bitmap_alloc(ARCH_NR_GPIOS, GFP_KERNEL); @@ -130,7 +101,8 @@ static int aggr_parse(struct gpio_aggregator *aggr) goto free_bitmap; } - if (!isrange(offsets)) { + p = get_options(offsets, 0, &error); + if (error == 0 || *p) { /* Named GPIO line */ error = aggr_add_gpio(aggr, name, U16_MAX, &n); if (error) @@ -271,7 +243,7 @@ static DRIVER_ATTR_WO(delete_device); static struct attribute *gpio_aggregator_attrs[] = { &driver_attr_new_device.attr, &driver_attr_delete_device.attr, - NULL, + NULL }; ATTRIBUTE_GROUPS(gpio_aggregator); @@ -545,7 +517,7 @@ static const struct of_device_id gpio_aggregator_dt_ids[] = { * Add GPIO-operated devices controlled from userspace below, * or use "driver_override" in sysfs */ - {}, + {} }; MODULE_DEVICE_TABLE(of, gpio_aggregator_dt_ids); #endif diff --git a/drivers/gpio/gpio-bd70528.c b/drivers/gpio/gpio-bd70528.c index 45b3da8da336..397a50d6bc65 100644 --- a/drivers/gpio/gpio-bd70528.c +++ b/drivers/gpio/gpio-bd70528.c @@ -12,7 +12,8 @@ #define GPIO_OUT_REG(offset) (BD70528_REG_GPIO1_OUT + (offset) * 2) struct bd70528_gpio { - struct rohm_regmap_dev chip; + struct regmap *regmap; + struct device *dev; struct gpio_chip gpio; }; @@ -35,11 +36,11 @@ static int bd70528_set_debounce(struct bd70528_gpio *bdgpio, val = BD70528_DEBOUNCE_50MS; break; default: - dev_err(bdgpio->chip.dev, + dev_err(bdgpio->dev, "Invalid debounce value %u\n", debounce); return -EINVAL; } - return regmap_update_bits(bdgpio->chip.regmap, GPIO_IN_REG(offset), + return regmap_update_bits(bdgpio->regmap, GPIO_IN_REG(offset), BD70528_DEBOUNCE_MASK, val); } @@ -49,9 +50,9 @@ static int bd70528_get_direction(struct gpio_chip *chip, unsigned int offset) int val, ret; /* Do we need to do something to IRQs here? */ - ret = regmap_read(bdgpio->chip.regmap, GPIO_OUT_REG(offset), &val); + ret = regmap_read(bdgpio->regmap, GPIO_OUT_REG(offset), &val); if (ret) { - dev_err(bdgpio->chip.dev, "Could not read gpio direction\n"); + dev_err(bdgpio->dev, "Could not read gpio direction\n"); return ret; } if (val & BD70528_GPIO_OUT_EN_MASK) @@ -67,13 +68,13 @@ static int bd70528_gpio_set_config(struct gpio_chip *chip, unsigned int offset, switch (pinconf_to_config_param(config)) { case PIN_CONFIG_DRIVE_OPEN_DRAIN: - return regmap_update_bits(bdgpio->chip.regmap, + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD70528_GPIO_DRIVE_MASK, BD70528_GPIO_OPEN_DRAIN); break; case PIN_CONFIG_DRIVE_PUSH_PULL: - return regmap_update_bits(bdgpio->chip.regmap, + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD70528_GPIO_DRIVE_MASK, BD70528_GPIO_PUSH_PULL); @@ -93,7 +94,7 @@ static int bd70528_direction_input(struct gpio_chip *chip, unsigned int offset) struct bd70528_gpio *bdgpio = gpiochip_get_data(chip); /* Do we need to do something to IRQs here? */ - return regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD70528_GPIO_OUT_EN_MASK, BD70528_GPIO_OUT_DISABLE); } @@ -105,10 +106,10 @@ static void bd70528_gpio_set(struct gpio_chip *chip, unsigned int offset, struct bd70528_gpio *bdgpio = gpiochip_get_data(chip); u8 val = (value) ? BD70528_GPIO_OUT_HI : BD70528_GPIO_OUT_LO; - ret = regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + ret = regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD70528_GPIO_OUT_MASK, val); if (ret) - dev_err(bdgpio->chip.dev, "Could not set gpio to %d\n", value); + dev_err(bdgpio->dev, "Could not set gpio to %d\n", value); } static int bd70528_direction_output(struct gpio_chip *chip, unsigned int offset, @@ -117,7 +118,7 @@ static int bd70528_direction_output(struct gpio_chip *chip, unsigned int offset, struct bd70528_gpio *bdgpio = gpiochip_get_data(chip); bd70528_gpio_set(chip, offset, value); - return regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD70528_GPIO_OUT_EN_MASK, BD70528_GPIO_OUT_ENABLE); } @@ -129,11 +130,11 @@ static int bd70528_gpio_get_o(struct bd70528_gpio *bdgpio, unsigned int offset) int ret; unsigned int val; - ret = regmap_read(bdgpio->chip.regmap, GPIO_OUT_REG(offset), &val); + ret = regmap_read(bdgpio->regmap, GPIO_OUT_REG(offset), &val); if (!ret) ret = !!(val & BD70528_GPIO_OUT_MASK); else - dev_err(bdgpio->chip.dev, "GPIO (out) state read failed\n"); + dev_err(bdgpio->dev, "GPIO (out) state read failed\n"); return ret; } @@ -143,12 +144,12 @@ static int bd70528_gpio_get_i(struct bd70528_gpio *bdgpio, unsigned int offset) unsigned int val; int ret; - ret = regmap_read(bdgpio->chip.regmap, BD70528_REG_GPIO_STATE, &val); + ret = regmap_read(bdgpio->regmap, BD70528_REG_GPIO_STATE, &val); if (!ret) ret = !(val & GPIO_IN_STATE_MASK(offset)); else - dev_err(bdgpio->chip.dev, "GPIO (in) state read failed\n"); + dev_err(bdgpio->dev, "GPIO (in) state read failed\n"); return ret; } @@ -173,29 +174,22 @@ static int bd70528_gpio_get(struct gpio_chip *chip, unsigned int offset) else if (ret == GPIO_LINE_DIRECTION_IN) ret = bd70528_gpio_get_i(bdgpio, offset); else - dev_err(bdgpio->chip.dev, "failed to read GPIO direction\n"); + dev_err(bdgpio->dev, "failed to read GPIO direction\n"); return ret; } static int bd70528_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct bd70528_gpio *bdgpio; - struct rohm_regmap_dev *bd70528; int ret; - bd70528 = dev_get_drvdata(pdev->dev.parent); - if (!bd70528) { - dev_err(&pdev->dev, "No MFD driver data\n"); - return -EINVAL; - } - - bdgpio = devm_kzalloc(&pdev->dev, sizeof(*bdgpio), - GFP_KERNEL); + bdgpio = devm_kzalloc(dev, sizeof(*bdgpio), GFP_KERNEL); if (!bdgpio) return -ENOMEM; - bdgpio->chip.dev = &pdev->dev; - bdgpio->gpio.parent = pdev->dev.parent; + bdgpio->dev = dev; + bdgpio->gpio.parent = dev->parent; bdgpio->gpio.label = "bd70528-gpio"; bdgpio->gpio.owner = THIS_MODULE; bdgpio->gpio.get_direction = bd70528_get_direction; @@ -208,14 +202,15 @@ static int bd70528_probe(struct platform_device *pdev) bdgpio->gpio.ngpio = 4; bdgpio->gpio.base = -1; #ifdef CONFIG_OF_GPIO - bdgpio->gpio.of_node = pdev->dev.parent->of_node; + bdgpio->gpio.of_node = dev->parent->of_node; #endif - bdgpio->chip.regmap = bd70528->regmap; + bdgpio->regmap = dev_get_regmap(dev->parent, NULL); + if (!bdgpio->regmap) + return -ENODEV; - ret = devm_gpiochip_add_data(&pdev->dev, &bdgpio->gpio, - bdgpio); + ret = devm_gpiochip_add_data(dev, &bdgpio->gpio, bdgpio); if (ret) - dev_err(&pdev->dev, "gpio_init: Failed to add bd70528-gpio\n"); + dev_err(dev, "gpio_init: Failed to add bd70528-gpio\n"); return ret; } diff --git a/drivers/gpio/gpio-bd71828.c b/drivers/gpio/gpio-bd71828.c index 3dbbc638e9a9..c8e382b53f2f 100644 --- a/drivers/gpio/gpio-bd71828.c +++ b/drivers/gpio/gpio-bd71828.c @@ -11,7 +11,8 @@ #define HALL_GPIO_OFFSET 3 struct bd71828_gpio { - struct rohm_regmap_dev chip; + struct regmap *regmap; + struct device *dev; struct gpio_chip gpio; }; @@ -29,10 +30,10 @@ static void bd71828_gpio_set(struct gpio_chip *chip, unsigned int offset, if (offset == HALL_GPIO_OFFSET) return; - ret = regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + ret = regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD71828_GPIO_OUT_MASK, val); if (ret) - dev_err(bdgpio->chip.dev, "Could not set gpio to %d\n", value); + dev_err(bdgpio->dev, "Could not set gpio to %d\n", value); } static int bd71828_gpio_get(struct gpio_chip *chip, unsigned int offset) @@ -42,10 +43,10 @@ static int bd71828_gpio_get(struct gpio_chip *chip, unsigned int offset) struct bd71828_gpio *bdgpio = gpiochip_get_data(chip); if (offset == HALL_GPIO_OFFSET) - ret = regmap_read(bdgpio->chip.regmap, BD71828_REG_IO_STAT, + ret = regmap_read(bdgpio->regmap, BD71828_REG_IO_STAT, &val); else - ret = regmap_read(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + ret = regmap_read(bdgpio->regmap, GPIO_OUT_REG(offset), &val); if (!ret) ret = (val & BD71828_GPIO_OUT_MASK); @@ -63,12 +64,12 @@ static int bd71828_gpio_set_config(struct gpio_chip *chip, unsigned int offset, switch (pinconf_to_config_param(config)) { case PIN_CONFIG_DRIVE_OPEN_DRAIN: - return regmap_update_bits(bdgpio->chip.regmap, + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD71828_GPIO_DRIVE_MASK, BD71828_GPIO_OPEN_DRAIN); case PIN_CONFIG_DRIVE_PUSH_PULL: - return regmap_update_bits(bdgpio->chip.regmap, + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD71828_GPIO_DRIVE_MASK, BD71828_GPIO_PUSH_PULL); @@ -96,22 +97,15 @@ static int bd71828_get_direction(struct gpio_chip *chip, unsigned int offset) static int bd71828_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct bd71828_gpio *bdgpio; - struct rohm_regmap_dev *bd71828; - bd71828 = dev_get_drvdata(pdev->dev.parent); - if (!bd71828) { - dev_err(&pdev->dev, "No MFD driver data\n"); - return -EINVAL; - } - - bdgpio = devm_kzalloc(&pdev->dev, sizeof(*bdgpio), - GFP_KERNEL); + bdgpio = devm_kzalloc(dev, sizeof(*bdgpio), GFP_KERNEL); if (!bdgpio) return -ENOMEM; - bdgpio->chip.dev = &pdev->dev; - bdgpio->gpio.parent = pdev->dev.parent; + bdgpio->dev = dev; + bdgpio->gpio.parent = dev->parent; bdgpio->gpio.label = "bd71828-gpio"; bdgpio->gpio.owner = THIS_MODULE; bdgpio->gpio.get_direction = bd71828_get_direction; @@ -127,11 +121,12 @@ static int bd71828_probe(struct platform_device *pdev) * "gpio-reserved-ranges" and exclude them from control */ bdgpio->gpio.ngpio = 4; - bdgpio->gpio.of_node = pdev->dev.parent->of_node; - bdgpio->chip.regmap = bd71828->regmap; + bdgpio->gpio.of_node = dev->parent->of_node; + bdgpio->regmap = dev_get_regmap(dev->parent, NULL); + if (!bdgpio->regmap) + return -ENODEV; - return devm_gpiochip_add_data(&pdev->dev, &bdgpio->gpio, - bdgpio); + return devm_gpiochip_add_data(dev, &bdgpio->gpio, bdgpio); } static struct platform_driver bd71828_gpio = { diff --git a/drivers/gpio/gpio-bd9571mwv.c b/drivers/gpio/gpio-bd9571mwv.c index c0abc9c6851b..df6102b57734 100644 --- a/drivers/gpio/gpio-bd9571mwv.c +++ b/drivers/gpio/gpio-bd9571mwv.c @@ -1,31 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only /* - * ROHM BD9571MWV-M GPIO driver + * ROHM BD9571MWV-M and BD9574MWF-M GPIO driver * * Copyright (C) 2017 Marek Vasut * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65086 driver * * NOTE: Interrupts are not supported yet. */ #include +#include #include #include #include struct bd9571mwv_gpio { + struct regmap *regmap; struct gpio_chip chip; - struct bd9571mwv *bd; }; static int bd9571mwv_gpio_get_direction(struct gpio_chip *chip, @@ -34,7 +27,7 @@ static int bd9571mwv_gpio_get_direction(struct gpio_chip *chip, struct bd9571mwv_gpio *gpio = gpiochip_get_data(chip); int ret, val; - ret = regmap_read(gpio->bd->regmap, BD9571MWV_GPIO_DIR, &val); + ret = regmap_read(gpio->regmap, BD9571MWV_GPIO_DIR, &val); if (ret < 0) return ret; if (val & BIT(offset)) @@ -48,8 +41,7 @@ static int bd9571mwv_gpio_direction_input(struct gpio_chip *chip, { struct bd9571mwv_gpio *gpio = gpiochip_get_data(chip); - regmap_update_bits(gpio->bd->regmap, BD9571MWV_GPIO_DIR, - BIT(offset), 0); + regmap_update_bits(gpio->regmap, BD9571MWV_GPIO_DIR, BIT(offset), 0); return 0; } @@ -60,9 +52,9 @@ static int bd9571mwv_gpio_direction_output(struct gpio_chip *chip, struct bd9571mwv_gpio *gpio = gpiochip_get_data(chip); /* Set the initial value */ - regmap_update_bits(gpio->bd->regmap, BD9571MWV_GPIO_OUT, + regmap_update_bits(gpio->regmap, BD9571MWV_GPIO_OUT, BIT(offset), value ? BIT(offset) : 0); - regmap_update_bits(gpio->bd->regmap, BD9571MWV_GPIO_DIR, + regmap_update_bits(gpio->regmap, BD9571MWV_GPIO_DIR, BIT(offset), BIT(offset)); return 0; @@ -73,7 +65,7 @@ static int bd9571mwv_gpio_get(struct gpio_chip *chip, unsigned int offset) struct bd9571mwv_gpio *gpio = gpiochip_get_data(chip); int ret, val; - ret = regmap_read(gpio->bd->regmap, BD9571MWV_GPIO_IN, &val); + ret = regmap_read(gpio->regmap, BD9571MWV_GPIO_IN, &val); if (ret < 0) return ret; @@ -85,7 +77,7 @@ static void bd9571mwv_gpio_set(struct gpio_chip *chip, unsigned int offset, { struct bd9571mwv_gpio *gpio = gpiochip_get_data(chip); - regmap_update_bits(gpio->bd->regmap, BD9571MWV_GPIO_OUT, + regmap_update_bits(gpio->regmap, BD9571MWV_GPIO_OUT, BIT(offset), value ? BIT(offset) : 0); } @@ -113,9 +105,9 @@ static int bd9571mwv_gpio_probe(struct platform_device *pdev) platform_set_drvdata(pdev, gpio); - gpio->bd = dev_get_drvdata(pdev->dev.parent); + gpio->regmap = dev_get_regmap(pdev->dev.parent, NULL); gpio->chip = template_chip; - gpio->chip.parent = gpio->bd->dev; + gpio->chip.parent = pdev->dev.parent; ret = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio); if (ret < 0) { @@ -127,7 +119,8 @@ static int bd9571mwv_gpio_probe(struct platform_device *pdev) } static const struct platform_device_id bd9571mwv_gpio_id_table[] = { - { "bd9571mwv-gpio", }, + { "bd9571mwv-gpio", ROHM_CHIP_TYPE_BD9571 }, + { "bd9574mwf-gpio", ROHM_CHIP_TYPE_BD9574 }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(platform, bd9571mwv_gpio_id_table); diff --git a/drivers/gpio/gpio-ep93xx.c b/drivers/gpio/gpio-ep93xx.c index 94d9fa0d6aa7..ef148b26b587 100644 --- a/drivers/gpio/gpio-ep93xx.c +++ b/drivers/gpio/gpio-ep93xx.c @@ -31,6 +31,8 @@ /* Maximum value for irq capable line identifiers */ #define EP93XX_GPIO_LINE_MAX_IRQ 23 +#define EP93XX_GPIO_A_IRQ_BASE 64 +#define EP93XX_GPIO_B_IRQ_BASE 72 /* * Static mapping of GPIO bank F IRQS: * F0..F7 (16..24) to irq 80..87. @@ -292,14 +294,14 @@ struct ep93xx_gpio_bank { static struct ep93xx_gpio_bank ep93xx_gpio_banks[] = { /* Bank A has 8 IRQs */ - EP93XX_GPIO_BANK("A", 0x00, 0x10, 0x90, 0, true, false, 64), + EP93XX_GPIO_BANK("A", 0x00, 0x10, 0x90, 0, true, false, EP93XX_GPIO_A_IRQ_BASE), /* Bank B has 8 IRQs */ - EP93XX_GPIO_BANK("B", 0x04, 0x14, 0xac, 8, true, false, 72), + EP93XX_GPIO_BANK("B", 0x04, 0x14, 0xac, 8, true, false, EP93XX_GPIO_B_IRQ_BASE), EP93XX_GPIO_BANK("C", 0x08, 0x18, 0x00, 40, false, false, 0), EP93XX_GPIO_BANK("D", 0x0c, 0x1c, 0x00, 24, false, false, 0), EP93XX_GPIO_BANK("E", 0x20, 0x24, 0x00, 32, false, false, 0), /* Bank F has 8 IRQs */ - EP93XX_GPIO_BANK("F", 0x30, 0x34, 0x4c, 16, false, true, 0), + EP93XX_GPIO_BANK("F", 0x30, 0x34, 0x4c, 16, false, true, EP93XX_GPIO_F_IRQ_BASE), EP93XX_GPIO_BANK("G", 0x38, 0x3c, 0x00, 48, false, false, 0), EP93XX_GPIO_BANK("H", 0x40, 0x44, 0x00, 56, false, false, 0), }; @@ -318,11 +320,6 @@ static int ep93xx_gpio_set_config(struct gpio_chip *gc, unsigned offset, return 0; } -static int ep93xx_gpio_f_to_irq(struct gpio_chip *gc, unsigned offset) -{ - return EP93XX_GPIO_F_IRQ_BASE + offset; -} - static void ep93xx_init_irq_chip(struct device *dev, struct irq_chip *ic) { ic->irq_ack = ep93xx_gpio_irq_ack; @@ -375,7 +372,7 @@ static int ep93xx_gpio_add_bank(struct ep93xx_gpio_chip *egc, girq->parent_handler = ep93xx_gpio_ab_irq_handler; girq->num_parents = 1; - girq->parents = devm_kcalloc(dev, 1, + girq->parents = devm_kcalloc(dev, girq->num_parents, sizeof(*girq->parents), GFP_KERNEL); if (!girq->parents) @@ -393,20 +390,19 @@ static int ep93xx_gpio_add_bank(struct ep93xx_gpio_chip *egc, /* * FIXME: convert this to use hierarchical IRQ support! - * this requires fixing the root irqchip to be hierarchial. + * this requires fixing the root irqchip to be hierarchical. */ girq->parent_handler = ep93xx_gpio_f_irq_handler; girq->num_parents = 8; - girq->parents = devm_kcalloc(dev, 8, + girq->parents = devm_kcalloc(dev, girq->num_parents, sizeof(*girq->parents), GFP_KERNEL); if (!girq->parents) return -ENOMEM; /* Pick resources 1..8 for these IRQs */ - for (i = 1; i <= 8; i++) - girq->parents[i - 1] = platform_get_irq(pdev, i); - for (i = 0; i < 8; i++) { - gpio_irq = EP93XX_GPIO_F_IRQ_BASE + i; + for (i = 0; i < girq->num_parents; i++) { + girq->parents[i] = platform_get_irq(pdev, i + 1); + gpio_irq = bank->irq_base + i; irq_set_chip_data(gpio_irq, &epg->gc[5]); irq_set_chip_and_handler(gpio_irq, girq->chip, @@ -415,7 +411,7 @@ static int ep93xx_gpio_add_bank(struct ep93xx_gpio_chip *egc, } girq->default_type = IRQ_TYPE_NONE; girq->handler = handle_level_irq; - gc->to_irq = ep93xx_gpio_f_to_irq; + girq->first = bank->irq_base; } return devm_gpiochip_add_data(dev, gc, epg); diff --git a/drivers/gpio/gpio-intel-mid.c b/drivers/gpio/gpio-intel-mid.c deleted file mode 100644 index 86a10c808ef6..000000000000 --- a/drivers/gpio/gpio-intel-mid.c +++ /dev/null @@ -1,414 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Intel MID GPIO driver - * - * Copyright (c) 2008-2014,2016 Intel Corporation. - */ - -/* Supports: - * Moorestown platform Langwell chip. - * Medfield platform Penwell chip. - * Clovertrail platform Cloverview chip. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define INTEL_MID_IRQ_TYPE_EDGE (1 << 0) -#define INTEL_MID_IRQ_TYPE_LEVEL (1 << 1) - -/* - * Langwell chip has 64 pins and thus there are 2 32bit registers to control - * each feature, while Penwell chip has 96 pins for each block, and need 3 32bit - * registers to control them, so we only define the order here instead of a - * structure, to get a bit offset for a pin (use GPDR as an example): - * - * nreg = ngpio / 32; - * reg = offset / 32; - * bit = offset % 32; - * reg_addr = reg_base + GPDR * nreg * 4 + reg * 4; - * - * so the bit of reg_addr is to control pin offset's GPDR feature -*/ - -enum GPIO_REG { - GPLR = 0, /* pin level read-only */ - GPDR, /* pin direction */ - GPSR, /* pin set */ - GPCR, /* pin clear */ - GRER, /* rising edge detect */ - GFER, /* falling edge detect */ - GEDR, /* edge detect result */ - GAFR, /* alt function */ -}; - -/* intel_mid gpio driver data */ -struct intel_mid_gpio_ddata { - u16 ngpio; /* number of gpio pins */ - u32 chip_irq_type; /* chip interrupt type */ -}; - -struct intel_mid_gpio { - struct gpio_chip chip; - void __iomem *reg_base; - spinlock_t lock; - struct pci_dev *pdev; -}; - -static void __iomem *gpio_reg(struct gpio_chip *chip, unsigned offset, - enum GPIO_REG reg_type) -{ - struct intel_mid_gpio *priv = gpiochip_get_data(chip); - unsigned nreg = chip->ngpio / 32; - u8 reg = offset / 32; - - return priv->reg_base + reg_type * nreg * 4 + reg * 4; -} - -static void __iomem *gpio_reg_2bit(struct gpio_chip *chip, unsigned offset, - enum GPIO_REG reg_type) -{ - struct intel_mid_gpio *priv = gpiochip_get_data(chip); - unsigned nreg = chip->ngpio / 32; - u8 reg = offset / 16; - - return priv->reg_base + reg_type * nreg * 4 + reg * 4; -} - -static int intel_gpio_request(struct gpio_chip *chip, unsigned offset) -{ - void __iomem *gafr = gpio_reg_2bit(chip, offset, GAFR); - u32 value = readl(gafr); - int shift = (offset % 16) << 1, af = (value >> shift) & 3; - - if (af) { - value &= ~(3 << shift); - writel(value, gafr); - } - return 0; -} - -static int intel_gpio_get(struct gpio_chip *chip, unsigned offset) -{ - void __iomem *gplr = gpio_reg(chip, offset, GPLR); - - return !!(readl(gplr) & BIT(offset % 32)); -} - -static void intel_gpio_set(struct gpio_chip *chip, unsigned offset, int value) -{ - void __iomem *gpsr, *gpcr; - - if (value) { - gpsr = gpio_reg(chip, offset, GPSR); - writel(BIT(offset % 32), gpsr); - } else { - gpcr = gpio_reg(chip, offset, GPCR); - writel(BIT(offset % 32), gpcr); - } -} - -static int intel_gpio_direction_input(struct gpio_chip *chip, unsigned offset) -{ - struct intel_mid_gpio *priv = gpiochip_get_data(chip); - void __iomem *gpdr = gpio_reg(chip, offset, GPDR); - u32 value; - unsigned long flags; - - if (priv->pdev) - pm_runtime_get(&priv->pdev->dev); - - spin_lock_irqsave(&priv->lock, flags); - value = readl(gpdr); - value &= ~BIT(offset % 32); - writel(value, gpdr); - spin_unlock_irqrestore(&priv->lock, flags); - - if (priv->pdev) - pm_runtime_put(&priv->pdev->dev); - - return 0; -} - -static int intel_gpio_direction_output(struct gpio_chip *chip, - unsigned offset, int value) -{ - struct intel_mid_gpio *priv = gpiochip_get_data(chip); - void __iomem *gpdr = gpio_reg(chip, offset, GPDR); - unsigned long flags; - - intel_gpio_set(chip, offset, value); - - if (priv->pdev) - pm_runtime_get(&priv->pdev->dev); - - spin_lock_irqsave(&priv->lock, flags); - value = readl(gpdr); - value |= BIT(offset % 32); - writel(value, gpdr); - spin_unlock_irqrestore(&priv->lock, flags); - - if (priv->pdev) - pm_runtime_put(&priv->pdev->dev); - - return 0; -} - -static int intel_mid_irq_type(struct irq_data *d, unsigned type) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct intel_mid_gpio *priv = gpiochip_get_data(gc); - u32 gpio = irqd_to_hwirq(d); - unsigned long flags; - u32 value; - void __iomem *grer = gpio_reg(&priv->chip, gpio, GRER); - void __iomem *gfer = gpio_reg(&priv->chip, gpio, GFER); - - if (gpio >= priv->chip.ngpio) - return -EINVAL; - - if (priv->pdev) - pm_runtime_get(&priv->pdev->dev); - - spin_lock_irqsave(&priv->lock, flags); - if (type & IRQ_TYPE_EDGE_RISING) - value = readl(grer) | BIT(gpio % 32); - else - value = readl(grer) & (~BIT(gpio % 32)); - writel(value, grer); - - if (type & IRQ_TYPE_EDGE_FALLING) - value = readl(gfer) | BIT(gpio % 32); - else - value = readl(gfer) & (~BIT(gpio % 32)); - writel(value, gfer); - spin_unlock_irqrestore(&priv->lock, flags); - - if (priv->pdev) - pm_runtime_put(&priv->pdev->dev); - - return 0; -} - -static void intel_mid_irq_unmask(struct irq_data *d) -{ -} - -static void intel_mid_irq_mask(struct irq_data *d) -{ -} - -static struct irq_chip intel_mid_irqchip = { - .name = "INTEL_MID-GPIO", - .irq_mask = intel_mid_irq_mask, - .irq_unmask = intel_mid_irq_unmask, - .irq_set_type = intel_mid_irq_type, -}; - -static const struct intel_mid_gpio_ddata gpio_lincroft = { - .ngpio = 64, -}; - -static const struct intel_mid_gpio_ddata gpio_penwell_aon = { - .ngpio = 96, - .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE, -}; - -static const struct intel_mid_gpio_ddata gpio_penwell_core = { - .ngpio = 96, - .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE, -}; - -static const struct intel_mid_gpio_ddata gpio_cloverview_aon = { - .ngpio = 96, - .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE | INTEL_MID_IRQ_TYPE_LEVEL, -}; - -static const struct intel_mid_gpio_ddata gpio_cloverview_core = { - .ngpio = 96, - .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE, -}; - -static const struct pci_device_id intel_gpio_ids[] = { - { - /* Lincroft */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x080f), - .driver_data = (kernel_ulong_t)&gpio_lincroft, - }, - { - /* Penwell AON */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x081f), - .driver_data = (kernel_ulong_t)&gpio_penwell_aon, - }, - { - /* Penwell Core */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x081a), - .driver_data = (kernel_ulong_t)&gpio_penwell_core, - }, - { - /* Cloverview Aon */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x08eb), - .driver_data = (kernel_ulong_t)&gpio_cloverview_aon, - }, - { - /* Cloverview Core */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x08f7), - .driver_data = (kernel_ulong_t)&gpio_cloverview_core, - }, - { } -}; - -static void intel_mid_irq_handler(struct irq_desc *desc) -{ - struct gpio_chip *gc = irq_desc_get_handler_data(desc); - struct intel_mid_gpio *priv = gpiochip_get_data(gc); - struct irq_data *data = irq_desc_get_irq_data(desc); - struct irq_chip *chip = irq_data_get_irq_chip(data); - u32 base, gpio, mask; - unsigned long pending; - void __iomem *gedr; - - /* check GPIO controller to check which pin triggered the interrupt */ - for (base = 0; base < priv->chip.ngpio; base += 32) { - gedr = gpio_reg(&priv->chip, base, GEDR); - while ((pending = readl(gedr))) { - gpio = __ffs(pending); - mask = BIT(gpio); - /* Clear before handling so we can't lose an edge */ - writel(mask, gedr); - generic_handle_irq(irq_find_mapping(gc->irq.domain, - base + gpio)); - } - } - - chip->irq_eoi(data); -} - -static int intel_mid_irq_init_hw(struct gpio_chip *chip) -{ - struct intel_mid_gpio *priv = gpiochip_get_data(chip); - void __iomem *reg; - unsigned base; - - for (base = 0; base < priv->chip.ngpio; base += 32) { - /* Clear the rising-edge detect register */ - reg = gpio_reg(&priv->chip, base, GRER); - writel(0, reg); - /* Clear the falling-edge detect register */ - reg = gpio_reg(&priv->chip, base, GFER); - writel(0, reg); - /* Clear the edge detect status register */ - reg = gpio_reg(&priv->chip, base, GEDR); - writel(~0, reg); - } - - return 0; -} - -static int __maybe_unused intel_gpio_runtime_idle(struct device *dev) -{ - int err = pm_schedule_suspend(dev, 500); - return err ?: -EBUSY; -} - -static const struct dev_pm_ops intel_gpio_pm_ops = { - SET_RUNTIME_PM_OPS(NULL, NULL, intel_gpio_runtime_idle) -}; - -static int intel_gpio_probe(struct pci_dev *pdev, - const struct pci_device_id *id) -{ - void __iomem *base; - struct intel_mid_gpio *priv; - u32 gpio_base; - u32 irq_base; - int retval; - struct gpio_irq_chip *girq; - struct intel_mid_gpio_ddata *ddata = - (struct intel_mid_gpio_ddata *)id->driver_data; - - retval = pcim_enable_device(pdev); - if (retval) - return retval; - - retval = pcim_iomap_regions(pdev, 1 << 0 | 1 << 1, pci_name(pdev)); - if (retval) { - dev_err(&pdev->dev, "I/O memory mapping error\n"); - return retval; - } - - base = pcim_iomap_table(pdev)[1]; - - irq_base = readl(base); - gpio_base = readl(sizeof(u32) + base); - - /* release the IO mapping, since we already get the info from bar1 */ - pcim_iounmap_regions(pdev, 1 << 1); - - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - priv->reg_base = pcim_iomap_table(pdev)[0]; - priv->chip.label = dev_name(&pdev->dev); - priv->chip.parent = &pdev->dev; - priv->chip.request = intel_gpio_request; - priv->chip.direction_input = intel_gpio_direction_input; - priv->chip.direction_output = intel_gpio_direction_output; - priv->chip.get = intel_gpio_get; - priv->chip.set = intel_gpio_set; - priv->chip.base = gpio_base; - priv->chip.ngpio = ddata->ngpio; - priv->chip.can_sleep = false; - priv->pdev = pdev; - - spin_lock_init(&priv->lock); - - girq = &priv->chip.irq; - girq->chip = &intel_mid_irqchip; - girq->init_hw = intel_mid_irq_init_hw; - girq->parent_handler = intel_mid_irq_handler; - girq->num_parents = 1; - girq->parents = devm_kcalloc(&pdev->dev, girq->num_parents, - sizeof(*girq->parents), - GFP_KERNEL); - if (!girq->parents) - return -ENOMEM; - girq->parents[0] = pdev->irq; - girq->first = irq_base; - girq->default_type = IRQ_TYPE_NONE; - girq->handler = handle_simple_irq; - - pci_set_drvdata(pdev, priv); - - retval = devm_gpiochip_add_data(&pdev->dev, &priv->chip, priv); - if (retval) { - dev_err(&pdev->dev, "gpiochip_add error %d\n", retval); - return retval; - } - - pm_runtime_put_noidle(&pdev->dev); - pm_runtime_allow(&pdev->dev); - - return 0; -} - -static struct pci_driver intel_gpio_driver = { - .name = "intel_mid_gpio", - .id_table = intel_gpio_ids, - .probe = intel_gpio_probe, - .driver = { - .pm = &intel_gpio_pm_ops, - }, -}; - -builtin_pci_driver(intel_gpio_driver); diff --git a/drivers/gpio/gpio-max77620.c b/drivers/gpio/gpio-max77620.c index 7c0a9ef0b500..82b3a913005d 100644 --- a/drivers/gpio/gpio-max77620.c +++ b/drivers/gpio/gpio-max77620.c @@ -325,7 +325,7 @@ static int max77620_gpio_probe(struct platform_device *pdev) girq->parents = NULL; girq->default_type = IRQ_TYPE_NONE; girq->handler = handle_edge_irq; - girq->init_hw = max77620_gpio_irq_init_hw, + girq->init_hw = max77620_gpio_irq_init_hw; girq->threaded = true; platform_set_drvdata(pdev, mgpio); diff --git a/drivers/gpio/gpio-merrifield.c b/drivers/gpio/gpio-merrifield.c index 706687fab634..22f3ce218f5d 100644 --- a/drivers/gpio/gpio-merrifield.c +++ b/drivers/gpio/gpio-merrifield.c @@ -194,6 +194,11 @@ static int mrfld_gpio_set_config(struct gpio_chip *chip, unsigned int offset, { u32 debounce; + if ((pinconf_to_config_param(config) == PIN_CONFIG_BIAS_DISABLE) || + (pinconf_to_config_param(config) == PIN_CONFIG_BIAS_PULL_UP) || + (pinconf_to_config_param(config) == PIN_CONFIG_BIAS_PULL_DOWN)) + return gpiochip_generic_config(chip, offset, config); + if (pinconf_to_config_param(config) != PIN_CONFIG_INPUT_DEBOUNCE) return -ENOTSUPP; diff --git a/drivers/gpio/gpio-msic.c b/drivers/gpio/gpio-msic.c deleted file mode 100644 index 7e3c96e4ab2c..000000000000 --- a/drivers/gpio/gpio-msic.c +++ /dev/null @@ -1,314 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Intel Medfield MSIC GPIO driver> - * Copyright (c) 2011, Intel Corporation. - * - * Author: Mathias Nyman - * Based on intel_pmic_gpio.c - */ - -#include -#include -#include -#include -#include -#include -#include - -/* the offset for the mapping of global gpio pin to irq */ -#define MSIC_GPIO_IRQ_OFFSET 0x100 - -#define MSIC_GPIO_DIR_IN 0 -#define MSIC_GPIO_DIR_OUT BIT(5) -#define MSIC_GPIO_TRIG_FALL BIT(1) -#define MSIC_GPIO_TRIG_RISE BIT(2) - -/* masks for msic gpio output GPIOxxxxCTLO registers */ -#define MSIC_GPIO_DIR_MASK BIT(5) -#define MSIC_GPIO_DRV_MASK BIT(4) -#define MSIC_GPIO_REN_MASK BIT(3) -#define MSIC_GPIO_RVAL_MASK (BIT(2) | BIT(1)) -#define MSIC_GPIO_DOUT_MASK BIT(0) - -/* masks for msic gpio input GPIOxxxxCTLI registers */ -#define MSIC_GPIO_GLBYP_MASK BIT(5) -#define MSIC_GPIO_DBNC_MASK (BIT(4) | BIT(3)) -#define MSIC_GPIO_INTCNT_MASK (BIT(2) | BIT(1)) -#define MSIC_GPIO_DIN_MASK BIT(0) - -#define MSIC_NUM_GPIO 24 - -struct msic_gpio { - struct platform_device *pdev; - struct mutex buslock; - struct gpio_chip chip; - int irq; - unsigned irq_base; - unsigned long trig_change_mask; - unsigned trig_type; -}; - -/* - * MSIC has 24 gpios, 16 low voltage (1.2-1.8v) and 8 high voltage (3v). - * Both the high and low voltage gpios are divided in two banks. - * GPIOs are numbered with GPIO0LV0 as gpio_base in the following order: - * GPIO0LV0..GPIO0LV7: low voltage, bank 0, gpio_base - * GPIO1LV0..GPIO1LV7: low voltage, bank 1, gpio_base + 8 - * GPIO0HV0..GPIO0HV3: high voltage, bank 0, gpio_base + 16 - * GPIO1HV0..GPIO1HV3: high voltage, bank 1, gpio_base + 20 - */ - -static int msic_gpio_to_ireg(unsigned offset) -{ - if (offset >= MSIC_NUM_GPIO) - return -EINVAL; - - if (offset < 8) - return INTEL_MSIC_GPIO0LV0CTLI - offset; - if (offset < 16) - return INTEL_MSIC_GPIO1LV0CTLI - offset + 8; - if (offset < 20) - return INTEL_MSIC_GPIO0HV0CTLI - offset + 16; - - return INTEL_MSIC_GPIO1HV0CTLI - offset + 20; -} - -static int msic_gpio_to_oreg(unsigned offset) -{ - if (offset >= MSIC_NUM_GPIO) - return -EINVAL; - - if (offset < 8) - return INTEL_MSIC_GPIO0LV0CTLO - offset; - if (offset < 16) - return INTEL_MSIC_GPIO1LV0CTLO - offset + 8; - if (offset < 20) - return INTEL_MSIC_GPIO0HV0CTLO - offset + 16; - - return INTEL_MSIC_GPIO1HV0CTLO - offset + 20; -} - -static int msic_gpio_direction_input(struct gpio_chip *chip, unsigned offset) -{ - int reg; - - reg = msic_gpio_to_oreg(offset); - if (reg < 0) - return reg; - - return intel_msic_reg_update(reg, MSIC_GPIO_DIR_IN, MSIC_GPIO_DIR_MASK); -} - -static int msic_gpio_direction_output(struct gpio_chip *chip, - unsigned offset, int value) -{ - int reg; - unsigned mask; - - value = (!!value) | MSIC_GPIO_DIR_OUT; - mask = MSIC_GPIO_DIR_MASK | MSIC_GPIO_DOUT_MASK; - - reg = msic_gpio_to_oreg(offset); - if (reg < 0) - return reg; - - return intel_msic_reg_update(reg, value, mask); -} - -static int msic_gpio_get(struct gpio_chip *chip, unsigned offset) -{ - u8 r; - int ret; - int reg; - - reg = msic_gpio_to_ireg(offset); - if (reg < 0) - return reg; - - ret = intel_msic_reg_read(reg, &r); - if (ret < 0) - return ret; - - return !!(r & MSIC_GPIO_DIN_MASK); -} - -static void msic_gpio_set(struct gpio_chip *chip, unsigned offset, int value) -{ - int reg; - - reg = msic_gpio_to_oreg(offset); - if (reg < 0) - return; - - intel_msic_reg_update(reg, !!value , MSIC_GPIO_DOUT_MASK); -} - -/* - * This is called from genirq with mg->buslock locked and - * irq_desc->lock held. We can not access the scu bus here, so we - * store the change and update in the bus_sync_unlock() function below - */ -static int msic_irq_type(struct irq_data *data, unsigned type) -{ - struct msic_gpio *mg = irq_data_get_irq_chip_data(data); - u32 gpio = data->irq - mg->irq_base; - - if (gpio >= mg->chip.ngpio) - return -EINVAL; - - /* mark for which gpio the trigger changed, protected by buslock */ - mg->trig_change_mask |= (1 << gpio); - mg->trig_type = type; - - return 0; -} - -static int msic_gpio_to_irq(struct gpio_chip *chip, unsigned offset) -{ - struct msic_gpio *mg = gpiochip_get_data(chip); - return mg->irq_base + offset; -} - -static void msic_bus_lock(struct irq_data *data) -{ - struct msic_gpio *mg = irq_data_get_irq_chip_data(data); - mutex_lock(&mg->buslock); -} - -static void msic_bus_sync_unlock(struct irq_data *data) -{ - struct msic_gpio *mg = irq_data_get_irq_chip_data(data); - int offset; - int reg; - u8 trig = 0; - - /* We can only get one change at a time as the buslock covers the - entire transaction. The irq_desc->lock is dropped before we are - called but that is fine */ - if (mg->trig_change_mask) { - offset = __ffs(mg->trig_change_mask); - - reg = msic_gpio_to_ireg(offset); - if (reg < 0) - goto out; - - if (mg->trig_type & IRQ_TYPE_EDGE_RISING) - trig |= MSIC_GPIO_TRIG_RISE; - if (mg->trig_type & IRQ_TYPE_EDGE_FALLING) - trig |= MSIC_GPIO_TRIG_FALL; - - intel_msic_reg_update(reg, trig, MSIC_GPIO_INTCNT_MASK); - mg->trig_change_mask = 0; - } -out: - mutex_unlock(&mg->buslock); -} - -/* Firmware does all the masking and unmasking for us, no masking here. */ -static void msic_irq_unmask(struct irq_data *data) { } - -static void msic_irq_mask(struct irq_data *data) { } - -static struct irq_chip msic_irqchip = { - .name = "MSIC-GPIO", - .irq_mask = msic_irq_mask, - .irq_unmask = msic_irq_unmask, - .irq_set_type = msic_irq_type, - .irq_bus_lock = msic_bus_lock, - .irq_bus_sync_unlock = msic_bus_sync_unlock, -}; - -static void msic_gpio_irq_handler(struct irq_desc *desc) -{ - struct irq_data *data = irq_desc_get_irq_data(desc); - struct msic_gpio *mg = irq_data_get_irq_handler_data(data); - struct irq_chip *chip = irq_data_get_irq_chip(data); - struct intel_msic *msic = pdev_to_intel_msic(mg->pdev); - unsigned long pending; - int i; - int bitnr; - u8 pin; - - for (i = 0; i < (mg->chip.ngpio / BITS_PER_BYTE); i++) { - intel_msic_irq_read(msic, INTEL_MSIC_GPIO0LVIRQ + i, &pin); - pending = pin; - - for_each_set_bit(bitnr, &pending, BITS_PER_BYTE) - generic_handle_irq(mg->irq_base + i * BITS_PER_BYTE + bitnr); - } - chip->irq_eoi(data); -} - -static int platform_msic_gpio_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct intel_msic_gpio_pdata *pdata = dev_get_platdata(dev); - struct msic_gpio *mg; - int irq = platform_get_irq(pdev, 0); - int retval; - int i; - - if (irq < 0) { - dev_err(dev, "no IRQ line: %d\n", irq); - return irq; - } - - if (!pdata || !pdata->gpio_base) { - dev_err(dev, "incorrect or missing platform data\n"); - return -EINVAL; - } - - mg = kzalloc(sizeof(*mg), GFP_KERNEL); - if (!mg) - return -ENOMEM; - - dev_set_drvdata(dev, mg); - - mg->pdev = pdev; - mg->irq = irq; - mg->irq_base = pdata->gpio_base + MSIC_GPIO_IRQ_OFFSET; - mg->chip.label = "msic_gpio"; - mg->chip.direction_input = msic_gpio_direction_input; - mg->chip.direction_output = msic_gpio_direction_output; - mg->chip.get = msic_gpio_get; - mg->chip.set = msic_gpio_set; - mg->chip.to_irq = msic_gpio_to_irq; - mg->chip.base = pdata->gpio_base; - mg->chip.ngpio = MSIC_NUM_GPIO; - mg->chip.can_sleep = true; - mg->chip.parent = dev; - - mutex_init(&mg->buslock); - - retval = gpiochip_add_data(&mg->chip, mg); - if (retval) { - dev_err(dev, "Adding MSIC gpio chip failed\n"); - goto err; - } - - for (i = 0; i < mg->chip.ngpio; i++) { - irq_set_chip_data(i + mg->irq_base, mg); - irq_set_chip_and_handler(i + mg->irq_base, - &msic_irqchip, - handle_simple_irq); - } - irq_set_chained_handler_and_data(mg->irq, msic_gpio_irq_handler, mg); - - return 0; -err: - kfree(mg); - return retval; -} - -static struct platform_driver platform_msic_gpio_driver = { - .driver = { - .name = "msic_gpio", - }, - .probe = platform_msic_gpio_probe, -}; - -static int __init platform_msic_gpio_init(void) -{ - return platform_driver_register(&platform_msic_gpio_driver); -} -subsys_initcall(platform_msic_gpio_init); diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c index a912a8fed197..8f429d9f3661 100644 --- a/drivers/gpio/gpio-mvebu.c +++ b/drivers/gpio/gpio-mvebu.c @@ -70,7 +70,12 @@ */ #define PWM_BLINK_ON_DURATION_OFF 0x0 #define PWM_BLINK_OFF_DURATION_OFF 0x4 +#define PWM_BLINK_COUNTER_B_OFF 0x8 +/* Armada 8k variant gpios register offsets */ +#define AP80X_GPIO0_OFF_A8K 0x1040 +#define CP11X_GPIO0_OFF_A8K 0x100 +#define CP11X_GPIO1_OFF_A8K 0x140 /* The MV78200 has per-CPU registers for edge mask and level mask */ #define GPIO_EDGE_MASK_MV78200_OFF(cpu) ((cpu) ? 0x30 : 0x18) @@ -93,6 +98,7 @@ struct mvebu_pwm { struct regmap *regs; + u32 offset; unsigned long clk_rate; struct gpio_desc *gpiod; struct pwm_chip chip; @@ -283,12 +289,12 @@ mvebu_gpio_write_level_mask(struct mvebu_gpio_chip *mvchip, u32 val) */ static unsigned int mvebu_pwmreg_blink_on_duration(struct mvebu_pwm *mvpwm) { - return PWM_BLINK_ON_DURATION_OFF; + return mvpwm->offset + PWM_BLINK_ON_DURATION_OFF; } static unsigned int mvebu_pwmreg_blink_off_duration(struct mvebu_pwm *mvpwm) { - return PWM_BLINK_OFF_DURATION_OFF; + return mvpwm->offset + PWM_BLINK_OFF_DURATION_OFF; } /* @@ -667,26 +673,21 @@ static void mvebu_pwm_get_state(struct pwm_chip *chip, spin_lock_irqsave(&mvpwm->lock, flags); regmap_read(mvpwm->regs, mvebu_pwmreg_blink_on_duration(mvpwm), &u); - val = (unsigned long long) u * NSEC_PER_SEC; - do_div(val, mvpwm->clk_rate); - if (val > UINT_MAX) - state->duty_cycle = UINT_MAX; - else if (val) - state->duty_cycle = val; + /* Hardware treats zero as 2^32. See mvebu_pwm_apply(). */ + if (u > 0) + val = u; else - state->duty_cycle = 1; + val = UINT_MAX + 1ULL; + state->duty_cycle = DIV_ROUND_UP_ULL(val * NSEC_PER_SEC, + mvpwm->clk_rate); - val = (unsigned long long) u; /* on duration */ regmap_read(mvpwm->regs, mvebu_pwmreg_blink_off_duration(mvpwm), &u); - val += (unsigned long long) u; /* period = on + off duration */ - val *= NSEC_PER_SEC; - do_div(val, mvpwm->clk_rate); - if (val > UINT_MAX) - state->period = UINT_MAX; - else if (val) - state->period = val; + /* period = on + off duration */ + if (u > 0) + val += u; else - state->period = 1; + val += UINT_MAX + 1ULL; + state->period = DIV_ROUND_UP_ULL(val * NSEC_PER_SEC, mvpwm->clk_rate); regmap_read(mvchip->regs, GPIO_BLINK_EN_OFF + mvchip->offset, &u); if (u) @@ -708,19 +709,27 @@ static int mvebu_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, val = (unsigned long long) mvpwm->clk_rate * state->duty_cycle; do_div(val, NSEC_PER_SEC); - if (val > UINT_MAX) + if (val > UINT_MAX + 1ULL) return -EINVAL; - if (val) + /* + * Zero on/off values don't work as expected. Experimentation shows + * that zero value is treated as 2^32. This behavior is not documented. + */ + if (val == UINT_MAX + 1ULL) + on = 0; + else if (val) on = val; else on = 1; - val = (unsigned long long) mvpwm->clk_rate * - (state->period - state->duty_cycle); + val = (unsigned long long) mvpwm->clk_rate * state->period; do_div(val, NSEC_PER_SEC); - if (val > UINT_MAX) + val -= on; + if (val > UINT_MAX + 1ULL) return -EINVAL; - if (val) + if (val == UINT_MAX + 1ULL) + off = 0; + else if (val) off = val; else off = 1; @@ -778,51 +787,80 @@ static int mvebu_pwm_probe(struct platform_device *pdev, struct device *dev = &pdev->dev; struct mvebu_pwm *mvpwm; void __iomem *base; + u32 offset; u32 set; - if (!of_device_is_compatible(mvchip->chip.of_node, - "marvell,armada-370-gpio")) - return 0; - - /* - * There are only two sets of PWM configuration registers for - * all the GPIO lines on those SoCs which this driver reserves - * for the first two GPIO chips. So if the resource is missing - * we can't treat it as an error. - */ - if (!platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwm")) + if (of_device_is_compatible(mvchip->chip.of_node, + "marvell,armada-370-gpio")) { + /* + * There are only two sets of PWM configuration registers for + * all the GPIO lines on those SoCs which this driver reserves + * for the first two GPIO chips. So if the resource is missing + * we can't treat it as an error. + */ + if (!platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwm")) + return 0; + offset = 0; + } else if (mvchip->soc_variant == MVEBU_GPIO_SOC_VARIANT_A8K) { + int ret = of_property_read_u32(dev->of_node, + "marvell,pwm-offset", &offset); + if (ret < 0) + return 0; + } else { return 0; + } if (IS_ERR(mvchip->clk)) return PTR_ERR(mvchip->clk); - /* - * Use set A for lines of GPIO chip with id 0, B for GPIO chip - * with id 1. Don't allow further GPIO chips to be used for PWM. - */ - if (id == 0) - set = 0; - else if (id == 1) - set = U32_MAX; - else - return -EINVAL; - regmap_write(mvchip->regs, - GPIO_BLINK_CNT_SELECT_OFF + mvchip->offset, set); - mvpwm = devm_kzalloc(dev, sizeof(struct mvebu_pwm), GFP_KERNEL); if (!mvpwm) return -ENOMEM; mvchip->mvpwm = mvpwm; mvpwm->mvchip = mvchip; + mvpwm->offset = offset; - base = devm_platform_ioremap_resource_byname(pdev, "pwm"); - if (IS_ERR(base)) - return PTR_ERR(base); + if (mvchip->soc_variant == MVEBU_GPIO_SOC_VARIANT_A8K) { + mvpwm->regs = mvchip->regs; - mvpwm->regs = devm_regmap_init_mmio(&pdev->dev, base, - &mvebu_gpio_regmap_config); - if (IS_ERR(mvpwm->regs)) - return PTR_ERR(mvpwm->regs); + switch (mvchip->offset) { + case AP80X_GPIO0_OFF_A8K: + case CP11X_GPIO0_OFF_A8K: + /* Blink counter A */ + set = 0; + break; + case CP11X_GPIO1_OFF_A8K: + /* Blink counter B */ + set = U32_MAX; + mvpwm->offset += PWM_BLINK_COUNTER_B_OFF; + break; + default: + return -EINVAL; + } + } else { + base = devm_platform_ioremap_resource_byname(pdev, "pwm"); + if (IS_ERR(base)) + return PTR_ERR(base); + + mvpwm->regs = devm_regmap_init_mmio(&pdev->dev, base, + &mvebu_gpio_regmap_config); + if (IS_ERR(mvpwm->regs)) + return PTR_ERR(mvpwm->regs); + + /* + * Use set A for lines of GPIO chip with id 0, B for GPIO chip + * with id 1. Don't allow further GPIO chips to be used for PWM. + */ + if (id == 0) + set = 0; + else if (id == 1) + set = U32_MAX; + else + return -EINVAL; + } + + regmap_write(mvchip->regs, + GPIO_BLINK_CNT_SELECT_OFF + mvchip->offset, set); mvpwm->clk_rate = clk_get_rate(mvchip->clk); if (!mvpwm->clk_rate) { diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index 825b362eb4b7..5ea09fd01544 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -73,6 +73,7 @@ static const struct i2c_device_id pca953x_id[] = { { "pca6416", 16 | PCA953X_TYPE | PCA_INT, }, { "pca9505", 40 | PCA953X_TYPE | PCA_INT, }, + { "pca9506", 40 | PCA953X_TYPE | PCA_INT, }, { "pca9534", 8 | PCA953X_TYPE | PCA_INT, }, { "pca9535", 16 | PCA953X_TYPE | PCA_INT, }, { "pca9536", 4 | PCA953X_TYPE, }, @@ -1236,6 +1237,7 @@ static int pca953x_resume(struct device *dev) static const struct of_device_id pca953x_dt_ids[] = { { .compatible = "nxp,pca6416", .data = OF_953X(16, PCA_INT), }, { .compatible = "nxp,pca9505", .data = OF_953X(40, PCA_INT), }, + { .compatible = "nxp,pca9506", .data = OF_953X(40, PCA_INT), }, { .compatible = "nxp,pca9534", .data = OF_953X( 8, PCA_INT), }, { .compatible = "nxp,pca9535", .data = OF_953X(16, PCA_INT), }, { .compatible = "nxp,pca9536", .data = OF_953X( 4, 0), }, diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c index a2a8d155c75e..b7568ee33696 100644 --- a/drivers/gpio/gpio-pcf857x.c +++ b/drivers/gpio/gpio-pcf857x.c @@ -332,7 +332,7 @@ static int pcf857x_probe(struct i2c_client *client, * reset state. Otherwise it flags pins to be driven low. */ gpio->out = ~n_latch; - gpio->status = gpio->out; + gpio->status = gpio->read(gpio->client); /* Enable irqchip if we have an interrupt */ if (client->irq) { diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c index 0b572dbc4a36..e7092d5fe700 100644 --- a/drivers/gpio/gpio-rcar.c +++ b/drivers/gpio/gpio-rcar.c @@ -35,6 +35,8 @@ struct gpio_rcar_bank_info { struct gpio_rcar_info { bool has_outdtsel; bool has_both_edge_trigger; + bool has_always_in; + bool has_inen; }; struct gpio_rcar_priv { @@ -62,6 +64,7 @@ struct gpio_rcar_priv { #define FILONOFF 0x28 /* Chattering Prevention On/Off Register */ #define OUTDTSEL 0x40 /* Output Data Select Register */ #define BOTHEDGE 0x4c /* One Edge/Both Edge Select Register */ +#define INEN 0x50 /* General Input Enable Register */ #define RCAR_MAX_GPIO_PER_BANK 32 @@ -302,9 +305,11 @@ static int gpio_rcar_get(struct gpio_chip *chip, unsigned offset) struct gpio_rcar_priv *p = gpiochip_get_data(chip); u32 bit = BIT(offset); - /* testing on r8a7790 shows that INDT does not show correct pin state - * when configured as output, so use OUTDT in case of output pins */ - if (gpio_rcar_read(p, INOUTSEL) & bit) + /* + * Before R-Car Gen3, INDT does not show correct pin state when + * configured as output, so use OUTDT in case of output pins + */ + if (!p->info.has_always_in && (gpio_rcar_read(p, INOUTSEL) & bit)) return !!(gpio_rcar_read(p, OUTDT) & bit); else return !!(gpio_rcar_read(p, INDT) & bit); @@ -324,6 +329,11 @@ static int gpio_rcar_get_multiple(struct gpio_chip *chip, unsigned long *mask, if (!bankmask) return 0; + if (p->info.has_always_in) { + bits[0] = gpio_rcar_read(p, INDT) & bankmask; + return 0; + } + spin_lock_irqsave(&p->lock, flags); outputs = gpio_rcar_read(p, INOUTSEL); m = outputs & bankmask; @@ -383,41 +393,35 @@ static int gpio_rcar_direction_output(struct gpio_chip *chip, unsigned offset, static const struct gpio_rcar_info gpio_rcar_info_gen1 = { .has_outdtsel = false, .has_both_edge_trigger = false, + .has_always_in = false, + .has_inen = false, }; static const struct gpio_rcar_info gpio_rcar_info_gen2 = { .has_outdtsel = true, .has_both_edge_trigger = true, + .has_always_in = false, + .has_inen = false, +}; + +static const struct gpio_rcar_info gpio_rcar_info_gen3 = { + .has_outdtsel = true, + .has_both_edge_trigger = true, + .has_always_in = true, + .has_inen = false, +}; + +static const struct gpio_rcar_info gpio_rcar_info_v3u = { + .has_outdtsel = true, + .has_both_edge_trigger = true, + .has_always_in = true, + .has_inen = true, }; static const struct of_device_id gpio_rcar_of_table[] = { { - .compatible = "renesas,gpio-r8a7743", - /* RZ/G1 GPIO is identical to R-Car Gen2. */ - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7790", - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7791", - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7792", - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7793", - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7794", - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7795", - /* Gen3 GPIO is identical to Gen2. */ - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7796", - /* Gen3 GPIO is identical to Gen2. */ - .data = &gpio_rcar_info_gen2, + .compatible = "renesas,gpio-r8a779a0", + .data = &gpio_rcar_info_v3u, }, { .compatible = "renesas,rcar-gen1-gpio", .data = &gpio_rcar_info_gen1, @@ -426,8 +430,7 @@ static const struct of_device_id gpio_rcar_of_table[] = { .data = &gpio_rcar_info_gen2, }, { .compatible = "renesas,rcar-gen3-gpio", - /* Gen3 GPIO is identical to Gen2. */ - .data = &gpio_rcar_info_gen2, + .data = &gpio_rcar_info_gen3, }, { .compatible = "renesas,gpio-rcar", .data = &gpio_rcar_info_gen1, @@ -460,6 +463,17 @@ static int gpio_rcar_parse_dt(struct gpio_rcar_priv *p, unsigned int *npins) return 0; } +static void gpio_rcar_enable_inputs(struct gpio_rcar_priv *p) +{ + u32 mask = GENMASK(p->gpio_chip.ngpio - 1, 0); + + /* Select "Input Enable" in INEN */ + if (p->gpio_chip.valid_mask) + mask &= p->gpio_chip.valid_mask[0]; + if (mask) + gpio_rcar_write(p, INEN, gpio_rcar_read(p, INEN) | mask); +} + static int gpio_rcar_probe(struct platform_device *pdev) { struct gpio_rcar_priv *p; @@ -549,6 +563,12 @@ static int gpio_rcar_probe(struct platform_device *pdev) goto err1; } + if (p->info.has_inen) { + pm_runtime_get_sync(p->dev); + gpio_rcar_enable_inputs(p); + pm_runtime_put(p->dev); + } + dev_info(dev, "driving %d GPIOs\n", npins); return 0; @@ -624,6 +644,9 @@ static int gpio_rcar_resume(struct device *dev) } } + if (p->info.has_inen) + gpio_rcar_enable_inputs(p); + return 0; } #endif /* CONFIG_PM_SLEEP*/ diff --git a/drivers/gpio/gpio-sl28cpld.c b/drivers/gpio/gpio-sl28cpld.c index 889b8f5622c2..52404736ac86 100644 --- a/drivers/gpio/gpio-sl28cpld.c +++ b/drivers/gpio/gpio-sl28cpld.c @@ -65,13 +65,13 @@ static int sl28cpld_gpio_irq_init(struct platform_device *pdev, if (!irq_chip) return -ENOMEM; - irq_chip->name = "sl28cpld-gpio-irq", + irq_chip->name = "sl28cpld-gpio-irq"; irq_chip->irqs = sl28cpld_gpio_irqs; irq_chip->num_irqs = ARRAY_SIZE(sl28cpld_gpio_irqs); irq_chip->num_regs = 1; irq_chip->status_base = base + GPIO_REG_IP; irq_chip->mask_base = base + GPIO_REG_IE; - irq_chip->mask_invert = true, + irq_chip->mask_invert = true; irq_chip->ack_base = base + GPIO_REG_IP; ret = devm_regmap_add_irq_chip_fwnode(dev, dev_fwnode(dev), diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c index e19ebff6018c..0025f613d9b3 100644 --- a/drivers/gpio/gpio-tegra.c +++ b/drivers/gpio/gpio-tegra.c @@ -60,7 +60,6 @@ struct tegra_gpio_info; struct tegra_gpio_bank { unsigned int bank; - unsigned int irq; /* * IRQ-core code uses raw locking, and thus, nested locking also @@ -81,7 +80,6 @@ struct tegra_gpio_bank { u32 dbc_enb[4]; #endif u32 dbc_cnt[4]; - struct tegra_gpio_info *tgi; }; struct tegra_gpio_soc_config { @@ -93,12 +91,12 @@ struct tegra_gpio_soc_config { struct tegra_gpio_info { struct device *dev; void __iomem *regs; - struct irq_domain *irq_domain; struct tegra_gpio_bank *bank_info; const struct tegra_gpio_soc_config *soc; struct gpio_chip gc; struct irq_chip ic; u32 bank_count; + unsigned int *irqs; }; static inline void tegra_gpio_writel(struct tegra_gpio_info *tgi, @@ -274,17 +272,10 @@ static int tegra_gpio_set_config(struct gpio_chip *chip, unsigned int offset, return tegra_gpio_set_debounce(chip, offset, debounce); } -static int tegra_gpio_to_irq(struct gpio_chip *chip, unsigned int offset) -{ - struct tegra_gpio_info *tgi = gpiochip_get_data(chip); - - return irq_find_mapping(tgi->irq_domain, offset); -} - static void tegra_gpio_irq_ack(struct irq_data *d) { - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); - struct tegra_gpio_info *tgi = bank->tgi; + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); unsigned int gpio = d->hwirq; tegra_gpio_writel(tgi, 1 << GPIO_BIT(gpio), GPIO_INT_CLR(tgi, gpio)); @@ -292,8 +283,8 @@ static void tegra_gpio_irq_ack(struct irq_data *d) static void tegra_gpio_irq_mask(struct irq_data *d) { - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); - struct tegra_gpio_info *tgi = bank->tgi; + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); unsigned int gpio = d->hwirq; tegra_gpio_mask_write(tgi, GPIO_MSK_INT_ENB(tgi, gpio), gpio, 0); @@ -301,8 +292,8 @@ static void tegra_gpio_irq_mask(struct irq_data *d) static void tegra_gpio_irq_unmask(struct irq_data *d) { - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); - struct tegra_gpio_info *tgi = bank->tgi; + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); unsigned int gpio = d->hwirq; tegra_gpio_mask_write(tgi, GPIO_MSK_INT_ENB(tgi, gpio), gpio, 1); @@ -311,11 +302,14 @@ static void tegra_gpio_irq_unmask(struct irq_data *d) static int tegra_gpio_irq_set_type(struct irq_data *d, unsigned int type) { unsigned int gpio = d->hwirq, port = GPIO_PORT(gpio), lvl_type; - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); - struct tegra_gpio_info *tgi = bank->tgi; + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); + struct tegra_gpio_bank *bank; unsigned long flags; - u32 val; int ret; + u32 val; + + bank = &tgi->bank_info[GPIO_BANK(d->hwirq)]; switch (type & IRQ_TYPE_SENSE_MASK) { case IRQ_TYPE_EDGE_RISING: @@ -367,13 +361,16 @@ static int tegra_gpio_irq_set_type(struct irq_data *d, unsigned int type) else if (type & (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING)) irq_set_handler_locked(d, handle_edge_irq); - return 0; + if (d->parent_data) + ret = irq_chip_set_type_parent(d, type); + + return ret; } static void tegra_gpio_irq_shutdown(struct irq_data *d) { - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); - struct tegra_gpio_info *tgi = bank->tgi; + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); unsigned int gpio = d->hwirq; tegra_gpio_irq_mask(d); @@ -382,13 +379,25 @@ static void tegra_gpio_irq_shutdown(struct irq_data *d) static void tegra_gpio_irq_handler(struct irq_desc *desc) { - unsigned int port, pin, gpio; - bool unmasked = false; - u32 lvl; - unsigned long sta; + struct tegra_gpio_info *tgi = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); - struct tegra_gpio_bank *bank = irq_desc_get_handler_data(desc); - struct tegra_gpio_info *tgi = bank->tgi; + struct irq_domain *domain = tgi->gc.irq.domain; + unsigned int irq = irq_desc_get_irq(desc); + struct tegra_gpio_bank *bank = NULL; + unsigned int port, pin, gpio, i; + bool unmasked = false; + unsigned long sta; + u32 lvl; + + for (i = 0; i < tgi->bank_count; i++) { + if (tgi->irqs[i] == irq) { + bank = &tgi->bank_info[i]; + break; + } + } + + if (WARN_ON(bank == NULL)) + return; chained_irq_enter(chip, desc); @@ -411,14 +420,47 @@ static void tegra_gpio_irq_handler(struct irq_desc *desc) chained_irq_exit(chip, desc); } - generic_handle_irq(irq_find_mapping(tgi->irq_domain, - gpio + pin)); + irq = irq_find_mapping(domain, gpio + pin); + if (WARN_ON(irq == 0)) + continue; + + generic_handle_irq(irq); } } if (!unmasked) chained_irq_exit(chip, desc); +} +static int tegra_gpio_child_to_parent_hwirq(struct gpio_chip *chip, + unsigned int hwirq, + unsigned int type, + unsigned int *parent_hwirq, + unsigned int *parent_type) +{ + *parent_hwirq = chip->irq.child_offset_to_irq(chip, hwirq); + *parent_type = type; + + return 0; +} + +static void *tegra_gpio_populate_parent_fwspec(struct gpio_chip *chip, + unsigned int parent_hwirq, + unsigned int parent_type) +{ + struct irq_fwspec *fwspec; + + fwspec = kmalloc(sizeof(*fwspec), GFP_KERNEL); + if (!fwspec) + return NULL; + + fwspec->fwnode = chip->irq.parent_domain->fwnode; + fwspec->param_count = 3; + fwspec->param[0] = 0; + fwspec->param[1] = parent_hwirq; + fwspec->param[2] = parent_type; + + return fwspec; } #ifdef CONFIG_PM_SLEEP @@ -497,19 +539,31 @@ static int tegra_gpio_suspend(struct device *dev) static int tegra_gpio_irq_set_wake(struct irq_data *d, unsigned int enable) { - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); + struct tegra_gpio_bank *bank; unsigned int gpio = d->hwirq; u32 port, bit, mask; int err; - err = irq_set_irq_wake(bank->irq, enable); - if (err) - return err; + bank = &tgi->bank_info[GPIO_BANK(d->hwirq)]; port = GPIO_PORT(gpio); bit = GPIO_BIT(gpio); mask = BIT(bit); + err = irq_set_irq_wake(tgi->irqs[bank->bank], enable); + if (err) + return err; + + if (d->parent_data) { + err = irq_chip_set_wake_parent(d, enable); + if (err) { + irq_set_irq_wake(tgi->irqs[bank->bank], !enable); + return err; + } + } + if (enable) bank->wake_enb[port] |= mask; else @@ -519,6 +573,35 @@ static int tegra_gpio_irq_set_wake(struct irq_data *d, unsigned int enable) } #endif +static int tegra_gpio_irq_set_affinity(struct irq_data *data, + const struct cpumask *dest, + bool force) +{ + if (data->parent_data) + return irq_chip_set_affinity_parent(data, dest, force); + + return -EINVAL; +} + +static int tegra_gpio_irq_request_resources(struct irq_data *d) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); + + tegra_gpio_enable(tgi, d->hwirq); + + return gpiochip_reqres_irq(chip, d->hwirq); +} + +static void tegra_gpio_irq_release_resources(struct irq_data *d) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); + + gpiochip_relres_irq(chip, d->hwirq); + tegra_gpio_enable(tgi, d->hwirq); +} + #ifdef CONFIG_DEBUG_FS #include @@ -526,7 +609,7 @@ static int tegra_gpio_irq_set_wake(struct irq_data *d, unsigned int enable) static int tegra_dbg_gpio_show(struct seq_file *s, void *unused) { - struct tegra_gpio_info *tgi = s->private; + struct tegra_gpio_info *tgi = dev_get_drvdata(s->private); unsigned int i, j; for (i = 0; i < tgi->bank_count; i++) { @@ -548,12 +631,10 @@ static int tegra_dbg_gpio_show(struct seq_file *s, void *unused) return 0; } -DEFINE_SHOW_ATTRIBUTE(tegra_dbg_gpio); - static void tegra_gpio_debuginit(struct tegra_gpio_info *tgi) { - debugfs_create_file("tegra_gpio", 0444, NULL, tgi, - &tegra_dbg_gpio_fops); + debugfs_create_devm_seqfile(tgi->dev, "tegra_gpio", NULL, + tegra_dbg_gpio_show); } #else @@ -568,14 +649,18 @@ static const struct dev_pm_ops tegra_gpio_pm_ops = { SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(tegra_gpio_suspend, tegra_gpio_resume) }; -static struct lock_class_key gpio_lock_class; -static struct lock_class_key gpio_request_class; +static const struct of_device_id tegra_pmc_of_match[] = { + { .compatible = "nvidia,tegra210-pmc", }, + { /* sentinel */ }, +}; static int tegra_gpio_probe(struct platform_device *pdev) { - struct tegra_gpio_info *tgi; struct tegra_gpio_bank *bank; - unsigned int gpio, i, j; + struct tegra_gpio_info *tgi; + struct gpio_irq_chip *irq; + struct device_node *np; + unsigned int i, j; int ret; tgi = devm_kzalloc(&pdev->dev, sizeof(*tgi), GFP_KERNEL); @@ -604,7 +689,6 @@ static int tegra_gpio_probe(struct platform_device *pdev) tgi->gc.direction_output = tegra_gpio_direction_output; tgi->gc.set = tegra_gpio_set; tgi->gc.get_direction = tegra_gpio_get_direction; - tgi->gc.to_irq = tegra_gpio_to_irq; tgi->gc.base = 0; tgi->gc.ngpio = tgi->bank_count * 32; tgi->gc.parent = &pdev->dev; @@ -619,6 +703,8 @@ static int tegra_gpio_probe(struct platform_device *pdev) #ifdef CONFIG_PM_SLEEP tgi->ic.irq_set_wake = tegra_gpio_irq_set_wake; #endif + tgi->ic.irq_request_resources = tegra_gpio_irq_request_resources; + tgi->ic.irq_release_resources = tegra_gpio_irq_release_resources; platform_set_drvdata(pdev, tgi); @@ -630,11 +716,10 @@ static int tegra_gpio_probe(struct platform_device *pdev) if (!tgi->bank_info) return -ENOMEM; - tgi->irq_domain = irq_domain_add_linear(pdev->dev.of_node, - tgi->gc.ngpio, - &irq_domain_simple_ops, NULL); - if (!tgi->irq_domain) - return -ENODEV; + tgi->irqs = devm_kcalloc(&pdev->dev, tgi->bank_count, + sizeof(*tgi->irqs), GFP_KERNEL); + if (!tgi->irqs) + return -ENOMEM; for (i = 0; i < tgi->bank_count; i++) { ret = platform_get_irq(pdev, i); @@ -643,8 +728,36 @@ static int tegra_gpio_probe(struct platform_device *pdev) bank = &tgi->bank_info[i]; bank->bank = i; - bank->irq = ret; - bank->tgi = tgi; + + tgi->irqs[i] = ret; + + for (j = 0; j < 4; j++) { + raw_spin_lock_init(&bank->lvl_lock[j]); + spin_lock_init(&bank->dbc_lock[j]); + } + } + + irq = &tgi->gc.irq; + irq->chip = &tgi->ic; + irq->fwnode = of_node_to_fwnode(pdev->dev.of_node); + irq->child_to_parent_hwirq = tegra_gpio_child_to_parent_hwirq; + irq->populate_parent_alloc_arg = tegra_gpio_populate_parent_fwspec; + irq->handler = handle_simple_irq; + irq->default_type = IRQ_TYPE_NONE; + irq->parent_handler = tegra_gpio_irq_handler; + irq->parent_handler_data = tgi; + irq->num_parents = tgi->bank_count; + irq->parents = tgi->irqs; + + np = of_find_matching_node(NULL, tegra_pmc_of_match); + if (np) { + irq->parent_domain = irq_find_host(np); + of_node_put(np); + + if (!irq->parent_domain) + return -EPROBE_DEFER; + + tgi->ic.irq_set_affinity = tegra_gpio_irq_set_affinity; } tgi->regs = devm_platform_ioremap_resource(pdev, 0); @@ -660,33 +773,8 @@ static int tegra_gpio_probe(struct platform_device *pdev) } ret = devm_gpiochip_add_data(&pdev->dev, &tgi->gc, tgi); - if (ret < 0) { - irq_domain_remove(tgi->irq_domain); + if (ret < 0) return ret; - } - - for (gpio = 0; gpio < tgi->gc.ngpio; gpio++) { - int irq = irq_create_mapping(tgi->irq_domain, gpio); - /* No validity check; all Tegra GPIOs are valid IRQs */ - - bank = &tgi->bank_info[GPIO_BANK(gpio)]; - - irq_set_chip_data(irq, bank); - irq_set_lockdep_class(irq, &gpio_lock_class, &gpio_request_class); - irq_set_chip_and_handler(irq, &tgi->ic, handle_simple_irq); - } - - for (i = 0; i < tgi->bank_count; i++) { - bank = &tgi->bank_info[i]; - - irq_set_chained_handler_and_data(bank->irq, - tegra_gpio_irq_handler, bank); - - for (j = 0; j < 4; j++) { - raw_spin_lock_init(&bank->lvl_lock[j]); - spin_lock_init(&bank->dbc_lock[j]); - } - } tegra_gpio_debuginit(tgi); @@ -715,18 +803,21 @@ static const struct of_device_id tegra_gpio_of_match[] = { { .compatible = "nvidia,tegra20-gpio", .data = &tegra20_gpio_config }, { }, }; +MODULE_DEVICE_TABLE(of, tegra_gpio_of_match); static struct platform_driver tegra_gpio_driver = { - .driver = { - .name = "tegra-gpio", - .pm = &tegra_gpio_pm_ops, + .driver = { + .name = "tegra-gpio", + .pm = &tegra_gpio_pm_ops, .of_match_table = tegra_gpio_of_match, }, - .probe = tegra_gpio_probe, + .probe = tegra_gpio_probe, }; +module_platform_driver(tegra_gpio_driver); -static int __init tegra_gpio_init(void) -{ - return platform_driver_register(&tegra_gpio_driver); -} -subsys_initcall(tegra_gpio_init); +MODULE_DESCRIPTION("NVIDIA Tegra GPIO controller driver"); +MODULE_AUTHOR("Laxman Dewangan "); +MODULE_AUTHOR("Stephen Warren "); +MODULE_AUTHOR("Thierry Reding "); +MODULE_AUTHOR("Erik Gilling "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpio/gpio-tegra186.c b/drivers/gpio/gpio-tegra186.c index 286e0b1f46e4..1bd9e44df718 100644 --- a/drivers/gpio/gpio-tegra186.c +++ b/drivers/gpio/gpio-tegra186.c @@ -657,7 +657,7 @@ static int tegra186_gpio_probe(struct platform_device *pdev) gpio->gpio.get_direction = tegra186_gpio_get_direction; gpio->gpio.direction_input = tegra186_gpio_direction_input; gpio->gpio.direction_output = tegra186_gpio_direction_output; - gpio->gpio.get = tegra186_gpio_get, + gpio->gpio.get = tegra186_gpio_get; gpio->gpio.set = tegra186_gpio_set; gpio->gpio.set_config = tegra186_gpio_set_config; gpio->gpio.add_pin_ranges = tegra186_gpio_add_pin_ranges; diff --git a/drivers/gpio/gpio-visconti.c b/drivers/gpio/gpio-visconti.c new file mode 100644 index 000000000000..0e3d19828eb1 --- /dev/null +++ b/drivers/gpio/gpio-visconti.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Toshiba Visconti GPIO Support + * + * (C) Copyright 2020 Toshiba Electronic Devices & Storage Corporation + * (C) Copyright 2020 TOSHIBA CORPORATION + * + * Nobuhiro Iwamatsu + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* register offset */ +#define GPIO_DIR 0x00 +#define GPIO_IDATA 0x08 +#define GPIO_ODATA 0x10 +#define GPIO_OSET 0x18 +#define GPIO_OCLR 0x20 +#define GPIO_INTMODE 0x30 + +#define BASE_HW_IRQ 24 + +struct visconti_gpio { + void __iomem *base; + spinlock_t lock; /* protect gpio register */ + struct gpio_chip gpio_chip; + struct irq_chip irq_chip; +}; + +static int visconti_gpio_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct visconti_gpio *priv = gpiochip_get_data(gc); + u32 offset = irqd_to_hwirq(d); + u32 bit = BIT(offset); + u32 intc_type = IRQ_TYPE_EDGE_RISING; + u32 intmode, odata; + int ret = 0; + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + + odata = readl(priv->base + GPIO_ODATA); + intmode = readl(priv->base + GPIO_INTMODE); + + switch (type) { + case IRQ_TYPE_EDGE_RISING: + odata &= ~bit; + intmode &= ~bit; + break; + case IRQ_TYPE_EDGE_FALLING: + odata |= bit; + intmode &= ~bit; + break; + case IRQ_TYPE_EDGE_BOTH: + intmode |= bit; + break; + case IRQ_TYPE_LEVEL_HIGH: + intc_type = IRQ_TYPE_LEVEL_HIGH; + odata &= ~bit; + intmode &= ~bit; + break; + case IRQ_TYPE_LEVEL_LOW: + intc_type = IRQ_TYPE_LEVEL_HIGH; + odata |= bit; + intmode &= ~bit; + break; + default: + ret = -EINVAL; + goto err; + } + + writel(odata, priv->base + GPIO_ODATA); + writel(intmode, priv->base + GPIO_INTMODE); + irq_set_irq_type(offset, intc_type); + + ret = irq_chip_set_type_parent(d, type); +err: + spin_unlock_irqrestore(&priv->lock, flags); + return ret; +} + +static int visconti_gpio_child_to_parent_hwirq(struct gpio_chip *gc, + unsigned int child, + unsigned int child_type, + unsigned int *parent, + unsigned int *parent_type) +{ + /* Interrupts 0..15 mapped to interrupts 24..39 on the GIC */ + if (child < 16) { + /* All these interrupts are level high in the CPU */ + *parent_type = IRQ_TYPE_LEVEL_HIGH; + *parent = child + BASE_HW_IRQ; + return 0; + } + return -EINVAL; +} + +static void *visconti_gpio_populate_parent_fwspec(struct gpio_chip *chip, + unsigned int parent_hwirq, + unsigned int parent_type) +{ + struct irq_fwspec *fwspec; + + fwspec = kmalloc(sizeof(*fwspec), GFP_KERNEL); + if (!fwspec) + return NULL; + + fwspec->fwnode = chip->irq.parent_domain->fwnode; + fwspec->param_count = 3; + fwspec->param[0] = 0; + fwspec->param[1] = parent_hwirq; + fwspec->param[2] = parent_type; + + return fwspec; +} + +static int visconti_gpio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct visconti_gpio *priv; + struct irq_chip *irq_chip; + struct gpio_irq_chip *girq; + struct irq_domain *parent; + struct device_node *irq_parent; + struct fwnode_handle *fwnode; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + spin_lock_init(&priv->lock); + + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + irq_parent = of_irq_find_parent(dev->of_node); + if (!irq_parent) { + dev_err(dev, "No IRQ parent node\n"); + return -ENODEV; + } + + parent = irq_find_host(irq_parent); + if (!parent) { + dev_err(dev, "No IRQ parent domain\n"); + return -ENODEV; + } + + fwnode = of_node_to_fwnode(irq_parent); + of_node_put(irq_parent); + + ret = bgpio_init(&priv->gpio_chip, dev, 4, + priv->base + GPIO_IDATA, + priv->base + GPIO_OSET, + priv->base + GPIO_OCLR, + priv->base + GPIO_DIR, + NULL, + 0); + if (ret) { + dev_err(dev, "unable to init generic GPIO\n"); + return ret; + } + + irq_chip = &priv->irq_chip; + irq_chip->name = dev_name(dev); + irq_chip->irq_mask = irq_chip_mask_parent; + irq_chip->irq_unmask = irq_chip_unmask_parent; + irq_chip->irq_eoi = irq_chip_eoi_parent; + irq_chip->irq_set_type = visconti_gpio_irq_set_type; + irq_chip->flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND; + + girq = &priv->gpio_chip.irq; + girq->chip = irq_chip; + girq->fwnode = fwnode; + girq->parent_domain = parent; + girq->child_to_parent_hwirq = visconti_gpio_child_to_parent_hwirq; + girq->populate_parent_alloc_arg = visconti_gpio_populate_parent_fwspec; + girq->default_type = IRQ_TYPE_NONE; + girq->handler = handle_level_irq; + + ret = devm_gpiochip_add_data(dev, &priv->gpio_chip, priv); + if (ret) { + dev_err(dev, "failed to add GPIO chip\n"); + return ret; + } + + platform_set_drvdata(pdev, priv); + + return ret; +} + +static const struct of_device_id visconti_gpio_of_match[] = { + { .compatible = "toshiba,gpio-tmpv7708", }, + { /* end of table */ } +}; +MODULE_DEVICE_TABLE(of, visconti_gpio_of_match); + +static struct platform_driver visconti_gpio_driver = { + .probe = visconti_gpio_probe, + .driver = { + .name = "visconti_gpio", + .of_match_table = of_match_ptr(visconti_gpio_of_match), + } +}; +module_platform_driver(visconti_gpio_driver); + +MODULE_AUTHOR("Nobuhiro Iwamatsu "); +MODULE_DESCRIPTION("Toshiba Visconti GPIO Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpio/gpio-vx855.c b/drivers/gpio/gpio-vx855.c index 3bf397b8dfbc..69713fd5485b 100644 --- a/drivers/gpio/gpio-vx855.c +++ b/drivers/gpio/gpio-vx855.c @@ -216,7 +216,7 @@ static void vx855gpio_gpio_setup(struct vx855_gpio *vg) c->direction_output = vx855gpio_direction_output; c->get = vx855gpio_get; c->set = vx855gpio_set; - c->set_config = vx855gpio_set_config, + c->set_config = vx855gpio_set_config; c->dbg_show = NULL; c->base = 0; c->ngpio = NR_VX855_GP; diff --git a/drivers/gpio/gpio-wcove.c b/drivers/gpio/gpio-wcove.c index b5fbba5a783a..a19eeef6cf1e 100644 --- a/drivers/gpio/gpio-wcove.c +++ b/drivers/gpio/gpio-wcove.c @@ -73,6 +73,8 @@ enum ctrl_register { CTRL_IN, CTRL_OUT, + IRQ_STATUS, + IRQ_MASK, }; /* @@ -112,22 +114,29 @@ static inline int to_reg(int gpio, enum ctrl_register reg_type) return reg; } -static void wcove_update_irq_mask(struct wcove_gpio *wg, int gpio) +static inline int to_ireg(int gpio, enum ctrl_register type, unsigned int *mask) { - unsigned int reg, mask; + unsigned int reg = type == IRQ_STATUS ? IRQ_STATUS_BASE : IRQ_MASK_BASE; if (gpio < GROUP0_NR_IRQS) { - reg = IRQ_MASK_BASE; - mask = BIT(gpio % GROUP0_NR_IRQS); + reg += 0; + *mask = BIT(gpio); } else { - reg = IRQ_MASK_BASE + 1; - mask = BIT((gpio - GROUP0_NR_IRQS) % GROUP1_NR_IRQS); + reg += 1; + *mask = BIT(gpio - GROUP0_NR_IRQS); } + return reg; +} + +static void wcove_update_irq_mask(struct wcove_gpio *wg, int gpio) +{ + unsigned int mask, reg = to_ireg(gpio, IRQ_MASK, &mask); + if (wg->set_irq_mask) - regmap_update_bits(wg->regmap, reg, mask, mask); + regmap_set_bits(wg->regmap, reg, mask); else - regmap_update_bits(wg->regmap, reg, mask, 0); + regmap_clear_bits(wg->regmap, reg, mask); } static void wcove_update_irq_ctrl(struct wcove_gpio *wg, int gpio) @@ -207,9 +216,9 @@ static void wcove_gpio_set(struct gpio_chip *chip, unsigned int gpio, int value) return; if (value) - regmap_update_bits(wg->regmap, reg, 1, 1); + regmap_set_bits(wg->regmap, reg, 1); else - regmap_update_bits(wg->regmap, reg, 1, 0); + regmap_clear_bits(wg->regmap, reg, 1); } static int wcove_gpio_set_config(struct gpio_chip *chip, unsigned int gpio, @@ -324,7 +333,8 @@ static struct irq_chip wcove_irqchip = { static irqreturn_t wcove_gpio_irq_handler(int irq, void *data) { struct wcove_gpio *wg = (struct wcove_gpio *)data; - unsigned int pending, virq, gpio, mask, offset; + unsigned int virq, gpio; + unsigned long pending; u8 p[2]; if (regmap_bulk_read(wg->regmap, IRQ_STATUS_BASE, p, 2)) { @@ -339,15 +349,12 @@ static irqreturn_t wcove_gpio_irq_handler(int irq, void *data) /* Iterate until no interrupt is pending */ while (pending) { /* One iteration is for all pending bits */ - for_each_set_bit(gpio, (const unsigned long *)&pending, - WCOVE_GPIO_NUM) { - offset = (gpio > GROUP0_NR_IRQS) ? 1 : 0; - mask = (offset == 1) ? BIT(gpio - GROUP0_NR_IRQS) : - BIT(gpio); + for_each_set_bit(gpio, &pending, WCOVE_GPIO_NUM) { + unsigned int mask, reg = to_ireg(gpio, IRQ_STATUS, &mask); + virq = irq_find_mapping(wg->chip.irq.domain, gpio); handle_nested_irq(virq); - regmap_update_bits(wg->regmap, IRQ_STATUS_BASE + offset, - mask, mask); + regmap_set_bits(wg->regmap, reg, mask); } /* Next iteration */ @@ -367,30 +374,26 @@ static void wcove_gpio_dbg_show(struct seq_file *s, { unsigned int ctlo, ctli, irq_mask, irq_status; struct wcove_gpio *wg = gpiochip_get_data(chip); - int gpio, offset, group, ret = 0; + int gpio, mask, ret = 0; for (gpio = 0; gpio < WCOVE_GPIO_NUM; gpio++) { - group = gpio < GROUP0_NR_IRQS ? 0 : 1; ret += regmap_read(wg->regmap, to_reg(gpio, CTRL_OUT), &ctlo); ret += regmap_read(wg->regmap, to_reg(gpio, CTRL_IN), &ctli); - ret += regmap_read(wg->regmap, IRQ_MASK_BASE + group, - &irq_mask); - ret += regmap_read(wg->regmap, IRQ_STATUS_BASE + group, - &irq_status); + ret += regmap_read(wg->regmap, to_ireg(gpio, IRQ_MASK, &mask), &irq_mask); + ret += regmap_read(wg->regmap, to_ireg(gpio, IRQ_STATUS, &mask), &irq_status); if (ret) { pr_err("Failed to read registers: ctrl out/in or irq status/mask\n"); break; } - offset = gpio % 8; seq_printf(s, " gpio-%-2d %s %s %s %s ctlo=%2x,%s %s\n", gpio, ctlo & CTLO_DIR_OUT ? "out" : "in ", ctli & 0x1 ? "hi" : "lo", ctli & CTLI_INTCNT_NE ? "fall" : " ", ctli & CTLI_INTCNT_PE ? "rise" : " ", ctlo, - irq_mask & BIT(offset) ? "mask " : "unmask", - irq_status & BIT(offset) ? "pending" : " "); + irq_mask & mask ? "mask " : "unmask", + irq_status & mask ? "pending" : " "); } } @@ -434,7 +437,7 @@ static int wcove_gpio_probe(struct platform_device *pdev) wg->chip.get_direction = wcove_gpio_get_direction; wg->chip.get = wcove_gpio_get; wg->chip.set = wcove_gpio_set; - wg->chip.set_config = wcove_gpio_set_config, + wg->chip.set_config = wcove_gpio_set_config; wg->chip.base = -1; wg->chip.ngpio = WCOVE_VGPIO_NUM; wg->chip.can_sleep = true; @@ -473,14 +476,12 @@ static int wcove_gpio_probe(struct platform_device *pdev) } /* Enable GPIO0 interrupts */ - ret = regmap_update_bits(wg->regmap, IRQ_MASK_BASE, GPIO_IRQ0_MASK, - 0x00); + ret = regmap_clear_bits(wg->regmap, IRQ_MASK_BASE + 0, GPIO_IRQ0_MASK); if (ret) return ret; /* Enable GPIO1 interrupts */ - ret = regmap_update_bits(wg->regmap, IRQ_MASK_BASE + 1, GPIO_IRQ1_MASK, - 0x00); + ret = regmap_clear_bits(wg->regmap, IRQ_MASK_BASE + 1, GPIO_IRQ1_MASK); if (ret) return ret; diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c index be539381fd82..b411d3156e0b 100644 --- a/drivers/gpio/gpio-xilinx.c +++ b/drivers/gpio/gpio-xilinx.c @@ -10,10 +10,13 @@ #include #include #include +#include #include +#include #include #include #include +#include #include /* Register Offset Definitions */ @@ -22,6 +25,11 @@ #define XGPIO_CHANNEL_OFFSET 0x8 +#define XGPIO_GIER_OFFSET 0x11c /* Global Interrupt Enable */ +#define XGPIO_GIER_IE BIT(31) +#define XGPIO_IPISR_OFFSET 0x120 /* IP Interrupt Status */ +#define XGPIO_IPIER_OFFSET 0x128 /* IP Interrupt Enable */ + /* Read/Write access to the GPIO registers */ #if defined(CONFIG_ARCH_ZYNQ) || defined(CONFIG_X86) # define xgpio_readreg(offset) readl(offset) @@ -36,9 +44,15 @@ * @gc: GPIO chip * @regs: register block * @gpio_width: GPIO width for every channel - * @gpio_state: GPIO state shadow register + * @gpio_state: GPIO write state shadow register + * @gpio_last_irq_read: GPIO read state register from last interrupt * @gpio_dir: GPIO direction shadow register * @gpio_lock: Lock used for synchronization + * @irq: IRQ used by GPIO device + * @irqchip: IRQ chip + * @irq_enable: GPIO IRQ enable/disable bitfield + * @irq_rising_edge: GPIO IRQ rising edge enable/disable bitfield + * @irq_falling_edge: GPIO IRQ falling edge enable/disable bitfield * @clk: clock resource for this driver */ struct xgpio_instance { @@ -46,8 +60,14 @@ struct xgpio_instance { void __iomem *regs; unsigned int gpio_width[2]; u32 gpio_state[2]; + u32 gpio_last_irq_read[2]; u32 gpio_dir[2]; - spinlock_t gpio_lock[2]; + spinlock_t gpio_lock; /* For serializing operations */ + int irq; + struct irq_chip irqchip; + u32 irq_enable[2]; + u32 irq_rising_edge[2]; + u32 irq_falling_edge[2]; struct clk *clk; }; @@ -113,7 +133,7 @@ static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val) int index = xgpio_index(chip, gpio); int offset = xgpio_offset(chip, gpio); - spin_lock_irqsave(&chip->gpio_lock[index], flags); + spin_lock_irqsave(&chip->gpio_lock, flags); /* Write to GPIO signal and set its direction to output */ if (val) @@ -124,7 +144,7 @@ static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val) xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET + xgpio_regoffset(chip, gpio), chip->gpio_state[index]); - spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + spin_unlock_irqrestore(&chip->gpio_lock, flags); } /** @@ -144,7 +164,7 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, int index = xgpio_index(chip, 0); int offset, i; - spin_lock_irqsave(&chip->gpio_lock[index], flags); + spin_lock_irqsave(&chip->gpio_lock, flags); /* Write to GPIO signals */ for (i = 0; i < gc->ngpio; i++) { @@ -155,9 +175,9 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET + index * XGPIO_CHANNEL_OFFSET, chip->gpio_state[index]); - spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + spin_unlock_irqrestore(&chip->gpio_lock, flags); index = xgpio_index(chip, i); - spin_lock_irqsave(&chip->gpio_lock[index], flags); + spin_lock_irqsave(&chip->gpio_lock, flags); } if (__test_and_clear_bit(i, mask)) { offset = xgpio_offset(chip, i); @@ -171,7 +191,7 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET + index * XGPIO_CHANNEL_OFFSET, chip->gpio_state[index]); - spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + spin_unlock_irqrestore(&chip->gpio_lock, flags); } /** @@ -190,14 +210,14 @@ static int xgpio_dir_in(struct gpio_chip *gc, unsigned int gpio) int index = xgpio_index(chip, gpio); int offset = xgpio_offset(chip, gpio); - spin_lock_irqsave(&chip->gpio_lock[index], flags); + spin_lock_irqsave(&chip->gpio_lock, flags); /* Set the GPIO bit in shadow register and set direction as input */ chip->gpio_dir[index] |= BIT(offset); xgpio_writereg(chip->regs + XGPIO_TRI_OFFSET + xgpio_regoffset(chip, gpio), chip->gpio_dir[index]); - spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + spin_unlock_irqrestore(&chip->gpio_lock, flags); return 0; } @@ -221,7 +241,7 @@ static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) int index = xgpio_index(chip, gpio); int offset = xgpio_offset(chip, gpio); - spin_lock_irqsave(&chip->gpio_lock[index], flags); + spin_lock_irqsave(&chip->gpio_lock, flags); /* Write state of GPIO signal */ if (val) @@ -236,7 +256,7 @@ static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) xgpio_writereg(chip->regs + XGPIO_TRI_OFFSET + xgpio_regoffset(chip, gpio), chip->gpio_dir[index]); - spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + spin_unlock_irqrestore(&chip->gpio_lock, flags); return 0; } @@ -259,6 +279,39 @@ static void xgpio_save_regs(struct xgpio_instance *chip) chip->gpio_dir[1]); } +static int xgpio_request(struct gpio_chip *chip, unsigned int offset) +{ + int ret; + + ret = pm_runtime_get_sync(chip->parent); + /* + * If the device is already active pm_runtime_get() will return 1 on + * success, but gpio_request still needs to return 0. + */ + return ret < 0 ? ret : 0; +} + +static void xgpio_free(struct gpio_chip *chip, unsigned int offset) +{ + pm_runtime_put(chip->parent); +} + +static int __maybe_unused xgpio_suspend(struct device *dev) +{ + struct xgpio_instance *gpio = dev_get_drvdata(dev); + struct irq_data *data = irq_get_irq_data(gpio->irq); + + if (!data) { + dev_err(dev, "irq_get_irq_data() failed\n"); + return -EINVAL; + } + + if (!irqd_is_wakeup_set(data)) + return pm_runtime_force_suspend(dev); + + return 0; +} + /** * xgpio_remove - Remove method for the GPIO device. * @pdev: pointer to the platform device @@ -271,11 +324,223 @@ static int xgpio_remove(struct platform_device *pdev) { struct xgpio_instance *gpio = platform_get_drvdata(pdev); + pm_runtime_get_sync(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); clk_disable_unprepare(gpio->clk); return 0; } +/** + * xgpio_irq_ack - Acknowledge a child GPIO interrupt. + * @irq_data: per IRQ and chip data passed down to chip functions + * This currently does nothing, but irq_ack is unconditionally called by + * handle_edge_irq and therefore must be defined. + */ +static void xgpio_irq_ack(struct irq_data *irq_data) +{ +} + +static int __maybe_unused xgpio_resume(struct device *dev) +{ + struct xgpio_instance *gpio = dev_get_drvdata(dev); + struct irq_data *data = irq_get_irq_data(gpio->irq); + + if (!data) { + dev_err(dev, "irq_get_irq_data() failed\n"); + return -EINVAL; + } + + if (!irqd_is_wakeup_set(data)) + return pm_runtime_force_resume(dev); + + return 0; +} + +static int __maybe_unused xgpio_runtime_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct xgpio_instance *gpio = platform_get_drvdata(pdev); + + clk_disable(gpio->clk); + + return 0; +} + +static int __maybe_unused xgpio_runtime_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct xgpio_instance *gpio = platform_get_drvdata(pdev); + + return clk_enable(gpio->clk); +} + +static const struct dev_pm_ops xgpio_dev_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(xgpio_suspend, xgpio_resume) + SET_RUNTIME_PM_OPS(xgpio_runtime_suspend, + xgpio_runtime_resume, NULL) +}; + +/** + * xgpio_irq_mask - Write the specified signal of the GPIO device. + * @irq_data: per IRQ and chip data passed down to chip functions + */ +static void xgpio_irq_mask(struct irq_data *irq_data) +{ + unsigned long flags; + struct xgpio_instance *chip = irq_data_get_irq_chip_data(irq_data); + int irq_offset = irqd_to_hwirq(irq_data); + int index = xgpio_index(chip, irq_offset); + int offset = xgpio_offset(chip, irq_offset); + + spin_lock_irqsave(&chip->gpio_lock, flags); + + chip->irq_enable[index] &= ~BIT(offset); + + if (!chip->irq_enable[index]) { + /* Disable per channel interrupt */ + u32 temp = xgpio_readreg(chip->regs + XGPIO_IPIER_OFFSET); + + temp &= ~BIT(index); + xgpio_writereg(chip->regs + XGPIO_IPIER_OFFSET, temp); + } + spin_unlock_irqrestore(&chip->gpio_lock, flags); +} + +/** + * xgpio_irq_unmask - Write the specified signal of the GPIO device. + * @irq_data: per IRQ and chip data passed down to chip functions + */ +static void xgpio_irq_unmask(struct irq_data *irq_data) +{ + unsigned long flags; + struct xgpio_instance *chip = irq_data_get_irq_chip_data(irq_data); + int irq_offset = irqd_to_hwirq(irq_data); + int index = xgpio_index(chip, irq_offset); + int offset = xgpio_offset(chip, irq_offset); + u32 old_enable = chip->irq_enable[index]; + + spin_lock_irqsave(&chip->gpio_lock, flags); + + chip->irq_enable[index] |= BIT(offset); + + if (!old_enable) { + /* Clear any existing per-channel interrupts */ + u32 val = xgpio_readreg(chip->regs + XGPIO_IPISR_OFFSET) & + BIT(index); + + if (val) + xgpio_writereg(chip->regs + XGPIO_IPISR_OFFSET, val); + + /* Update GPIO IRQ read data before enabling interrupt*/ + val = xgpio_readreg(chip->regs + XGPIO_DATA_OFFSET + + index * XGPIO_CHANNEL_OFFSET); + chip->gpio_last_irq_read[index] = val; + + /* Enable per channel interrupt */ + val = xgpio_readreg(chip->regs + XGPIO_IPIER_OFFSET); + val |= BIT(index); + xgpio_writereg(chip->regs + XGPIO_IPIER_OFFSET, val); + } + + spin_unlock_irqrestore(&chip->gpio_lock, flags); +} + +/** + * xgpio_set_irq_type - Write the specified signal of the GPIO device. + * @irq_data: Per IRQ and chip data passed down to chip functions + * @type: Interrupt type that is to be set for the gpio pin + * + * Return: + * 0 if interrupt type is supported otherwise -EINVAL + */ +static int xgpio_set_irq_type(struct irq_data *irq_data, unsigned int type) +{ + struct xgpio_instance *chip = irq_data_get_irq_chip_data(irq_data); + int irq_offset = irqd_to_hwirq(irq_data); + int index = xgpio_index(chip, irq_offset); + int offset = xgpio_offset(chip, irq_offset); + + /* + * The Xilinx GPIO hardware provides a single interrupt status + * indication for any state change in a given GPIO channel (bank). + * Therefore, only rising edge or falling edge triggers are + * supported. + */ + switch (type & IRQ_TYPE_SENSE_MASK) { + case IRQ_TYPE_EDGE_BOTH: + chip->irq_rising_edge[index] |= BIT(offset); + chip->irq_falling_edge[index] |= BIT(offset); + break; + case IRQ_TYPE_EDGE_RISING: + chip->irq_rising_edge[index] |= BIT(offset); + chip->irq_falling_edge[index] &= ~BIT(offset); + break; + case IRQ_TYPE_EDGE_FALLING: + chip->irq_rising_edge[index] &= ~BIT(offset); + chip->irq_falling_edge[index] |= BIT(offset); + break; + default: + return -EINVAL; + } + + irq_set_handler_locked(irq_data, handle_edge_irq); + return 0; +} + +/** + * xgpio_irqhandler - Gpio interrupt service routine + * @desc: Pointer to interrupt description + */ +static void xgpio_irqhandler(struct irq_desc *desc) +{ + struct xgpio_instance *chip = irq_desc_get_handler_data(desc); + struct irq_chip *irqchip = irq_desc_get_chip(desc); + u32 num_channels = chip->gpio_width[1] ? 2 : 1; + u32 offset = 0, index; + u32 status = xgpio_readreg(chip->regs + XGPIO_IPISR_OFFSET); + + xgpio_writereg(chip->regs + XGPIO_IPISR_OFFSET, status); + + chained_irq_enter(irqchip, desc); + for (index = 0; index < num_channels; index++) { + if ((status & BIT(index))) { + unsigned long rising_events, falling_events, all_events; + unsigned long flags; + u32 data, bit; + unsigned int irq; + + spin_lock_irqsave(&chip->gpio_lock, flags); + data = xgpio_readreg(chip->regs + XGPIO_DATA_OFFSET + + index * XGPIO_CHANNEL_OFFSET); + rising_events = data & + ~chip->gpio_last_irq_read[index] & + chip->irq_enable[index] & + chip->irq_rising_edge[index]; + falling_events = ~data & + chip->gpio_last_irq_read[index] & + chip->irq_enable[index] & + chip->irq_falling_edge[index]; + dev_dbg(chip->gc.parent, + "IRQ chan %u rising 0x%lx falling 0x%lx\n", + index, rising_events, falling_events); + all_events = rising_events | falling_events; + chip->gpio_last_irq_read[index] = data; + spin_unlock_irqrestore(&chip->gpio_lock, flags); + + for_each_set_bit(bit, &all_events, 32) { + irq = irq_find_mapping(chip->gc.irq.domain, + offset + bit); + generic_handle_irq(irq); + } + } + offset += chip->gpio_width[index]; + } + + chained_irq_exit(irqchip, desc); +} + /** * xgpio_of_probe - Probe method for the GPIO device. * @pdev: pointer to the platform device @@ -289,7 +554,10 @@ static int xgpio_probe(struct platform_device *pdev) struct xgpio_instance *chip; int status = 0; struct device_node *np = pdev->dev.of_node; - u32 is_dual; + u32 is_dual = 0; + u32 cells = 2; + struct gpio_irq_chip *girq; + u32 temp; chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL); if (!chip) @@ -305,6 +573,15 @@ static int xgpio_probe(struct platform_device *pdev) if (of_property_read_u32(np, "xlnx,tri-default", &chip->gpio_dir[0])) chip->gpio_dir[0] = 0xFFFFFFFF; + /* Update cells with gpio-cells value */ + if (of_property_read_u32(np, "#gpio-cells", &cells)) + dev_dbg(&pdev->dev, "Missing gpio-cells property\n"); + + if (cells != 2) { + dev_err(&pdev->dev, "#gpio-cells mismatch\n"); + return -EINVAL; + } + /* * Check device node and parent device node for device width * and assume default width of 32 @@ -312,7 +589,10 @@ static int xgpio_probe(struct platform_device *pdev) if (of_property_read_u32(np, "xlnx,gpio-width", &chip->gpio_width[0])) chip->gpio_width[0] = 32; - spin_lock_init(&chip->gpio_lock[0]); + if (chip->gpio_width[0] > 32) + return -EINVAL; + + spin_lock_init(&chip->gpio_lock); if (of_property_read_u32(np, "xlnx,is-dual", &is_dual)) is_dual = 0; @@ -336,7 +616,8 @@ static int xgpio_probe(struct platform_device *pdev) &chip->gpio_width[1])) chip->gpio_width[1] = 32; - spin_lock_init(&chip->gpio_lock[1]); + if (chip->gpio_width[1] > 32) + return -EINVAL; } chip->gc.base = -1; @@ -344,8 +625,11 @@ static int xgpio_probe(struct platform_device *pdev) chip->gc.parent = &pdev->dev; chip->gc.direction_input = xgpio_dir_in; chip->gc.direction_output = xgpio_dir_out; + chip->gc.of_gpio_n_cells = cells; chip->gc.get = xgpio_get; chip->gc.set = xgpio_set; + chip->gc.request = xgpio_request; + chip->gc.free = xgpio_free; chip->gc.set_multiple = xgpio_set_multiple; chip->gc.label = dev_name(&pdev->dev); @@ -357,28 +641,68 @@ static int xgpio_probe(struct platform_device *pdev) } chip->clk = devm_clk_get_optional(&pdev->dev, NULL); - if (IS_ERR(chip->clk)) { - if (PTR_ERR(chip->clk) != -EPROBE_DEFER) - dev_dbg(&pdev->dev, "Input clock not found\n"); - return PTR_ERR(chip->clk); - } + if (IS_ERR(chip->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(chip->clk), "input clock not found.\n"); status = clk_prepare_enable(chip->clk); if (status < 0) { dev_err(&pdev->dev, "Failed to prepare clk\n"); return status; } + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); xgpio_save_regs(chip); + chip->irq = platform_get_irq_optional(pdev, 0); + if (chip->irq <= 0) + goto skip_irq; + + chip->irqchip.name = "gpio-xilinx"; + chip->irqchip.irq_ack = xgpio_irq_ack; + chip->irqchip.irq_mask = xgpio_irq_mask; + chip->irqchip.irq_unmask = xgpio_irq_unmask; + chip->irqchip.irq_set_type = xgpio_set_irq_type; + + /* Disable per-channel interrupts */ + xgpio_writereg(chip->regs + XGPIO_IPIER_OFFSET, 0); + /* Clear any existing per-channel interrupts */ + temp = xgpio_readreg(chip->regs + XGPIO_IPISR_OFFSET); + xgpio_writereg(chip->regs + XGPIO_IPISR_OFFSET, temp); + /* Enable global interrupts */ + xgpio_writereg(chip->regs + XGPIO_GIER_OFFSET, XGPIO_GIER_IE); + + girq = &chip->gc.irq; + girq->chip = &chip->irqchip; + girq->parent_handler = xgpio_irqhandler; + girq->num_parents = 1; + girq->parents = devm_kcalloc(&pdev->dev, 1, + sizeof(*girq->parents), + GFP_KERNEL); + if (!girq->parents) { + status = -ENOMEM; + goto err_pm_put; + } + girq->parents[0] = chip->irq; + girq->default_type = IRQ_TYPE_NONE; + girq->handler = handle_bad_irq; + +skip_irq: status = devm_gpiochip_add_data(&pdev->dev, &chip->gc, chip); if (status) { dev_err(&pdev->dev, "failed to add GPIO chip\n"); - clk_disable_unprepare(chip->clk); - return status; + goto err_pm_put; } + pm_runtime_put(&pdev->dev); return 0; + +err_pm_put: + pm_runtime_disable(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + clk_disable_unprepare(chip->clk); + return status; } static const struct of_device_id xgpio_of_match[] = { @@ -394,6 +718,7 @@ static struct platform_driver xgpio_plat_driver = { .driver = { .name = "gpio-xilinx", .of_match_table = xgpio_of_match, + .pm = &xgpio_dev_pm_ops, }, }; diff --git a/drivers/gpio/gpio-zx.c b/drivers/gpio/gpio-zx.c deleted file mode 100644 index 64bfb722756a..000000000000 --- a/drivers/gpio/gpio-zx.c +++ /dev/null @@ -1,289 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * ZTE ZX296702 GPIO driver - * - * Author: Jun Nie - * - * Copyright (C) 2015 Linaro Ltd. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define ZX_GPIO_DIR 0x00 -#define ZX_GPIO_IVE 0x04 -#define ZX_GPIO_IV 0x08 -#define ZX_GPIO_IEP 0x0C -#define ZX_GPIO_IEN 0x10 -#define ZX_GPIO_DI 0x14 -#define ZX_GPIO_DO1 0x18 -#define ZX_GPIO_DO0 0x1C -#define ZX_GPIO_DO 0x20 - -#define ZX_GPIO_IM 0x28 -#define ZX_GPIO_IE 0x2C - -#define ZX_GPIO_MIS 0x30 -#define ZX_GPIO_IC 0x34 - -#define ZX_GPIO_NR 16 - -struct zx_gpio { - raw_spinlock_t lock; - - void __iomem *base; - struct gpio_chip gc; -}; - -static int zx_direction_input(struct gpio_chip *gc, unsigned offset) -{ - struct zx_gpio *chip = gpiochip_get_data(gc); - unsigned long flags; - u16 gpiodir; - - if (offset >= gc->ngpio) - return -EINVAL; - - raw_spin_lock_irqsave(&chip->lock, flags); - gpiodir = readw_relaxed(chip->base + ZX_GPIO_DIR); - gpiodir &= ~BIT(offset); - writew_relaxed(gpiodir, chip->base + ZX_GPIO_DIR); - raw_spin_unlock_irqrestore(&chip->lock, flags); - - return 0; -} - -static int zx_direction_output(struct gpio_chip *gc, unsigned offset, - int value) -{ - struct zx_gpio *chip = gpiochip_get_data(gc); - unsigned long flags; - u16 gpiodir; - - if (offset >= gc->ngpio) - return -EINVAL; - - raw_spin_lock_irqsave(&chip->lock, flags); - gpiodir = readw_relaxed(chip->base + ZX_GPIO_DIR); - gpiodir |= BIT(offset); - writew_relaxed(gpiodir, chip->base + ZX_GPIO_DIR); - - if (value) - writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO1); - else - writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO0); - raw_spin_unlock_irqrestore(&chip->lock, flags); - - return 0; -} - -static int zx_get_value(struct gpio_chip *gc, unsigned offset) -{ - struct zx_gpio *chip = gpiochip_get_data(gc); - - return !!(readw_relaxed(chip->base + ZX_GPIO_DI) & BIT(offset)); -} - -static void zx_set_value(struct gpio_chip *gc, unsigned offset, int value) -{ - struct zx_gpio *chip = gpiochip_get_data(gc); - - if (value) - writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO1); - else - writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO0); -} - -static int zx_irq_type(struct irq_data *d, unsigned trigger) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct zx_gpio *chip = gpiochip_get_data(gc); - int offset = irqd_to_hwirq(d); - unsigned long flags; - u16 gpiois, gpioi_epos, gpioi_eneg, gpioiev; - u16 bit = BIT(offset); - - if (offset < 0 || offset >= ZX_GPIO_NR) - return -EINVAL; - - raw_spin_lock_irqsave(&chip->lock, flags); - - gpioiev = readw_relaxed(chip->base + ZX_GPIO_IV); - gpiois = readw_relaxed(chip->base + ZX_GPIO_IVE); - gpioi_epos = readw_relaxed(chip->base + ZX_GPIO_IEP); - gpioi_eneg = readw_relaxed(chip->base + ZX_GPIO_IEN); - - if (trigger & (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW)) { - gpiois |= bit; - if (trigger & IRQ_TYPE_LEVEL_HIGH) - gpioiev |= bit; - else - gpioiev &= ~bit; - } else - gpiois &= ~bit; - - if ((trigger & IRQ_TYPE_EDGE_BOTH) == IRQ_TYPE_EDGE_BOTH) { - gpioi_epos |= bit; - gpioi_eneg |= bit; - } else { - if (trigger & IRQ_TYPE_EDGE_RISING) { - gpioi_epos |= bit; - gpioi_eneg &= ~bit; - } else if (trigger & IRQ_TYPE_EDGE_FALLING) { - gpioi_eneg |= bit; - gpioi_epos &= ~bit; - } - } - - writew_relaxed(gpiois, chip->base + ZX_GPIO_IVE); - writew_relaxed(gpioi_epos, chip->base + ZX_GPIO_IEP); - writew_relaxed(gpioi_eneg, chip->base + ZX_GPIO_IEN); - writew_relaxed(gpioiev, chip->base + ZX_GPIO_IV); - raw_spin_unlock_irqrestore(&chip->lock, flags); - - return 0; -} - -static void zx_irq_handler(struct irq_desc *desc) -{ - unsigned long pending; - int offset; - struct gpio_chip *gc = irq_desc_get_handler_data(desc); - struct zx_gpio *chip = gpiochip_get_data(gc); - struct irq_chip *irqchip = irq_desc_get_chip(desc); - - chained_irq_enter(irqchip, desc); - - pending = readw_relaxed(chip->base + ZX_GPIO_MIS); - writew_relaxed(pending, chip->base + ZX_GPIO_IC); - if (pending) { - for_each_set_bit(offset, &pending, ZX_GPIO_NR) - generic_handle_irq(irq_find_mapping(gc->irq.domain, - offset)); - } - - chained_irq_exit(irqchip, desc); -} - -static void zx_irq_mask(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct zx_gpio *chip = gpiochip_get_data(gc); - u16 mask = BIT(irqd_to_hwirq(d) % ZX_GPIO_NR); - u16 gpioie; - - raw_spin_lock(&chip->lock); - gpioie = readw_relaxed(chip->base + ZX_GPIO_IM) | mask; - writew_relaxed(gpioie, chip->base + ZX_GPIO_IM); - gpioie = readw_relaxed(chip->base + ZX_GPIO_IE) & ~mask; - writew_relaxed(gpioie, chip->base + ZX_GPIO_IE); - raw_spin_unlock(&chip->lock); -} - -static void zx_irq_unmask(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct zx_gpio *chip = gpiochip_get_data(gc); - u16 mask = BIT(irqd_to_hwirq(d) % ZX_GPIO_NR); - u16 gpioie; - - raw_spin_lock(&chip->lock); - gpioie = readw_relaxed(chip->base + ZX_GPIO_IM) & ~mask; - writew_relaxed(gpioie, chip->base + ZX_GPIO_IM); - gpioie = readw_relaxed(chip->base + ZX_GPIO_IE) | mask; - writew_relaxed(gpioie, chip->base + ZX_GPIO_IE); - raw_spin_unlock(&chip->lock); -} - -static struct irq_chip zx_irqchip = { - .name = "zx-gpio", - .irq_mask = zx_irq_mask, - .irq_unmask = zx_irq_unmask, - .irq_set_type = zx_irq_type, -}; - -static int zx_gpio_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct zx_gpio *chip; - struct gpio_irq_chip *girq; - int irq, id, ret; - - chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL); - if (!chip) - return -ENOMEM; - - chip->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(chip->base)) - return PTR_ERR(chip->base); - - id = of_alias_get_id(dev->of_node, "gpio"); - - raw_spin_lock_init(&chip->lock); - chip->gc.request = gpiochip_generic_request; - chip->gc.free = gpiochip_generic_free; - chip->gc.direction_input = zx_direction_input; - chip->gc.direction_output = zx_direction_output; - chip->gc.get = zx_get_value; - chip->gc.set = zx_set_value; - chip->gc.base = ZX_GPIO_NR * id; - chip->gc.ngpio = ZX_GPIO_NR; - chip->gc.label = dev_name(dev); - chip->gc.parent = dev; - chip->gc.owner = THIS_MODULE; - - /* - * irq_chip support - */ - writew_relaxed(0xffff, chip->base + ZX_GPIO_IM); - writew_relaxed(0, chip->base + ZX_GPIO_IE); - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; - girq = &chip->gc.irq; - girq->chip = &zx_irqchip; - girq->parent_handler = zx_irq_handler; - girq->num_parents = 1; - girq->parents = devm_kcalloc(&pdev->dev, 1, - sizeof(*girq->parents), - GFP_KERNEL); - if (!girq->parents) - return -ENOMEM; - girq->parents[0] = irq; - girq->default_type = IRQ_TYPE_NONE; - girq->handler = handle_simple_irq; - - ret = gpiochip_add_data(&chip->gc, chip); - if (ret) - return ret; - - platform_set_drvdata(pdev, chip); - dev_info(dev, "ZX GPIO chip registered\n"); - - return 0; -} - -static const struct of_device_id zx_gpio_match[] = { - { - .compatible = "zte,zx296702-gpio", - }, - { }, -}; - -static struct platform_driver zx_gpio_driver = { - .probe = zx_gpio_probe, - .driver = { - .name = "zx_gpio", - .of_match_table = of_match_ptr(zx_gpio_match), - }, -}; -builtin_platform_driver(zx_gpio_driver) diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index b4a71119a4b0..baf0153b7bca 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -1039,3 +1039,14 @@ void of_gpiochip_remove(struct gpio_chip *chip) { of_node_put(chip->of_node); } + +void of_gpio_dev_init(struct gpio_chip *gc, struct gpio_device *gdev) +{ + /* If the gpiochip has an assigned OF node this takes precedence */ + if (gc->of_node) + gdev->dev.of_node = gc->of_node; + else + gc->of_node = gdev->dev.of_node; + if (gdev->dev.of_node) + gdev->dev.fwnode = of_fwnode_handle(gdev->dev.of_node); +} diff --git a/drivers/gpio/gpiolib-of.h b/drivers/gpio/gpiolib-of.h index ed26664f1537..8af2bc899aab 100644 --- a/drivers/gpio/gpiolib-of.h +++ b/drivers/gpio/gpiolib-of.h @@ -15,6 +15,7 @@ int of_gpiochip_add(struct gpio_chip *gc); void of_gpiochip_remove(struct gpio_chip *gc); int of_gpio_get_count(struct device *dev, const char *con_id); bool of_gpio_need_valid_mask(const struct gpio_chip *gc); +void of_gpio_dev_init(struct gpio_chip *gc, struct gpio_device *gdev); #else static inline struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, @@ -33,6 +34,10 @@ static inline bool of_gpio_need_valid_mask(const struct gpio_chip *gc) { return false; } +static inline void of_gpio_dev_init(struct gpio_chip *gc, + struct gpio_device *gdev) +{ +} #endif /* CONFIG_OF_GPIO */ extern struct notifier_block gpio_of_notifier; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 97eec8d8dbdc..adf55db080d8 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -56,8 +56,10 @@ static DEFINE_IDA(gpio_ida); static dev_t gpio_devt; #define GPIO_DEV_MAX 256 /* 256 GPIO chip devices supported */ +static int gpio_bus_match(struct device *dev, struct device_driver *drv); static struct bus_type gpio_bus_type = { .name = "gpio", + .match = gpio_bus_match, }; /* @@ -590,13 +592,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, gdev->dev.of_node = gc->parent->of_node; } -#ifdef CONFIG_OF_GPIO - /* If the gpiochip has an assigned OF node this takes precedence */ - if (gc->of_node) - gdev->dev.of_node = gc->of_node; - else - gc->of_node = gdev->dev.of_node; -#endif + of_gpio_dev_init(gc, gdev); gdev->id = ida_alloc(&gpio_ida, GFP_KERNEL); if (gdev->id < 0) { @@ -3469,6 +3465,10 @@ EXPORT_SYMBOL_GPL(gpiod_add_lookup_table); */ void gpiod_remove_lookup_table(struct gpiod_lookup_table *table) { + /* Nothing to remove */ + if (!table) + return; + mutex_lock(&gpio_lookup_lock); list_del(&table->list); @@ -4211,6 +4211,41 @@ void gpiod_put_array(struct gpio_descs *descs) } EXPORT_SYMBOL_GPL(gpiod_put_array); + +static int gpio_bus_match(struct device *dev, struct device_driver *drv) +{ + /* + * Only match if the fwnode doesn't already have a proper struct device + * created for it. + */ + if (dev->fwnode && dev->fwnode->dev != dev) + return 0; + return 1; +} + +static int gpio_stub_drv_probe(struct device *dev) +{ + /* + * The DT node of some GPIO chips have a "compatible" property, but + * never have a struct device added and probed by a driver to register + * the GPIO chip with gpiolib. In such cases, fw_devlink=on will cause + * the consumers of the GPIO chip to get probe deferred forever because + * they will be waiting for a device associated with the GPIO chip + * firmware node to get added and bound to a driver. + * + * To allow these consumers to probe, we associate the struct + * gpio_device of the GPIO chip with the firmware node and then simply + * bind it to this stub driver. + */ + return 0; +} + +static struct device_driver gpio_stub_drv = { + .name = "gpio_stub_drv", + .bus = &gpio_bus_type, + .probe = gpio_stub_drv_probe, +}; + static int __init gpiolib_dev_init(void) { int ret; @@ -4222,9 +4257,16 @@ static int __init gpiolib_dev_init(void) return ret; } + if (driver_register(&gpio_stub_drv) < 0) { + pr_err("gpiolib: could not register GPIO stub driver\n"); + bus_unregister(&gpio_bus_type); + return ret; + } + ret = alloc_chrdev_region(&gpio_devt, 0, GPIO_DEV_MAX, GPIOCHIP_NAME); if (ret < 0) { pr_err("gpiolib: failed to allocate char dev region\n"); + driver_unregister(&gpio_stub_drv); bus_unregister(&gpio_bus_type); return ret; } diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 0973f408d75f..e392a90ca687 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -15,6 +15,9 @@ menuconfig DRM select I2C_ALGOBIT select DMA_SHARED_BUFFER select SYNC_FILE +# gallium uses SYS_kcmp for os_same_file_description() to de-duplicate +# device and dmabuf fd. Let's make sure that is available for our userspace. + select KCMP help Kernel-level support for the Direct Rendering Infrastructure (DRI) introduced in XFree86 4.0. If you say Y here, you need to select @@ -214,10 +217,6 @@ config DRM_GEM_SHMEM_HELPER help Choose this if you need the GEM shmem helper functions -config DRM_VM - bool - depends on DRM && MMU - config DRM_SCHED tristate depends on DRM @@ -391,7 +390,6 @@ source "drivers/gpu/drm/xlnx/Kconfig" menuconfig DRM_LEGACY bool "Enable legacy drivers (DANGEROUS)" depends on DRM && MMU - select DRM_VM help Enable legacy DRI1 drivers. Those drivers expose unsafe and dangerous APIs to user-space, which can be used to circumvent access diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index fefaff4c832d..926adef289db 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -5,7 +5,7 @@ drm-y := drm_auth.o drm_cache.o \ drm_file.o drm_gem.o drm_ioctl.o drm_irq.o \ - drm_memory.o drm_drv.o \ + drm_drv.o \ drm_sysfs.o drm_hashtab.o drm_mm.o \ drm_crtc.o drm_fourcc.o drm_modes.o drm_edid.o \ drm_encoder_slave.o \ @@ -20,9 +20,9 @@ drm-y := drm_auth.o drm_cache.o \ drm_client_modeset.o drm_atomic_uapi.o drm_hdcp.o \ drm_managed.o drm_vblank_work.o -drm-$(CONFIG_DRM_LEGACY) += drm_legacy_misc.o drm_bufs.o drm_context.o drm_dma.o drm_scatter.o drm_lock.o +drm-$(CONFIG_DRM_LEGACY) += drm_bufs.o drm_context.o drm_dma.o drm_legacy_misc.o drm_lock.o \ + drm_memory.o drm_scatter.o drm_vm.o drm-$(CONFIG_DRM_LIB_RANDOM) += lib/drm_random.o -drm-$(CONFIG_DRM_VM) += drm_vm.o drm-$(CONFIG_COMPAT) += drm_ioc32.o drm-$(CONFIG_DRM_GEM_CMA_HELPER) += drm_gem_cma_helper.o drm-$(CONFIG_DRM_GEM_SHMEM_HELPER) += drm_gem_shmem_helper.o diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 6bf6cfaea3f1..13ebb1f71e49 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -56,7 +56,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \ amdgpu_gmc.o amdgpu_mmhub.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \ amdgpu_vm_sdma.o amdgpu_discovery.o amdgpu_ras_eeprom.o amdgpu_nbio.o \ amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \ - amdgpu_fw_attestation.o + amdgpu_fw_attestation.o amdgpu_securedisplay.o amdgpu-$(CONFIG_PERF_EVENTS) += amdgpu_pmu.o @@ -71,7 +71,7 @@ amdgpu-y += \ vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o \ vega20_reg_init.o nbio_v7_4.o nbio_v2_3.o nv.o navi10_reg_init.o navi14_reg_init.o \ arct_reg_init.o navi12_reg_init.o mxgpu_nv.o sienna_cichlid_reg_init.o vangogh_reg_init.o \ - nbio_v7_2.o dimgrey_cavefish_reg_init.o + nbio_v7_2.o dimgrey_cavefish_reg_init.o hdp_v4_0.o hdp_v5_0.o # add DF block amdgpu-y += \ @@ -97,6 +97,7 @@ amdgpu-y += \ tonga_ih.o \ cz_ih.o \ vega10_ih.o \ + vega20_ih.o \ navi10_ih.o # add PSP block @@ -170,7 +171,8 @@ amdgpu-y += \ # add SMUIO block amdgpu-y += \ smuio_v9_0.o \ - smuio_v11_0.o + smuio_v11_0.o \ + smuio_v11_0_6.o # add amdkfd interfaces amdgpu-y += amdgpu_amdkfd.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 5993dd0fdd8e..b6879d97c9c9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -55,7 +55,6 @@ #include #include #include -#include #include #include @@ -89,6 +88,7 @@ #include "amdgpu_gfx.h" #include "amdgpu_sdma.h" #include "amdgpu_nbio.h" +#include "amdgpu_hdp.h" #include "amdgpu_dm.h" #include "amdgpu_virt.h" #include "amdgpu_csa.h" @@ -107,6 +107,7 @@ #include "amdgpu_gfxhub.h" #include "amdgpu_df.h" #include "amdgpu_smuio.h" +#include "amdgpu_hdp.h" #define MAX_GPU_INSTANCE 16 @@ -286,7 +287,7 @@ enum amdgpu_kiq_irq { #define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */ #define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */ -#define MAX_KIQ_REG_TRY 80 /* 20 -> 80 */ +#define MAX_KIQ_REG_TRY 1000 int amdgpu_device_ip_set_clockgating_state(void *dev, enum amd_ip_block_type block_type, @@ -578,7 +579,8 @@ enum amd_reset_method { AMD_RESET_METHOD_MODE0, AMD_RESET_METHOD_MODE1, AMD_RESET_METHOD_MODE2, - AMD_RESET_METHOD_BACO + AMD_RESET_METHOD_BACO, + AMD_RESET_METHOD_PCI, }; /* @@ -608,7 +610,6 @@ struct amdgpu_asic_funcs { /* invalidate hdp read cache */ void (*invalidate_hdp)(struct amdgpu_device *adev, struct amdgpu_ring *ring); - void (*reset_hdp_ras_error_count)(struct amdgpu_device *adev); /* check if the asic needs a full reset of if soft reset will work */ bool (*need_full_reset)(struct amdgpu_device *adev); /* initialize doorbell layout for specific asic*/ @@ -891,6 +892,7 @@ struct amdgpu_device { /* For pre-DCE11. DCE11 and later are in "struct amdgpu_device->dm" */ struct work_struct hotplug_work; struct amdgpu_irq_src crtc_irq; + struct amdgpu_irq_src vline0_irq; struct amdgpu_irq_src vupdate_irq; struct amdgpu_irq_src pageflip_irq; struct amdgpu_irq_src hpd_irq; @@ -921,6 +923,9 @@ struct amdgpu_device { /* nbio */ struct amdgpu_nbio nbio; + /* hdp */ + struct amdgpu_hdp hdp; + /* smuio */ struct amdgpu_smuio smuio; @@ -1003,6 +1008,12 @@ struct amdgpu_device { bool in_suspend; bool in_hibernate; + /* + * The combination flag in_poweroff_reboot_com used to identify the poweroff + * and reboot opt in the s0i3 system-wide suspend. + */ + bool in_poweroff_reboot_com; + atomic_t in_gpu_reset; enum pp_mp1_state mp1_state; struct rw_semaphore reset_sem; @@ -1202,8 +1213,10 @@ int emu_soc_asic_init(struct amdgpu_device *adev); #define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l)) #define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v))) #define amdgpu_asic_get_config_memsize(adev) (adev)->asic_funcs->get_config_memsize((adev)) -#define amdgpu_asic_flush_hdp(adev, r) (adev)->asic_funcs->flush_hdp((adev), (r)) -#define amdgpu_asic_invalidate_hdp(adev, r) (adev)->asic_funcs->invalidate_hdp((adev), (r)) +#define amdgpu_asic_flush_hdp(adev, r) \ + ((adev)->asic_funcs->flush_hdp ? (adev)->asic_funcs->flush_hdp((adev), (r)) : (adev)->hdp.funcs->flush_hdp((adev), (r))) +#define amdgpu_asic_invalidate_hdp(adev, r) \ + ((adev)->asic_funcs->invalidate_hdp ? (adev)->asic_funcs->invalidate_hdp((adev), (r)) : (adev)->hdp.funcs->invalidate_hdp((adev), (r))) #define amdgpu_asic_need_full_reset(adev) (adev)->asic_funcs->need_full_reset((adev)) #define amdgpu_asic_init_doorbell_index(adev) (adev)->asic_funcs->init_doorbell_index((adev)) #define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1))) @@ -1222,6 +1235,7 @@ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev); int amdgpu_device_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job* job); void amdgpu_device_pci_config_reset(struct amdgpu_device *adev); +int amdgpu_device_pci_reset(struct amdgpu_device *adev); bool amdgpu_device_need_post(struct amdgpu_device *adev); void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index db96d69eb45e..c5343a5eecbe 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -47,12 +47,8 @@ int amdgpu_amdkfd_init(void) amdgpu_amdkfd_total_mem_size = si.totalram - si.totalhigh; amdgpu_amdkfd_total_mem_size *= si.mem_unit; -#ifdef CONFIG_HSA_AMD ret = kgd2kfd_init(); amdgpu_amdkfd_gpuvm_init_mem_limits(); -#else - ret = -ENOENT; -#endif kfd_initialized = !ret; return ret; @@ -696,86 +692,3 @@ bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd) return adev->have_atomics_support; } - -#ifndef CONFIG_HSA_AMD -bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) -{ - return false; -} - -void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) -{ -} - -int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) -{ - return 0; -} - -void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, - struct amdgpu_vm *vm) -{ -} - -struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f) -{ - return NULL; -} - -int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm) -{ - return 0; -} - -struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev, - unsigned int asic_type, bool vf) -{ - return NULL; -} - -bool kgd2kfd_device_init(struct kfd_dev *kfd, - struct drm_device *ddev, - const struct kgd2kfd_shared_resources *gpu_resources) -{ - return false; -} - -void kgd2kfd_device_exit(struct kfd_dev *kfd) -{ -} - -void kgd2kfd_exit(void) -{ -} - -void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm) -{ -} - -int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm) -{ - return 0; -} - -int kgd2kfd_pre_reset(struct kfd_dev *kfd) -{ - return 0; -} - -int kgd2kfd_post_reset(struct kfd_dev *kfd) -{ - return 0; -} - -void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) -{ -} - -void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) -{ -} - -void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask) -{ -} -#endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index ea391ca7f2f1..a81d9cacf9b8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -94,11 +94,6 @@ enum kgd_engine_type { KGD_ENGINE_MAX }; -struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, - struct mm_struct *mm); -bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm); -struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f); -int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo); struct amdkfd_process_info { /* List head of all VMs that belong to a KFD process */ @@ -132,8 +127,6 @@ void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev); void amdgpu_amdkfd_device_init(struct amdgpu_device *adev); void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev); - -int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm); int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, uint32_t vmid, uint64_t gpu_addr, uint32_t *ib_cmd, uint32_t ib_len); @@ -153,6 +146,38 @@ void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd); int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev, int queue_bit); +struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, + struct mm_struct *mm); +#if IS_ENABLED(CONFIG_HSA_AMD) +bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm); +struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f); +int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo); +int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm); +#else +static inline +bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) +{ + return false; +} + +static inline +struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f) +{ + return NULL; +} + +static inline +int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) +{ + return 0; +} + +static inline +int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm) +{ + return 0; +} +#endif /* Shared API */ int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size, void **mem_obj, uint64_t *gpu_addr, @@ -215,8 +240,6 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd, struct file *filp, u32 pasid, void **vm, void **process_info, struct dma_fence **ef); -void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, - struct amdgpu_vm *vm); void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm); void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm); uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm); @@ -236,23 +259,43 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd, struct kgd_mem *mem, void **kptr, uint64_t *size); int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info, struct dma_fence **ef); - int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd, struct kfd_vm_fault_info *info); - int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd, struct dma_buf *dmabuf, uint64_t va, void *vm, struct kgd_mem **mem, uint64_t *size, uint64_t *mmap_offset); - -void amdgpu_amdkfd_gpuvm_init_mem_limits(void); -void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo); - int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd, struct tile_config *config); +#if IS_ENABLED(CONFIG_HSA_AMD) +void amdgpu_amdkfd_gpuvm_init_mem_limits(void); +void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, + struct amdgpu_vm *vm); +void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo); +#else +static inline +void amdgpu_amdkfd_gpuvm_init_mem_limits(void) +{ +} +static inline +void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ +} + +static inline +void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) +{ +} +#endif /* KGD2KFD callbacks */ +int kgd2kfd_quiesce_mm(struct mm_struct *mm); +int kgd2kfd_resume_mm(struct mm_struct *mm); +int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, + struct dma_fence *fence); +#if IS_ENABLED(CONFIG_HSA_AMD) int kgd2kfd_init(void); void kgd2kfd_exit(void); struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev, @@ -266,11 +309,68 @@ int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm); int kgd2kfd_pre_reset(struct kfd_dev *kfd); int kgd2kfd_post_reset(struct kfd_dev *kfd); void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); -int kgd2kfd_quiesce_mm(struct mm_struct *mm); -int kgd2kfd_resume_mm(struct mm_struct *mm); -int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, - struct dma_fence *fence); void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd); void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask); +#else +static inline int kgd2kfd_init(void) +{ + return -ENOENT; +} +static inline void kgd2kfd_exit(void) +{ +} + +static inline +struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev, + unsigned int asic_type, bool vf) +{ + return NULL; +} + +static inline +bool kgd2kfd_device_init(struct kfd_dev *kfd, struct drm_device *ddev, + const struct kgd2kfd_shared_resources *gpu_resources) +{ + return false; +} + +static inline void kgd2kfd_device_exit(struct kfd_dev *kfd) +{ +} + +static inline void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm) +{ +} + +static inline int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm) +{ + return 0; +} + +static inline int kgd2kfd_pre_reset(struct kfd_dev *kfd) +{ + return 0; +} + +static inline int kgd2kfd_post_reset(struct kfd_dev *kfd) +{ + return 0; +} + +static inline +void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) +{ +} + +static inline +void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) +{ +} + +static inline +void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask) +{ +} +#endif #endif /* AMDGPU_AMDKFD_H_INCLUDED */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c index 4763bab7a4d0..62aa1a6f64ed 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c @@ -23,7 +23,6 @@ #include "amdgpu_amdkfd.h" #include "gc/gc_10_1_0_offset.h" #include "gc/gc_10_1_0_sh_mask.h" -#include "navi10_enum.h" #include "athub/athub_2_0_0_offset.h" #include "athub/athub_2_0_0_sh_mask.h" #include "oss/osssys_5_0_0_offset.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c index 50016bf9c427..fad3b91f74f5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c @@ -24,7 +24,6 @@ #include "amdgpu_amdkfd.h" #include "gc/gc_10_3_0_offset.h" #include "gc/gc_10_3_0_sh_mask.h" -#include "navi10_enum.h" #include "oss/osssys_5_0_0_offset.h" #include "oss/osssys_5_0_0_sh_mask.h" #include "soc15_common.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index d1ed4f8df2b7..ac0a432a9bf7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -454,7 +454,7 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, struct amdgpu_bo *bo = mem->bo; uint64_t va = mem->va; struct list_head *list_bo_va = &mem->bo_va_list; - unsigned long bo_size = bo->tbo.mem.size; + unsigned long bo_size = bo->tbo.base.size; if (!va) { pr_err("Invalid VA when adding BO to VM\n"); @@ -1277,7 +1277,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( struct kgd_dev *kgd, struct kgd_mem *mem, uint64_t *size) { struct amdkfd_process_info *process_info = mem->process_info; - unsigned long bo_size = mem->bo->tbo.mem.size; + unsigned long bo_size = mem->bo->tbo.base.size; struct kfd_bo_va_list *entry, *tmp; struct bo_vm_reservation_context ctx; struct ttm_validate_buffer *bo_list_entry; @@ -1398,7 +1398,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( mutex_lock(&mem->lock); domain = mem->domain; - bo_size = bo->tbo.mem.size; + bo_size = bo->tbo.base.size; pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n", mem->va, @@ -1502,7 +1502,7 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( struct amdgpu_device *adev = get_amdgpu_device(kgd); struct amdkfd_process_info *process_info = ((struct amdgpu_vm *)vm)->process_info; - unsigned long bo_size = mem->bo->tbo.mem.size; + unsigned long bo_size = mem->bo->tbo.base.size; struct kfd_bo_va_list *entry; struct bo_vm_reservation_context ctx; int ret; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c index 055f600eeed8..cfb1a9a04477 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c @@ -155,7 +155,7 @@ static bool amdgpu_read_bios_from_rom(struct amdgpu_device *adev) u8 header[AMD_VBIOS_SIGNATURE_END+1] = {0}; int len; - if (!adev->asic_funcs->read_bios_from_rom) + if (!adev->asic_funcs || !adev->asic_funcs->read_bios_from_rom) return false; /* validate VBIOS signature */ @@ -348,7 +348,8 @@ static bool amdgpu_read_disabled_bios(struct amdgpu_device *adev) if (adev->flags & AMD_IS_APU) return igp_read_bios_from_vram(adev); else - return amdgpu_asic_read_disabled_bios(adev); + return (!adev->asic_funcs || !adev->asic_funcs->read_disabled_bios) ? + false : amdgpu_asic_read_disabled_bios(adev); } #ifdef CONFIG_ACPI diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 594a0108e90f..3e240b952e79 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -98,8 +98,7 @@ static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p, return 0; error_free: - if (info) - kvfree(info); + kvfree(info); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index a6667a2ca0db..0a25fecf488a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -35,6 +35,7 @@ #include "amdgpu_dm_debugfs.h" #include "amdgpu_ras.h" #include "amdgpu_rap.h" +#include "amdgpu_securedisplay.h" #include "amdgpu_fw_attestation.h" /** @@ -1427,7 +1428,7 @@ static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched) struct dma_fence *fence; spin_lock(&sched->job_list_lock); - list_for_each_entry(s_job, &sched->ring_mirror_list, node) { + list_for_each_entry(s_job, &sched->pending_list, list) { fence = sched->ops->run_job(s_job); dma_fence_put(fence); } @@ -1459,10 +1460,10 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring) no_preempt: spin_lock(&sched->job_list_lock); - list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { + list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) { if (dma_fence_is_signaled(&s_job->s_fence->finished)) { /* remove job from ring_mirror_list */ - list_del_init(&s_job->node); + list_del_init(&s_job->list); sched->ops->free_job(s_job); continue; } @@ -1669,6 +1670,8 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev) amdgpu_rap_debugfs_init(adev); + amdgpu_securedisplay_debugfs_init(adev); + amdgpu_fw_attestation_debugfs_init(adev); return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index cab1ebaf6d62..6447cd6ca5a8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -929,6 +929,18 @@ void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); } +/** + * amdgpu_device_pci_reset - reset the GPU using generic PCI means + * + * @adev: amdgpu_device pointer + * + * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.). + */ +int amdgpu_device_pci_reset(struct amdgpu_device *adev) +{ + return pci_reset_function(adev->pdev); +} + /* * GPU doorbell aperture helpers function. */ @@ -1105,8 +1117,7 @@ void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) */ int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) { - u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size); - u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1; + int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); struct pci_bus *root; struct resource *res; unsigned i; @@ -1137,6 +1148,10 @@ int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) if (!res) return 0; + /* Limit the BAR size to what is available */ + rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1, + rbar_size); + /* Disable memory decoding while we change the BAR addresses and size */ pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); pci_write_config_word(adev->pdev, PCI_COMMAND, @@ -1422,24 +1437,22 @@ static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, /* don't suspend or resume card normally */ dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - pci_set_power_state(dev->pdev, PCI_D0); - amdgpu_device_load_pci_state(dev->pdev); - r = pci_enable_device(dev->pdev); + pci_set_power_state(pdev, PCI_D0); + amdgpu_device_load_pci_state(pdev); + r = pci_enable_device(pdev); if (r) DRM_WARN("pci_enable_device failed (%d)\n", r); amdgpu_device_resume(dev, true); dev->switch_power_state = DRM_SWITCH_POWER_ON; - drm_kms_helper_poll_enable(dev); } else { pr_info("switched off\n"); - drm_kms_helper_poll_disable(dev); dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; amdgpu_device_suspend(dev, true); - amdgpu_device_cache_pci_state(dev->pdev); + amdgpu_device_cache_pci_state(pdev); /* Shut down the device */ - pci_disable_device(dev->pdev); - pci_set_power_state(dev->pdev, PCI_D3cold); + pci_disable_device(pdev); + pci_set_power_state(pdev, PCI_D3cold); dev->switch_power_state = DRM_SWITCH_POWER_OFF; } } @@ -1702,8 +1715,7 @@ static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) adev->enable_virtual_display = false; if (amdgpu_virtual_display) { - struct drm_device *ddev = adev_to_drm(adev); - const char *pci_address_name = pci_name(ddev->pdev); + const char *pci_address_name = pci_name(adev->pdev); char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); @@ -2666,7 +2678,8 @@ static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) { int i, r; - if (!amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) { + if (adev->in_poweroff_reboot_com || + !amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) { amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); } @@ -3116,7 +3129,10 @@ static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) */ adev->gfx_timeout = msecs_to_jiffies(10000); adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; - if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev)) + if (amdgpu_sriov_vf(adev)) + adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ? + msecs_to_jiffies(60000) : msecs_to_jiffies(10000); + else if (amdgpu_passthrough(adev)) adev->compute_timeout = msecs_to_jiffies(60000); else adev->compute_timeout = MAX_SCHEDULE_TIMEOUT; @@ -3396,7 +3412,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, } } - pci_enable_pcie_error_reporting(adev->ddev.pdev); + pci_enable_pcie_error_reporting(adev->pdev); /* Post card if necessary */ if (amdgpu_device_need_post(adev)) { @@ -3719,14 +3735,15 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) r = amdgpu_device_ip_suspend_phase1(adev); - amdgpu_amdkfd_suspend(adev, !fbcon); + amdgpu_amdkfd_suspend(adev, adev->in_runpm); /* evict vram memory */ amdgpu_bo_evict_vram(adev); amdgpu_fence_driver_suspend(adev); - if (!amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) + if (adev->in_poweroff_reboot_com || + !amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) r = amdgpu_device_ip_suspend_phase2(adev); else amdgpu_gfx_state_change_set(adev, sGpuChangeState_D3Entry); @@ -3803,7 +3820,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) } } } - r = amdgpu_amdkfd_resume(adev, !fbcon); + r = amdgpu_amdkfd_resume(adev, adev->in_runpm); if (r) return r; @@ -4154,8 +4171,8 @@ bool amdgpu_device_has_job_running(struct amdgpu_device *adev) continue; spin_lock(&ring->sched.job_list_lock); - job = list_first_entry_or_null(&ring->sched.ring_mirror_list, - struct drm_sched_job, node); + job = list_first_entry_or_null(&ring->sched.pending_list, + struct drm_sched_job, list); spin_unlock(&ring->sched.job_list_lock); if (job) return true; @@ -4205,6 +4222,8 @@ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) case CHIP_NAVI14: case CHIP_NAVI12: case CHIP_SIENNA_CICHLID: + case CHIP_NAVY_FLOUNDER: + case CHIP_DIMGREY_CAVEFISH: break; default: goto disabled; @@ -4454,6 +4473,46 @@ static void amdgpu_device_unlock_adev(struct amdgpu_device *adev) up_write(&adev->reset_sem); } +/* + * to lockup a list of amdgpu devices in a hive safely, if not a hive + * with multiple nodes, it will be similar as amdgpu_device_lock_adev. + * + * unlock won't require roll back. + */ +static int amdgpu_device_lock_hive_adev(struct amdgpu_device *adev, struct amdgpu_hive_info *hive) +{ + struct amdgpu_device *tmp_adev = NULL; + + if (adev->gmc.xgmi.num_physical_nodes > 1) { + if (!hive) { + dev_err(adev->dev, "Hive is NULL while device has multiple xgmi nodes"); + return -ENODEV; + } + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { + if (!amdgpu_device_lock_adev(tmp_adev, hive)) + goto roll_back; + } + } else if (!amdgpu_device_lock_adev(adev, hive)) + return -EAGAIN; + + return 0; +roll_back: + if (!list_is_first(&tmp_adev->gmc.xgmi.head, &hive->device_list)) { + /* + * if the lockup iteration break in the middle of a hive, + * it may means there may has a race issue, + * or a hive device locked up independently. + * we may be in trouble and may not, so will try to roll back + * the lock and give out a warnning. + */ + dev_warn(tmp_adev->dev, "Hive lock iteration broke in the middle. Rolling back to unlock"); + list_for_each_entry_continue_reverse(tmp_adev, &hive->device_list, gmc.xgmi.head) { + amdgpu_device_unlock_adev(tmp_adev); + } + } + return -EAGAIN; +} + static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) { struct pci_dev *p = NULL; @@ -4567,11 +4626,29 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress", job ? job->base.id : -1, hive->hive_id); amdgpu_put_xgmi_hive(hive); + if (job) + drm_sched_increase_karma(&job->base); return 0; } mutex_lock(&hive->hive_lock); } + /* + * lock the device before we try to operate the linked list + * if didn't get the device lock, don't touch the linked list since + * others may iterating it. + */ + r = amdgpu_device_lock_hive_adev(adev, hive); + if (r) { + dev_info(adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress", + job ? job->base.id : -1); + + /* even we skipped this reset, still need to set the job to guilty */ + if (job) + drm_sched_increase_karma(&job->base); + goto skip_recovery; + } + /* * Build list of devices to reset. * In case we are in XGMI hive mode, resort the device list @@ -4579,8 +4656,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, */ INIT_LIST_HEAD(&device_list); if (adev->gmc.xgmi.num_physical_nodes > 1) { - if (!hive) - return -ENODEV; if (!list_is_first(&adev->gmc.xgmi.head, &hive->device_list)) list_rotate_to_front(&adev->gmc.xgmi.head, &hive->device_list); device_list_handle = &hive->device_list; @@ -4591,13 +4666,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, /* block all schedulers and reset given job's ring */ list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { - if (!amdgpu_device_lock_adev(tmp_adev, hive)) { - dev_info(tmp_adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress", - job ? job->base.id : -1); - r = 0; - goto skip_recovery; - } - /* * Try to put the audio codec into suspend state * before gpu reset started. @@ -4735,7 +4803,7 @@ skip_recovery: amdgpu_put_xgmi_hive(hive); } - if (r) + if (r && r != -EAGAIN) dev_info(adev->dev, "GPU reset end with ret = %d\n", r); return r; } @@ -4785,7 +4853,13 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); } else { - if (speed_cap == PCIE_SPEED_16_0GT) + if (speed_cap == PCIE_SPEED_32_0GT) + adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5); + else if (speed_cap == PCIE_SPEED_16_0GT) adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | @@ -4805,7 +4879,13 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); } else { - if (platform_speed_cap == PCIE_SPEED_16_0GT) + if (platform_speed_cap == PCIE_SPEED_32_0GT) + adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5); + else if (platform_speed_cap == PCIE_SPEED_16_0GT) adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | @@ -4949,8 +5029,8 @@ pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_sta case pci_channel_io_normal: return PCI_ERS_RESULT_CAN_RECOVER; /* Fatal error, prepare for slot reset */ - case pci_channel_io_frozen: - /* + case pci_channel_io_frozen: + /* * Cancel and wait for all TDRs in progress if failing to * set adev->in_gpu_reset in amdgpu_device_lock_adev * @@ -5041,7 +5121,7 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) goto out; } - adev->in_pci_err_recovery = true; + adev->in_pci_err_recovery = true; r = amdgpu_device_pre_asic_reset(adev, NULL, &need_full_reset); adev->in_pci_err_recovery = false; if (r) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index e42175e1acf1..47e0b48dc26f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -40,6 +40,7 @@ #include #include #include +#include /** * amdgpu_gem_prime_mmap - &drm_driver.gem_prime_mmap implementation @@ -151,9 +152,13 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf, if (attach->dev->driver == adev->dev->driver) return 0; + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); + if (r < 0) + goto out; + r = amdgpu_bo_reserve(bo, false); if (unlikely(r != 0)) - return r; + goto out; /* * We only create shared fences for internal use, but importers @@ -165,11 +170,15 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf, */ r = __dma_resv_make_exclusive(bo->tbo.base.resv); if (r) - return r; + goto out; bo->prime_shared_count++; amdgpu_bo_unreserve(bo); return 0; + +out: + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); + return r; } /** @@ -189,6 +198,9 @@ static void amdgpu_dma_buf_detach(struct dma_buf *dmabuf, if (attach->dev->driver != adev->dev->driver && bo->prime_shared_count) bo->prime_shared_count--; + + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); } /** @@ -269,7 +281,7 @@ static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach, case TTM_PL_TT: sgt = drm_prime_pages_to_sg(obj->dev, bo->tbo.ttm->pages, - bo->tbo.num_pages); + bo->tbo.ttm->num_pages); if (IS_ERR(sgt)) return sgt; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 7169fb5e3d9c..4575192d9b08 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -132,8 +132,12 @@ uint amdgpu_pg_mask = 0xffffffff; uint amdgpu_sdma_phase_quantum = 32; char *amdgpu_disable_cu = NULL; char *amdgpu_virtual_display = NULL; -/* OverDrive(bit 14) disabled by default*/ -uint amdgpu_pp_feature_mask = 0xffffbfff; + +/* + * OverDrive(bit 14) disabled by default + * GFX DCS(bit 19) disabled by default + */ +uint amdgpu_pp_feature_mask = 0xfff7bfff; uint amdgpu_force_long_training; int amdgpu_job_hang_limit; int amdgpu_lbpw = -1; @@ -789,9 +793,9 @@ module_param_named(tmz, amdgpu_tmz, int, 0444); /** * DOC: reset_method (int) - * GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco) + * GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco, 5 = pci) */ -MODULE_PARM_DESC(reset_method, "GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco/bamaco)"); +MODULE_PARM_DESC(reset_method, "GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco/bamaco, 5 = pci)"); module_param_named(reset_method, amdgpu_reset_method, int, 0444); /** @@ -1094,6 +1098,7 @@ static const struct pci_device_id pciidlist[] = { /* Sienna_Cichlid */ {0x1002, 0x73A0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, + {0x1002, 0x73A1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73A2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73A3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73AB, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, @@ -1206,7 +1211,6 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, if (ret) return ret; - ddev->pdev = pdev; pci_set_drvdata(pdev, ddev); ret = amdgpu_driver_load_kms(adev, ent->driver_data); @@ -1266,7 +1270,9 @@ amdgpu_pci_shutdown(struct pci_dev *pdev) */ if (!amdgpu_passthrough(adev)) adev->mp1_state = PP_MP1_STATE_UNLOAD; + adev->in_poweroff_reboot_com = true; amdgpu_device_ip_suspend(adev); + adev->in_poweroff_reboot_com = false; adev->mp1_state = PP_MP1_STATE_NONE; } @@ -1308,8 +1314,13 @@ static int amdgpu_pmops_thaw(struct device *dev) static int amdgpu_pmops_poweroff(struct device *dev) { struct drm_device *drm_dev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(drm_dev); + int r; - return amdgpu_device_suspend(drm_dev, true); + adev->in_poweroff_reboot_com = true; + r = amdgpu_device_suspend(drm_dev, true); + adev->in_poweroff_reboot_com = false; + return r; } static int amdgpu_pmops_restore(struct device *dev) @@ -1344,11 +1355,12 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev) adev->in_runpm = true; if (amdgpu_device_supports_atpx(drm_dev)) drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - drm_kms_helper_poll_disable(drm_dev); ret = amdgpu_device_suspend(drm_dev, false); - if (ret) + if (ret) { + adev->in_runpm = false; return ret; + } if (amdgpu_device_supports_atpx(drm_dev)) { /* Only need to handle PCI state in the driver for ATPX @@ -1401,7 +1413,6 @@ static int amdgpu_pmops_runtime_resume(struct device *dev) amdgpu_device_baco_exit(drm_dev); } ret = amdgpu_device_resume(drm_dev, false); - drm_kms_helper_poll_enable(drm_dev); if (amdgpu_device_supports_atpx(drm_dev)) drm_dev->switch_power_state = DRM_SWITCH_POWER_ON; adev->in_runpm = false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c index 0bf7d36c6686..51cd49c6f38f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c @@ -271,7 +271,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper, DRM_INFO("fb depth is %d\n", fb->format->depth); DRM_INFO(" pitch is %d\n", fb->pitches[0]); - vga_switcheroo_client_fb_set(adev_to_drm(adev)->pdev, info); + vga_switcheroo_client_fb_set(adev->pdev, info); return 0; out: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fw_attestation.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fw_attestation.c index 7c6e02e35573..8d1ad294cb02 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fw_attestation.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fw_attestation.c @@ -47,10 +47,9 @@ typedef struct FW_ATT_RECORD uint16_t AttFwIdV2; /* V2 FW ID field */ uint32_t AttFWVersion; /* FW Version */ uint16_t AttFWActiveFunctionID; /* The VF ID (only in VF Attestation Table) */ - uint16_t AttSource; /* FW source indicator */ - uint16_t RecordValid; /* Indicates whether the record is a valid entry */ - uint8_t AttFwTaId; /* Ta ID (only in TA Attestation Table) */ - uint8_t Reserved; + uint8_t AttSource; /* FW source indicator */ + uint8_t RecordValid; /* Indicates whether the record is a valid entry */ + uint32_t AttFwTaId; /* Ta ID (only in TA Attestation Table) */ } FW_ATT_RECORD; static ssize_t amdgpu_fw_attestation_debugfs_read(struct file *f, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 174a73eb23f0..b443907afcea 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -619,7 +619,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, int r = 0; if (args->va_address < AMDGPU_VA_RESERVED_SIZE) { - dev_dbg(&dev->pdev->dev, + dev_dbg(dev->dev, "va_address 0x%LX is in reserved area 0x%LX\n", args->va_address, AMDGPU_VA_RESERVED_SIZE); return -EINVAL; @@ -627,7 +627,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, if (args->va_address >= AMDGPU_GMC_HOLE_START && args->va_address < AMDGPU_GMC_HOLE_END) { - dev_dbg(&dev->pdev->dev, + dev_dbg(dev->dev, "va_address 0x%LX is in VA hole 0x%LX-0x%LX\n", args->va_address, AMDGPU_GMC_HOLE_START, AMDGPU_GMC_HOLE_END); @@ -639,14 +639,14 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE; vm_size -= AMDGPU_VA_RESERVED_SIZE; if (args->va_address + args->map_size > vm_size) { - dev_dbg(&dev->pdev->dev, + dev_dbg(dev->dev, "va_address 0x%llx is in top reserved area 0x%llx\n", args->va_address + args->map_size, vm_size); return -EINVAL; } if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) { - dev_dbg(&dev->pdev->dev, "invalid flags combination 0x%08X\n", + dev_dbg(dev->dev, "invalid flags combination 0x%08X\n", args->flags); return -EINVAL; } @@ -658,7 +658,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, case AMDGPU_VA_OP_REPLACE: break; default: - dev_dbg(&dev->pdev->dev, "unsupported operation %d\n", + dev_dbg(dev->dev, "unsupported operation %d\n", args->operation); return -EINVAL; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index cd2c676a2797..8e0a6c62322e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -193,15 +193,16 @@ static bool amdgpu_gfx_is_multipipe_capable(struct amdgpu_device *adev) } bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev, - int pipe, int queue) + struct amdgpu_ring *ring) { - bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev); - int cond; - /* Policy: alternate between normal and high priority */ - cond = multipipe_policy ? pipe : queue; - - return ((cond % 2) != 0); + /* Policy: use 1st queue as high priority compute queue if we + * have more than one compute queue. + */ + if (adev->gfx.num_compute_rings > 1 && + ring == &adev->gfx.compute_ring[0]) + return true; + return false; } void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index 6b5a8f4642cc..72dbcd2bc6a6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -380,7 +380,7 @@ void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit, bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev, int mec, int pipe, int queue); bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev, - int pipe, int queue); + struct amdgpu_ring *ring); int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev, int me, int pipe, int queue); void amdgpu_gfx_bit_to_me_queue(struct amdgpu_device *adev, int bit, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 6e679db5e46f..fe1a39ffda72 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -120,7 +120,7 @@ uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); - if (bo->num_pages != 1 || bo->ttm->caching == ttm_cached) + if (bo->ttm->num_pages != 1 || bo->ttm->caching == ttm_cached) return AMDGPU_BO_INVALID_OFFSET; if (bo->ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h new file mode 100644 index 000000000000..43caf9f8cc11 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_HDP_H__ +#define __AMDGPU_HDP_H__ + +struct amdgpu_hdp_funcs { + void (*flush_hdp)(struct amdgpu_device *adev, struct amdgpu_ring *ring); + void (*invalidate_hdp)(struct amdgpu_device *adev, + struct amdgpu_ring *ring); + void (*reset_ras_error_count)(struct amdgpu_device *adev); + void (*update_clock_gating)(struct amdgpu_device *adev, bool enable); + void (*get_clock_gating_state)(struct amdgpu_device *adev, u32 *flags); + void (*init_registers)(struct amdgpu_device *adev); +}; + +struct amdgpu_hdp { + const struct amdgpu_hdp_funcs *funcs; +}; + +#endif /* __AMDGPU_HDP_H__ */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c index 47cad23a6b9e..bca4dddd5a15 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c @@ -176,7 +176,7 @@ struct amdgpu_i2c_chan *amdgpu_i2c_create(struct drm_device *dev, i2c->rec = *rec; i2c->adapter.owner = THIS_MODULE; i2c->adapter.class = I2C_CLASS_DDC; - i2c->adapter.dev.parent = &dev->pdev->dev; + i2c->adapter.dev.parent = dev->dev; i2c->dev = dev; i2c_set_adapdata(&i2c->adapter, i2c); mutex_init(&i2c->mutex); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 024d0a563a65..7645223ea0ef 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -195,6 +195,10 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, if ((ib->flags & AMDGPU_IB_FLAG_EMIT_MEM_SYNC) && ring->funcs->emit_mem_sync) ring->funcs->emit_mem_sync(ring); + if (ring->funcs->emit_wave_limit && + ring->hw_prio == AMDGPU_GFX_PIPE_PRIO_HIGH) + ring->funcs->emit_wave_limit(ring, true); + if (ring->funcs->insert_start) ring->funcs->insert_start(ring); @@ -295,6 +299,11 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, ring->current_ctx = fence_ctx; if (vm && ring->funcs->emit_switch_buffer) amdgpu_ring_emit_switch_buffer(ring); + + if (ring->funcs->emit_wave_limit && + ring->hw_prio == AMDGPU_GFX_PIPE_PRIO_HIGH) + ring->funcs->emit_wave_limit(ring, false); + amdgpu_ring_commit(ring); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c index dcd9b4a8e20b..dc852af4f3b7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c @@ -205,3 +205,48 @@ restart_ih: return IRQ_HANDLED; } +/** + * amdgpu_ih_decode_iv_helper - decode an interrupt vector + * + * @adev: amdgpu_device pointer + * @ih: ih ring to process + * @entry: IV entry + * + * Decodes the interrupt vector at the current rptr + * position and also advance the position for for Vega10 + * and later GPUs. + */ +void amdgpu_ih_decode_iv_helper(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + struct amdgpu_iv_entry *entry) +{ + /* wptr/rptr are in bytes! */ + u32 ring_index = ih->rptr >> 2; + uint32_t dw[8]; + + dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); + dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); + dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); + dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); + dw[4] = le32_to_cpu(ih->ring[ring_index + 4]); + dw[5] = le32_to_cpu(ih->ring[ring_index + 5]); + dw[6] = le32_to_cpu(ih->ring[ring_index + 6]); + dw[7] = le32_to_cpu(ih->ring[ring_index + 7]); + + entry->client_id = dw[0] & 0xff; + entry->src_id = (dw[0] >> 8) & 0xff; + entry->ring_id = (dw[0] >> 16) & 0xff; + entry->vmid = (dw[0] >> 24) & 0xf; + entry->vmid_src = (dw[0] >> 31); + entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32); + entry->timestamp_src = dw[2] >> 31; + entry->pasid = dw[3] & 0xffff; + entry->pasid_src = dw[3] >> 31; + entry->src_data[0] = dw[4]; + entry->src_data[1] = dw[5]; + entry->src_data[2] = dw[6]; + entry->src_data[3] = dw[7]; + + /* wptr/rptr are in bytes! */ + ih->rptr += 32; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h index 3c9cfe7eecff..6ed4a85fc7c3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h @@ -30,6 +30,18 @@ struct amdgpu_device; struct amdgpu_iv_entry; +struct amdgpu_ih_regs { + uint32_t ih_rb_base; + uint32_t ih_rb_base_hi; + uint32_t ih_rb_cntl; + uint32_t ih_rb_wptr; + uint32_t ih_rb_rptr; + uint32_t ih_doorbell_rptr; + uint32_t ih_rb_wptr_addr_lo; + uint32_t ih_rb_wptr_addr_hi; + uint32_t psp_reg_id; +}; + /* * R6xx+ IH ring */ @@ -53,6 +65,7 @@ struct amdgpu_ih_ring { bool enabled; unsigned rptr; atomic_t lock; + struct amdgpu_ih_regs ih_regs; }; /* provided by the ih block */ @@ -75,5 +88,7 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); void amdgpu_ih_ring_write(struct amdgpu_ih_ring *ih, const uint32_t *iv, unsigned int num_dw); int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); - +void amdgpu_ih_decode_iv_helper(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + struct amdgpu_iv_entry *entry); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index bea57e8e793f..afbbec82a289 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -444,7 +444,8 @@ void amdgpu_irq_dispatch(struct amdgpu_device *adev, } else if (src_id >= AMDGPU_MAX_IRQ_SRC_ID) { DRM_DEBUG("Invalid src_id in IV: %d\n", src_id); - } else if (adev->irq.virq[src_id]) { + } else if ((client_id == AMDGPU_IRQ_CLIENTID_LEGACY) && + adev->irq.virq[src_id]) { generic_handle_irq(irq_find_mapping(adev->irq.domain, src_id)); } else if (!adev->irq.client[client_id].sources) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index dcfe8a3b03ff..ff48101bab55 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -271,7 +271,7 @@ void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched) } /* Signal all jobs already scheduled to HW */ - list_for_each_entry(s_job, &sched->ring_mirror_list, node) { + list_for_each_entry(s_job, &sched->pending_list, list) { struct drm_sched_fence *s_fence = s_job->s_fence; dma_fence_set_error(&s_fence->finished, -EHWPOISON); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index b16b32797624..3c37cf1ae8b7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -142,7 +142,7 @@ int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags) (amdgpu_is_atpx_hybrid() || amdgpu_has_atpx_dgpu_power_cntl()) && ((flags & AMD_IS_APU) == 0) && - !pci_is_thunderbolt_attached(dev->pdev)) + !pci_is_thunderbolt_attached(to_pci_dev(dev->dev))) flags |= AMD_IS_PX; parent = pci_upstream_bridge(adev->pdev); @@ -156,7 +156,7 @@ int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags) */ r = amdgpu_device_init(adev, flags); if (r) { - dev_err(&dev->pdev->dev, "Fatal error during GPU init\n"); + dev_err(dev->dev, "Fatal error during GPU init\n"); goto out; } @@ -199,7 +199,7 @@ int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags) acpi_status = amdgpu_acpi_init(adev); if (acpi_status) - dev_dbg(&dev->pdev->dev, "Error during ACPI methods call\n"); + dev_dbg(dev->dev, "Error during ACPI methods call\n"); if (adev->runpm) { /* only need to skip on ATPX */ @@ -735,10 +735,10 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) if (!dev_info) return -ENOMEM; - dev_info->device_id = dev->pdev->device; + dev_info->device_id = adev->pdev->device; dev_info->chip_rev = adev->rev_id; dev_info->external_rev = adev->external_rev_id; - dev_info->pci_rev = dev->pdev->revision; + dev_info->pci_rev = adev->pdev->revision; dev_info->family = adev->family; dev_info->num_shader_engines = adev->gfx.config.max_shader_engines; dev_info->num_shader_arrays_per_engine = adev->gfx.config.max_sh_per_se; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h index e62cc0e1a5ad..7c11bce4514b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h @@ -57,7 +57,6 @@ struct amdgpu_nbio_funcs { u32 (*get_pcie_port_data_offset)(struct amdgpu_device *adev); u32 (*get_rev_id)(struct amdgpu_device *adev); void (*mc_access_enable)(struct amdgpu_device *adev, bool enable); - void (*hdp_flush)(struct amdgpu_device *adev, struct amdgpu_ring *ring); u32 (*get_memsize)(struct amdgpu_device *adev); void (*sdma_doorbell_range)(struct amdgpu_device *adev, int instance, bool use_doorbell, int doorbell_index, int doorbell_size); @@ -89,6 +88,7 @@ struct amdgpu_nbio_funcs { int (*ras_late_init)(struct amdgpu_device *adev); void (*enable_aspm)(struct amdgpu_device *adev, bool enable); + void (*program_aspm)(struct amdgpu_device *adev); }; struct amdgpu_nbio { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index b4c8e5d5c763..4b29b8205442 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -787,7 +787,7 @@ int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) if (r < 0) return r; - r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap); + r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.mem.num_pages, &bo->kmap); if (r) return r; @@ -911,10 +911,16 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, if (bo->tbo.pin_count) { uint32_t mem_type = bo->tbo.mem.mem_type; + uint32_t mem_flags = bo->tbo.mem.placement; if (!(domain & amdgpu_mem_type_to_domain(mem_type))) return -EINVAL; + if ((mem_type == TTM_PL_VRAM) && + (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && + !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) + return -EINVAL; + ttm_bo_pin(&bo->tbo); if (max_offset != 0) { @@ -930,7 +936,6 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, if (bo->tbo.base.import_attach) dma_buf_pin(bo->tbo.base.import_attach); - bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; /* force to pin into visible video ram */ if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; @@ -983,6 +988,7 @@ error: */ int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) { + bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; return amdgpu_bo_pin_restricted(bo, domain, 0, 0); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 79120ec41396..9ac37569823f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -174,12 +174,12 @@ static inline void amdgpu_bo_unreserve(struct amdgpu_bo *bo) static inline unsigned long amdgpu_bo_size(struct amdgpu_bo *bo) { - return bo->tbo.num_pages << PAGE_SHIFT; + return bo->tbo.base.size; } static inline unsigned amdgpu_bo_ngpu_pages(struct amdgpu_bo *bo) { - return (bo->tbo.num_pages << PAGE_SHIFT) / AMDGPU_GPU_PAGE_SIZE; + return bo->tbo.base.size / AMDGPU_GPU_PAGE_SIZE; } static inline unsigned amdgpu_bo_gpu_page_alignment(struct amdgpu_bo *bo) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 347fec669424..839917eb7bc3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -36,6 +36,7 @@ #include "psp_v12_0.h" #include "amdgpu_ras.h" +#include "amdgpu_securedisplay.h" static int psp_sysfs_init(struct amdgpu_device *adev); static void psp_sysfs_fini(struct amdgpu_device *adev); @@ -249,7 +250,7 @@ psp_cmd_submit_buf(struct psp_context *psp, { int ret; int index; - int timeout = 2000; + int timeout = 20000; bool ras_intr = false; bool skip_unsupport = false; @@ -282,7 +283,7 @@ psp_cmd_submit_buf(struct psp_context *psp, ras_intr = amdgpu_ras_intr_triggered(); if (ras_intr) break; - msleep(1); + usleep_range(10, 100); amdgpu_asic_invalidate_hdp(psp->adev, NULL); } @@ -1652,6 +1653,175 @@ int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id) } // RAP end +/* securedisplay start */ +static int psp_securedisplay_init_shared_buf(struct psp_context *psp) +{ + int ret; + + /* + * Allocate 16k memory aligned to 4k from Frame Buffer (local + * physical) for sa ta <-> Driver + */ + ret = amdgpu_bo_create_kernel(psp->adev, PSP_SECUREDISPLAY_SHARED_MEM_SIZE, + PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, + &psp->securedisplay_context.securedisplay_shared_bo, + &psp->securedisplay_context.securedisplay_shared_mc_addr, + &psp->securedisplay_context.securedisplay_shared_buf); + + return ret; +} + +static int psp_securedisplay_load(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + memset(psp->fw_pri_buf, 0, PSP_1_MEG); + memcpy(psp->fw_pri_buf, psp->ta_securedisplay_start_addr, psp->ta_securedisplay_ucode_size); + + psp_prep_ta_load_cmd_buf(cmd, + psp->fw_pri_mc_addr, + psp->ta_securedisplay_ucode_size, + psp->securedisplay_context.securedisplay_shared_mc_addr, + PSP_SECUREDISPLAY_SHARED_MEM_SIZE); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + + if (ret) + goto failed; + + psp->securedisplay_context.securedisplay_initialized = true; + psp->securedisplay_context.session_id = cmd->resp.session_id; + mutex_init(&psp->securedisplay_context.mutex); + +failed: + kfree(cmd); + return ret; +} + +static int psp_securedisplay_unload(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + psp_prep_ta_unload_cmd_buf(cmd, psp->securedisplay_context.session_id); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + + kfree(cmd); + + return ret; +} + +static int psp_securedisplay_initialize(struct psp_context *psp) +{ + int ret; + struct securedisplay_cmd *securedisplay_cmd; + + /* + * TODO: bypass the initialize in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + if (!psp->adev->psp.ta_securedisplay_ucode_size || + !psp->adev->psp.ta_securedisplay_start_addr) { + dev_info(psp->adev->dev, "SECUREDISPLAY: securedisplay ta ucode is not available\n"); + return 0; + } + + if (!psp->securedisplay_context.securedisplay_initialized) { + ret = psp_securedisplay_init_shared_buf(psp); + if (ret) + return ret; + } + + ret = psp_securedisplay_load(psp); + if (ret) + return ret; + + psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, + TA_SECUREDISPLAY_COMMAND__QUERY_TA); + + ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA); + if (ret) { + psp_securedisplay_unload(psp); + + amdgpu_bo_free_kernel(&psp->securedisplay_context.securedisplay_shared_bo, + &psp->securedisplay_context.securedisplay_shared_mc_addr, + &psp->securedisplay_context.securedisplay_shared_buf); + + psp->securedisplay_context.securedisplay_initialized = false; + + dev_err(psp->adev->dev, "SECUREDISPLAY TA initialize fail.\n"); + return -EINVAL; + } + + if (securedisplay_cmd->status != TA_SECUREDISPLAY_STATUS__SUCCESS) { + psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status); + dev_err(psp->adev->dev, "SECUREDISPLAY: query securedisplay TA failed. ret 0x%x\n", + securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret); + } + + return 0; +} + +static int psp_securedisplay_terminate(struct psp_context *psp) +{ + int ret; + + /* + * TODO:bypass the terminate in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + if (!psp->securedisplay_context.securedisplay_initialized) + return 0; + + ret = psp_securedisplay_unload(psp); + if (ret) + return ret; + + psp->securedisplay_context.securedisplay_initialized = false; + + /* free securedisplay shared memory */ + amdgpu_bo_free_kernel(&psp->securedisplay_context.securedisplay_shared_bo, + &psp->securedisplay_context.securedisplay_shared_mc_addr, + &psp->securedisplay_context.securedisplay_shared_buf); + + return ret; +} + +int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id) +{ + int ret; + + if (!psp->securedisplay_context.securedisplay_initialized) + return -EINVAL; + + if (ta_cmd_id != TA_SECUREDISPLAY_COMMAND__QUERY_TA && + ta_cmd_id != TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC) + return -EINVAL; + + mutex_lock(&psp->securedisplay_context.mutex); + + ret = psp_ta_invoke(psp, ta_cmd_id, psp->securedisplay_context.session_id); + + mutex_unlock(&psp->securedisplay_context.mutex); + + return ret; +} +/* SECUREDISPLAY end */ + static int psp_hw_start(struct psp_context *psp) { struct amdgpu_device *adev = psp->adev; @@ -2126,6 +2296,11 @@ skip_memalloc: if (ret) dev_err(psp->adev->dev, "RAP: Failed to initialize RAP\n"); + + ret = psp_securedisplay_initialize(psp); + if (ret) + dev_err(psp->adev->dev, + "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n"); } return 0; @@ -2176,6 +2351,7 @@ static int psp_hw_fini(void *handle) if (psp->adev->psp.ta_fw) { psp_ras_terminate(psp); + psp_securedisplay_terminate(psp); psp_rap_terminate(psp); psp_dtm_terminate(psp); psp_hdcp_terminate(psp); @@ -2240,6 +2416,11 @@ static int psp_suspend(void *handle) DRM_ERROR("Failed to terminate rap ta\n"); return ret; } + ret = psp_securedisplay_terminate(psp); + if (ret) { + DRM_ERROR("Failed to terminate securedisplay ta\n"); + return ret; + } } ret = psp_asd_unload(psp); @@ -2323,6 +2504,11 @@ static int psp_resume(void *handle) if (ret) dev_err(psp->adev->dev, "RAP: Failed to initialize RAP\n"); + + ret = psp_securedisplay_initialize(psp); + if (ret) + dev_err(psp->adev->dev, + "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n"); } mutex_unlock(&adev->firmware.mutex); @@ -2629,6 +2815,11 @@ static int parse_ta_bin_descriptor(struct psp_context *psp, psp->ta_rap_ucode_size = le32_to_cpu(desc->size_bytes); psp->ta_rap_start_addr = ucode_start_addr; break; + case TA_FW_TYPE_PSP_SECUREDISPLAY: + psp->ta_securedisplay_ucode_version = le32_to_cpu(desc->fw_version); + psp->ta_securedisplay_ucode_size = le32_to_cpu(desc->size_bytes); + psp->ta_securedisplay_start_addr = ucode_start_addr; + break; default: dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type); break; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index da250bc1ac57..cb50ba445f8c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -30,6 +30,7 @@ #include "ta_xgmi_if.h" #include "ta_ras_if.h" #include "ta_rap_if.h" +#include "ta_secureDisplay_if.h" #define PSP_FENCE_BUFFER_SIZE 0x1000 #define PSP_CMD_BUFFER_SIZE 0x1000 @@ -40,6 +41,7 @@ #define PSP_HDCP_SHARED_MEM_SIZE 0x4000 #define PSP_DTM_SHARED_MEM_SIZE 0x4000 #define PSP_RAP_SHARED_MEM_SIZE 0x4000 +#define PSP_SECUREDISPLAY_SHARED_MEM_SIZE 0x4000 #define PSP_SHARED_MEM_SIZE 0x4000 #define PSP_FW_NAME_LEN 0x24 @@ -171,6 +173,15 @@ struct psp_rap_context { struct mutex mutex; }; +struct psp_securedisplay_context { + bool securedisplay_initialized; + uint32_t session_id; + struct amdgpu_bo *securedisplay_shared_bo; + uint64_t securedisplay_shared_mc_addr; + void *securedisplay_shared_buf; + struct mutex mutex; +}; + #define MEM_TRAIN_SYSTEM_SIGNATURE 0x54534942 #define GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES 0x1000 #define GDDR6_MEM_TRAINING_OFFSET 0x8000 @@ -298,12 +309,17 @@ struct psp_context uint32_t ta_rap_ucode_size; uint8_t *ta_rap_start_addr; + uint32_t ta_securedisplay_ucode_version; + uint32_t ta_securedisplay_ucode_size; + uint8_t *ta_securedisplay_start_addr; + struct psp_asd_context asd_context; struct psp_xgmi_context xgmi_context; struct psp_ras_context ras; struct psp_hdcp_context hdcp_context; struct psp_dtm_context dtm_context; struct psp_rap_context rap_context; + struct psp_securedisplay_context securedisplay_context; struct mutex mutex; struct psp_memory_training_context mem_train_ctx; }; @@ -380,6 +396,7 @@ int psp_ras_trigger_error(struct psp_context *psp, int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id); int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id); int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id); +int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id); int psp_rlc_autoload_start(struct psp_context *psp); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 82e952696d24..1fb2a91ad30a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -846,7 +846,7 @@ static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev, if (amdgpu_dpm_allow_xgmi_power_down(adev, true)) dev_warn(adev->dev, "Failed to allow XGMI power down"); - if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW)) + if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW)) dev_warn(adev->dev, "Failed to allow df cstate"); return ret; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index 1a612f51ecd9..b644c78475fd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -166,7 +166,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int max_dw, struct amdgpu_irq_src *irq_src, unsigned int irq_type, unsigned int hw_prio) { - int r, i; + int r; int sched_hw_submission = amdgpu_sched_hw_submission; u32 *num_sched; u32 hw_ip; @@ -258,8 +258,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, } ring->max_dw = max_dw; - ring->priority = DRM_SCHED_PRIORITY_NORMAL; - mutex_init(&ring->priority_mutex); + ring->hw_prio = hw_prio; if (!ring->no_scheduler) { hw_ip = ring->funcs->type; @@ -268,9 +267,6 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, &ring->sched; } - for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_COUNT; ++i) - atomic_set(&ring->num_jobs[i], 0); - return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index 7112137689db..56acec1075ac 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -197,6 +197,7 @@ struct amdgpu_ring_funcs { void (*soft_recovery)(struct amdgpu_ring *ring, unsigned vmid); int (*preempt_ib)(struct amdgpu_ring *ring); void (*emit_mem_sync)(struct amdgpu_ring *ring); + void (*emit_wave_limit)(struct amdgpu_ring *ring, bool enable); }; struct amdgpu_ring { @@ -242,11 +243,7 @@ struct amdgpu_ring { struct dma_fence *vmid_wait; bool has_compute_vm_bug; bool no_scheduler; - - atomic_t num_jobs[DRM_SCHED_PRIORITY_COUNT]; - struct mutex priority_mutex; - /* protected by priority_mutex */ - int priority; + int hw_prio; #if defined(CONFIG_DEBUG_FS) struct dentry *ent; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c new file mode 100644 index 000000000000..834440ab9ff7 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c @@ -0,0 +1,176 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include +#include + +#include "amdgpu.h" +#include "amdgpu_securedisplay.h" + +/** + * DOC: AMDGPU SECUREDISPLAY debugfs test interface + * + * how to use? + * echo opcode > /dri/xxx/securedisplay_test + * eg. echo 1 > /dri/xxx/securedisplay_test + * eg. echo 2 phy_id > /dri/xxx/securedisplay_test + * + * opcode: + * 1：Query whether TA is responding used only for validation pupose + * 2: Send region of Interest and CRC value to I2C. (uint32)phy_id is + * send to determine which DIO scratch register should be used to get + * ROI and receive i2c_buf as the output. + * + * You can refer more detail from header file ta_securedisplay_if.h + * + */ + +void psp_securedisplay_parse_resp_status(struct psp_context *psp, + enum ta_securedisplay_status status) +{ + switch (status) { + case TA_SECUREDISPLAY_STATUS__SUCCESS: + break; + case TA_SECUREDISPLAY_STATUS__GENERIC_FAILURE: + dev_err(psp->adev->dev, "Secure display: Generic Failure."); + break; + case TA_SECUREDISPLAY_STATUS__INVALID_PARAMETER: + dev_err(psp->adev->dev, "Secure display: Invalid Parameter."); + break; + case TA_SECUREDISPLAY_STATUS__NULL_POINTER: + dev_err(psp->adev->dev, "Secure display: Null Pointer."); + break; + case TA_SECUREDISPLAY_STATUS__I2C_WRITE_ERROR: + dev_err(psp->adev->dev, "Secure display: Failed to write to I2C."); + break; + case TA_SECUREDISPLAY_STATUS__READ_DIO_SCRATCH_ERROR: + dev_err(psp->adev->dev, "Secure display: Failed to Read DIO Scratch Register."); + break; + case TA_SECUREDISPLAY_STATUS__READ_CRC_ERROR: + dev_err(psp->adev->dev, "Secure display: Failed to Read CRC"); + break; + default: + dev_err(psp->adev->dev, "Secure display: Failed to parse status: %d\n", status); + } +} + +void psp_prep_securedisplay_cmd_buf(struct psp_context *psp, struct securedisplay_cmd **cmd, + enum ta_securedisplay_command command_id) +{ + *cmd = (struct securedisplay_cmd *)psp->securedisplay_context.securedisplay_shared_buf; + memset(*cmd, 0, sizeof(struct securedisplay_cmd)); + (*cmd)->status = TA_SECUREDISPLAY_STATUS__GENERIC_FAILURE; + (*cmd)->cmd_id = command_id; +} + +static ssize_t amdgpu_securedisplay_debugfs_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; + struct psp_context *psp = &adev->psp; + struct securedisplay_cmd *securedisplay_cmd; + struct drm_device *dev = adev_to_drm(adev); + uint32_t phy_id; + uint32_t op; + int i; + char str[64]; + char i2c_output[256]; + int ret; + + if (*pos || size > sizeof(str) - 1) + return -EINVAL; + + memset(str, 0, sizeof(str)); + ret = copy_from_user(str, buf, size); + if (ret) + return -EFAULT; + + ret = pm_runtime_get_sync(dev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(dev->dev); + return ret; + } + + if (size < 3) + sscanf(str, "%u ", &op); + else + sscanf(str, "%u %u", &op, &phy_id); + + switch (op) { + case 1: + psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, + TA_SECUREDISPLAY_COMMAND__QUERY_TA); + ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA); + if (!ret) { + if (securedisplay_cmd->status == TA_SECUREDISPLAY_STATUS__SUCCESS) + dev_info(adev->dev, "SECUREDISPLAY: query securedisplay TA ret is 0x%X\n", + securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret); + else + psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status); + } + break; + case 2: + psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, + TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC); + securedisplay_cmd->securedisplay_in_message.send_roi_crc.phy_id = phy_id; + ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC); + if (!ret) { + if (securedisplay_cmd->status == TA_SECUREDISPLAY_STATUS__SUCCESS) { + memset(i2c_output, 0, sizeof(i2c_output)); + for (i = 0; i < TA_SECUREDISPLAY_I2C_BUFFER_SIZE; i++) + sprintf(i2c_output, "%s 0x%X", i2c_output, + securedisplay_cmd->securedisplay_out_message.send_roi_crc.i2c_buf[i]); + dev_info(adev->dev, "SECUREDISPLAY: I2C buffer out put is :%s\n", i2c_output); + } else { + psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status); + } + } + break; + default: + dev_err(adev->dev, "Invalid input: %s\n", str); + } + + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + + return size; +} + +static const struct file_operations amdgpu_securedisplay_debugfs_ops = { + .owner = THIS_MODULE, + .read = NULL, + .write = amdgpu_securedisplay_debugfs_write, + .llseek = default_llseek +}; + +void amdgpu_securedisplay_debugfs_init(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + + if (!adev->psp.securedisplay_context.securedisplay_initialized) + return; + + debugfs_create_file("securedisplay_test", S_IWUSR, adev_to_drm(adev)->primary->debugfs_root, + adev, &amdgpu_securedisplay_debugfs_ops); +#endif +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.h new file mode 100644 index 000000000000..fe98574748f4 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.h @@ -0,0 +1,36 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#ifndef _AMDGPU_SECUREDISPLAY_H +#define _AMDGPU_SECUREDISPLAY_H + +#include "amdgpu.h" +#include "ta_secureDisplay_if.h" + +void amdgpu_securedisplay_debugfs_init(struct amdgpu_device *adev); +void psp_securedisplay_parse_resp_status(struct psp_context *psp, + enum ta_securedisplay_status status); +void psp_prep_securedisplay_cmd_buf(struct psp_context *psp, struct securedisplay_cmd **cmd, + enum ta_securedisplay_command command_id); + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h index 6752d8b13118..792d20261846 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h @@ -21,7 +21,7 @@ * */ -#if !defined(_AMDGPU_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#if !defined(_AMDGPU_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) #define _AMDGPU_TRACE_H_ #include @@ -127,7 +127,7 @@ TRACE_EVENT(amdgpu_bo_create, TP_fast_assign( __entry->bo = bo; - __entry->pages = bo->tbo.num_pages; + __entry->pages = bo->tbo.mem.num_pages; __entry->type = bo->tbo.mem.mem_type; __entry->prefer = bo->preferred_domains; __entry->allow = bo->allowed_domains; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 4d8f19ab1014..9fd2157b133a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include @@ -637,7 +636,7 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, out: /* update statistics */ - atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved); + atomic64_add(bo->base.size, &adev->num_bytes_moved); amdgpu_bo_move_notify(bo, evict, new_mem); return 0; } @@ -918,8 +917,8 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_bo_device *bdev, goto release_sg; /* convert SG to linear array of pages and dma addresses */ - drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, - gtt->ttm.dma_address, ttm->num_pages); + drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, + ttm->num_pages); return 0; @@ -1265,9 +1264,8 @@ static int amdgpu_ttm_tt_populate(struct ttm_bo_device *bdev, ttm->sg = sgt; } - drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, - gtt->ttm.dma_address, - ttm->num_pages); + drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, + ttm->num_pages); return 0; } @@ -2124,7 +2122,7 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo, return r; } - num_pages = bo->tbo.num_pages; + num_pages = bo->tbo.mem.num_pages; mm_node = bo->tbo.mem.mm_node; num_loops = 0; while (num_pages) { @@ -2154,7 +2152,7 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo, } } - num_pages = bo->tbo.num_pages; + num_pages = bo->tbo.mem.num_pages; mm_node = bo->tbo.mem.mm_node; while (num_pages) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h index 0e43b46d3ab5..46449e70348b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h @@ -122,6 +122,9 @@ struct ta_firmware_header_v1_0 { uint32_t ta_dtm_ucode_version; uint32_t ta_dtm_offset_bytes; uint32_t ta_dtm_size_bytes; + uint32_t ta_securedisplay_ucode_version; + uint32_t ta_securedisplay_offset_bytes; + uint32_t ta_securedisplay_size_bytes; }; enum ta_fw_type { @@ -132,6 +135,7 @@ enum ta_fw_type { TA_FW_TYPE_PSP_HDCP, TA_FW_TYPE_PSP_DTM, TA_FW_TYPE_PSP_RAP, + TA_FW_TYPE_PSP_SECUREDISPLAY, }; struct ta_fw_bin_desc { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 8b989670ed66..e2ed4689118a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -1170,7 +1170,7 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, int r, i; r = amdgpu_bo_create_reserved(adev, 1024, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_DOMAIN_GTT, &bo, NULL, (void **)&msg); if (r) return r; @@ -1202,7 +1202,7 @@ int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, int r, i; r = amdgpu_bo_create_reserved(adev, 1024, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_DOMAIN_GTT, &bo, NULL, (void **)&msg); if (r) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index 0d5284b936e4..ea6a62f67e38 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -1160,6 +1160,6 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout) error: dma_fence_put(fence); amdgpu_bo_unreserve(bo); - amdgpu_bo_unref(&bo); + amdgpu_bo_free_kernel(&bo, NULL, NULL); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index 4a77c7424dfc..99b82f3c2617 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c @@ -496,6 +496,7 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, struct amdgpu_job *job; struct amdgpu_ib *ib; uint64_t addr; + void *msg = NULL; int i, r; r = amdgpu_job_alloc_with_ib(adev, 64, @@ -505,6 +506,7 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, ib = &job->ibs[0]; addr = amdgpu_bo_gpu_offset(bo); + msg = amdgpu_bo_kptr(bo); ib->ptr[0] = PACKET0(adev->vcn.internal.data0, 0); ib->ptr[1] = addr; ib->ptr[2] = PACKET0(adev->vcn.internal.data1, 0); @@ -523,7 +525,7 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, amdgpu_bo_fence(bo, f, false); amdgpu_bo_unreserve(bo); - amdgpu_bo_unref(&bo); + amdgpu_bo_free_kernel(&bo, NULL, (void **)&msg); if (fence) *fence = dma_fence_get(f); @@ -536,7 +538,7 @@ err_free: err: amdgpu_bo_unreserve(bo); - amdgpu_bo_unref(&bo); + amdgpu_bo_free_kernel(&bo, NULL, (void **)&msg); return r; } @@ -890,6 +892,7 @@ int amdgpu_vcn_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout) error: dma_fence_put(fence); amdgpu_bo_unreserve(bo); - amdgpu_bo_unref(&bo); + amdgpu_bo_free_kernel(&bo, NULL, NULL); + return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 2d51b7694d1f..5da04d45b637 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -560,10 +560,14 @@ static int amdgpu_virt_write_vf2pf_data(struct amdgpu_device *adev) static void amdgpu_virt_update_vf2pf_work_item(struct work_struct *work) { struct amdgpu_device *adev = container_of(work, struct amdgpu_device, virt.vf2pf_work.work); + int ret; - amdgpu_virt_read_pf2vf_data(adev); + ret = amdgpu_virt_read_pf2vf_data(adev); + if (ret) + goto out; amdgpu_virt_write_vf2pf_data(adev); +out: schedule_delayed_work(&(adev->virt.vf2pf_work), adev->virt.vf2pf_update_interval_ms); } @@ -571,8 +575,8 @@ void amdgpu_virt_fini_data_exchange(struct amdgpu_device *adev) { if (adev->virt.vf2pf_update_interval_ms != 0) { DRM_INFO("clean up the vf2pf work item\n"); - flush_delayed_work(&adev->virt.vf2pf_work); cancel_delayed_work_sync(&adev->virt.vf2pf_work); + adev->virt.vf2pf_update_interval_ms = 0; } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 0768c8686983..ad91c0c3c423 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -653,9 +653,11 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, if (!bo->parent) continue; - ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move); + ttm_bo_move_to_lru_tail(&bo->tbo, &bo->tbo.mem, + &vm->lru_bulk_move); if (bo->shadow) ttm_bo_move_to_lru_tail(&bo->shadow->tbo, + &bo->shadow->tbo.mem, &vm->lru_bulk_move); } spin_unlock(&ttm_bo_glob.lru_lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index d2de2a720a3d..c89b66bb70e2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -473,6 +473,9 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, for (i = 0; pages_left >= pages_per_node; ++i) { unsigned long pages = rounddown_pow_of_two(pages_left); + /* Limit maximum size to 2GB due to SG table limitations */ + pages = min(pages, (2UL << (30 - PAGE_SHIFT))); + r = drm_mm_insert_node_in_range(mm, &nodes[i], pages, pages_per_node, 0, place->fpfn, lpfn, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c index 541ef6be390f..659b385b27b5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c @@ -324,7 +324,7 @@ static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev, struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev) { - struct amdgpu_hive_info *hive = NULL, *tmp = NULL; + struct amdgpu_hive_info *hive = NULL; int ret; if (!adev->gmc.xgmi.hive_id) @@ -337,11 +337,9 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev) mutex_lock(&xgmi_mutex); - if (!list_empty(&xgmi_hive_list)) { - list_for_each_entry_safe(hive, tmp, &xgmi_hive_list, node) { - if (hive->hive_id == adev->gmc.xgmi.hive_id) - goto pro_end; - } + list_for_each_entry(hive, &xgmi_hive_list, node) { + if (hive->hive_id == adev->gmc.xgmi.hive_id) + goto pro_end; } hive = kzalloc(sizeof(*hive), GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c index 921a69abda55..5b90efd6f6d0 100644 --- a/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c @@ -27,7 +27,6 @@ #include "athub/athub_2_0_0_offset.h" #include "athub/athub_2_0_0_sh_mask.h" #include "athub/athub_2_0_0_default.h" -#include "navi10_enum.h" #include "soc15_common.h" diff --git a/drivers/gpu/drm/amd/amdgpu/athub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/athub_v2_1.c index 66c183ddd43e..7b1b18350bf9 100644 --- a/drivers/gpu/drm/amd/amdgpu/athub_v2_1.c +++ b/drivers/gpu/drm/amd/amdgpu/athub_v2_1.c @@ -26,7 +26,6 @@ #include "athub/athub_2_1_0_offset.h" #include "athub/athub_2_1_0_sh_mask.h" -#include "navi10_enum.h" #include "soc15_common.h" diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c index 13737b317f7c..4d6832cc7fb0 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik.c +++ b/drivers/gpu/drm/amd/amdgpu/cik.c @@ -1251,13 +1251,22 @@ static void kv_restore_regs_for_reset(struct amdgpu_device *adev, WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute); } -static int cik_gpu_pci_config_reset(struct amdgpu_device *adev) +/** + * cik_asic_pci_config_reset - soft reset GPU + * + * @adev: amdgpu_device pointer + * + * Use PCI Config method to reset the GPU. + * + * Returns 0 for success. + */ +static int cik_asic_pci_config_reset(struct amdgpu_device *adev) { struct kv_reset_save_regs kv_save = { 0 }; u32 i; int r = -EINVAL; - dev_info(adev->dev, "GPU pci config reset\n"); + amdgpu_atombios_scratch_regs_engine_hung(adev, true); if (adev->flags & AMD_IS_APU) kv_save_regs_for_reset(adev, &kv_save); @@ -1285,26 +1294,6 @@ static int cik_gpu_pci_config_reset(struct amdgpu_device *adev) if (adev->flags & AMD_IS_APU) kv_restore_regs_for_reset(adev, &kv_save); - return r; -} - -/** - * cik_asic_pci_config_reset - soft reset GPU - * - * @adev: amdgpu_device pointer - * - * Use PCI Config method to reset the GPU. - * - * Returns 0 for success. - */ -static int cik_asic_pci_config_reset(struct amdgpu_device *adev) -{ - int r; - - amdgpu_atombios_scratch_regs_engine_hung(adev, true); - - r = cik_gpu_pci_config_reset(adev); - amdgpu_atombios_scratch_regs_engine_hung(adev, false); return r; diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c index da37f8a900af..307c01301c87 100644 --- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c @@ -194,19 +194,30 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev, wptr = le32_to_cpu(*ih->wptr_cpu); - if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { - wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); - /* When a ring buffer overflow happen start parsing interrupt - * from the last not overwritten vector (wptr + 16). Hopefully - * this should allow us to catchup. - */ - dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); - ih->rptr = (wptr + 16) & ih->ptr_mask; - tmp = RREG32(mmIH_RB_CNTL); - tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32(mmIH_RB_CNTL, tmp); - } + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + /* Double check that the overflow wasn't already cleared. */ + wptr = RREG32(mmIH_RB_WPTR); + + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); + + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 16). Hopefully + * this should allow us to catchup. + */ + dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; + tmp = RREG32(mmIH_RB_CNTL); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); + WREG32(mmIH_RB_CNTL, tmp); + + +out: return (wptr & ih->ptr_mask); } diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c index ffcc64ec6473..9810af712cc0 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c @@ -294,7 +294,7 @@ static int dce_virtual_get_modes(struct drm_connector *connector) static const struct mode_size { int w; int h; - } common_modes[21] = { + } common_modes[] = { { 640, 480}, { 720, 480}, { 800, 600}, @@ -312,13 +312,14 @@ static int dce_virtual_get_modes(struct drm_connector *connector) {1600, 1200}, {1920, 1080}, {1920, 1200}, + {2560, 1440}, {4096, 3112}, {3656, 2664}, {3840, 2160}, {4096, 2160}, }; - for (i = 0; i < 21; i++) { + for (i = 0; i < ARRAY_SIZE(common_modes); i++) { mode = drm_cvt_mode(dev, common_modes[i].w, common_modes[i].h, 60, false, false, false); drm_mode_probed_add(connector, mode); } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index d86b42a36560..45d1172b7bff 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -38,7 +38,6 @@ #include "smuio/smuio_11_0_0_offset.h" #include "smuio/smuio_11_0_0_sh_mask.h" #include "navi10_enum.h" -#include "hdp/hdp_5_0_0_offset.h" #include "ivsrcid/gfx/irqsrcs_gfx_10_1.h" #include "soc15.h" @@ -71,6 +70,11 @@ #define GB_ADDR_CONFIG__NUM_PKRS__SHIFT 0x8 #define GB_ADDR_CONFIG__NUM_PKRS_MASK 0x00000700L +#define mmCGTS_TCC_DISABLE_gc_10_3 0x5006 +#define mmCGTS_TCC_DISABLE_gc_10_3_BASE_IDX 1 +#define mmCGTS_USER_TCC_DISABLE_gc_10_3 0x5007 +#define mmCGTS_USER_TCC_DISABLE_gc_10_3_BASE_IDX 1 + #define mmCP_MEC_CNTL_Sienna_Cichlid 0x0f55 #define mmCP_MEC_CNTL_Sienna_Cichlid_BASE_IDX 0 #define mmRLC_SAFE_MODE_Sienna_Cichlid 0x4ca0 @@ -99,10 +103,6 @@ #define mmGCR_GENERAL_CNTL_Sienna_Cichlid 0x1580 #define mmGCR_GENERAL_CNTL_Sienna_Cichlid_BASE_IDX 0 -#define mmCGTS_TCC_DISABLE_Vangogh 0x5006 -#define mmCGTS_TCC_DISABLE_Vangogh_BASE_IDX 1 -#define mmCGTS_USER_TCC_DISABLE_Vangogh 0x5007 -#define mmCGTS_USER_TCC_DISABLE_Vangogh_BASE_IDX 1 #define mmGOLDEN_TSC_COUNT_UPPER_Vangogh 0x0025 #define mmGOLDEN_TSC_COUNT_UPPER_Vangogh_BASE_IDX 1 #define mmGOLDEN_TSC_COUNT_LOWER_Vangogh 0x0026 @@ -125,6 +125,7 @@ #define mmSPI_CONFIG_CNTL_Vangogh_BASE_IDX 1 #define mmGCR_GENERAL_CNTL_Vangogh 0x1580 #define mmGCR_GENERAL_CNTL_Vangogh_BASE_IDX 0 +#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK_Vangogh 0x0000FFFFL #define mmCP_HYP_PFP_UCODE_ADDR 0x5814 #define mmCP_HYP_PFP_UCODE_ADDR_BASE_IDX 1 @@ -3782,9 +3783,6 @@ static void gfx_v10_0_check_gfxoff_flag(struct amdgpu_device *adev) if (!gfx_v10_0_navi10_gfxoff_should_enable(adev)) adev->pm.pp_feature &= ~PP_GFXOFF_MASK; break; - case CHIP_VANGOGH: - adev->pm.pp_feature &= ~PP_GFXOFF_MASK; - break; default: break; } @@ -4494,8 +4492,7 @@ static int gfx_v10_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) + ring->pipe; - hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, - ring->queue) ? + hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; /* type-2 packets are deprecated on MEC, use type-3 instead */ r = amdgpu_ring_init(adev, ring, 1024, @@ -4942,15 +4939,12 @@ static void gfx_v10_0_get_tcc_info(struct amdgpu_device *adev) /* TCCs are global (not instanced). */ uint32_t tcc_disable; - switch (adev->asic_type) { - case CHIP_VANGOGH: - tcc_disable = RREG32_SOC15(GC, 0, mmCGTS_TCC_DISABLE_Vangogh) | - RREG32_SOC15(GC, 0, mmCGTS_USER_TCC_DISABLE_Vangogh); - break; - default: + if (adev->asic_type >= CHIP_SIENNA_CICHLID) { + tcc_disable = RREG32_SOC15(GC, 0, mmCGTS_TCC_DISABLE_gc_10_3) | + RREG32_SOC15(GC, 0, mmCGTS_USER_TCC_DISABLE_gc_10_3); + } else { tcc_disable = RREG32_SOC15(GC, 0, mmCGTS_TCC_DISABLE) | - RREG32_SOC15(GC, 0, mmCGTS_USER_TCC_DISABLE); - break; + RREG32_SOC15(GC, 0, mmCGTS_USER_TCC_DISABLE); } adev->gfx.config.tcc_disabled_mask = @@ -5715,7 +5709,7 @@ static int gfx_v10_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev) } if (amdgpu_emu_mode == 1) - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_CNTL); tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); @@ -5793,7 +5787,7 @@ static int gfx_v10_0_cp_gfx_load_ce_microcode(struct amdgpu_device *adev) } if (amdgpu_emu_mode == 1) - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_CNTL); tmp = REG_SET_FIELD(tmp, CP_CE_IC_BASE_CNTL, VMID, 0); @@ -5870,7 +5864,7 @@ static int gfx_v10_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev) } if (amdgpu_emu_mode == 1) - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_CNTL); tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); @@ -6239,7 +6233,7 @@ static int gfx_v10_0_cp_compute_load_microcode(struct amdgpu_device *adev) } if (amdgpu_emu_mode == 1) - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL); tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); @@ -6547,8 +6541,7 @@ static void gfx_v10_0_compute_mqd_set_priority(struct amdgpu_ring *ring, struct struct amdgpu_device *adev = ring->adev; if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { - if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, - ring->queue)) { + if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; mqd->cp_hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; @@ -7847,6 +7840,20 @@ static void gfx_v10_cntl_power_gating(struct amdgpu_device *adev, bool enable) data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; WREG32_SOC15(GC, 0, mmRLC_PG_CNTL, data); + + /* + * CGPG enablement required and the register to program the hysteresis value + * RLC_PG_DELAY_3.CGCG_ACTIVE_BEFORE_CGPG to the desired CGPG hysteresis value + * in refclk count. Note that RLC FW is modified to take 16 bits from + * RLC_PG_DELAY_3[15:0] as the hysteresis instead of just 8 bits. + * + * The recommendation from RLC team is setting RLC_PG_DELAY_3 to 200us(0x4E20) + * as part of CGPG enablement starting point. + */ + if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && adev->asic_type == CHIP_VANGOGH) { + data = 0x4E20 & RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK_Vangogh; + WREG32_SOC15(GC, 0, mmRLC_PG_DELAY_3, data); + } } static void gfx_v10_cntl_pg(struct amdgpu_device *adev, bool enable) @@ -7908,6 +7915,7 @@ static int gfx_v10_0_set_powergating_state(void *handle, break; case CHIP_VANGOGH: gfx_v10_cntl_pg(adev, enable); + amdgpu_gfx_off_ctrl(adev, enable); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 37639214cbbb..84d2eaa38101 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -29,6 +29,7 @@ #include "amdgpu.h" #include "amdgpu_gfx.h" +#include "amdgpu_ring.h" #include "vi.h" #include "vi_structs.h" #include "vid.h" @@ -1923,8 +1924,7 @@ static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) + ring->pipe; - hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, - ring->queue) ? + hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_RING_PRIO_DEFAULT; /* type-2 packets are deprecated on MEC, use type-3 instead */ r = amdgpu_ring_init(adev, ring, 1024, @@ -4442,8 +4442,7 @@ static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *m struct amdgpu_device *adev = ring->adev; if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { - if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, - ring->queue)) { + if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; mqd->cp_hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; @@ -6847,6 +6846,66 @@ static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring) amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ } + +/* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ +#define mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT 0x0000007f +static void gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring *ring, + uint32_t pipe, bool enable) +{ + uint32_t val; + uint32_t wcl_cs_reg; + + val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT; + + switch (pipe) { + case 0: + wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS0; + break; + case 1: + wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS1; + break; + case 2: + wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS2; + break; + case 3: + wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS3; + break; + default: + DRM_DEBUG("invalid pipe %d\n", pipe); + return; + } + + amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); + +} + +#define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT 0x07ffffff +static void gfx_v8_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t val; + int i; + + /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit + * number of gfx waves. Setting 5 bit will make sure gfx only gets + * around 25% of gpu resources. + */ + val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; + amdgpu_ring_emit_wreg(ring, mmSPI_WCL_PIPE_PERCENT_GFX, val); + + /* Restrict waves for normal/low priority compute queues as well + * to get best QoS for high priority compute jobs. + * + * amdgpu controls only 1st ME(0-3 CS pipes). + */ + for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { + if (i != ring->pipe) + gfx_v8_0_emit_wave_limit_cs(ring, i, enable); + + } + +} + static const struct amd_ip_funcs gfx_v8_0_ip_funcs = { .name = "gfx_v8_0", .early_init = gfx_v8_0_early_init, @@ -6930,7 +6989,9 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */ 7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */ - 7, /* gfx_v8_0_emit_mem_sync_compute */ + 7 + /* gfx_v8_0_emit_mem_sync_compute */ + 5 + /* gfx_v8_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ + 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ .emit_ib = gfx_v8_0_ring_emit_ib_compute, .emit_fence = gfx_v8_0_ring_emit_fence_compute, @@ -6944,6 +7005,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { .pad_ib = amdgpu_ring_generic_pad_ib, .emit_wreg = gfx_v8_0_ring_emit_wreg, .emit_mem_sync = gfx_v8_0_emit_mem_sync_compute, + .emit_wave_limit = gfx_v8_0_emit_wave_limit, }; static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 5f4805e4d04a..65db88bb6cbc 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -38,7 +38,6 @@ #include "gc/gc_9_0_sh_mask.h" #include "vega10_enum.h" -#include "hdp/hdp_4_0_offset.h" #include "soc15_common.h" #include "clearstate_gfx9.h" @@ -53,6 +52,7 @@ #include "asic_reg/pwr/pwr_10_0_offset.h" #include "asic_reg/pwr/pwr_10_0_sh_mask.h" +#include "asic_reg/gc/gc_9_0_default.h" #define GFX9_NUM_GFX_RINGS 1 #define GFX9_MEC_HPD_SIZE 4096 @@ -2228,8 +2228,7 @@ static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) + ring->pipe; - hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, - ring->queue) ? + hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; /* type-2 packets are deprecated on MEC, use type-3 instead */ return amdgpu_ring_init(adev, ring, 1024, @@ -3391,9 +3390,7 @@ static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *m struct amdgpu_device *adev = ring->adev; if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { - if (amdgpu_gfx_is_high_priority_compute_queue(adev, - ring->pipe, - ring->queue)) { + if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; mqd->cp_hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; @@ -6671,6 +6668,65 @@ static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ } +static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, + uint32_t pipe, bool enable) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t val; + uint32_t wcl_cs_reg; + + /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ + val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; + + switch (pipe) { + case 0: + wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); + break; + case 1: + wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); + break; + case 2: + wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); + break; + case 3: + wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); + break; + default: + DRM_DEBUG("invalid pipe %d\n", pipe); + return; + } + + amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); + +} +static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t val; + int i; + + + /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit + * number of gfx waves. Setting 5 bit will make sure gfx only gets + * around 25% of gpu resources. + */ + val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; + amdgpu_ring_emit_wreg(ring, + SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), + val); + + /* Restrict waves for normal/low priority compute queues as well + * to get best QoS for high priority compute jobs. + * + * amdgpu controls only 1st ME(0-3 CS pipes). + */ + for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { + if (i != ring->pipe) + gfx_v9_0_emit_wave_limit_cs(ring, i, enable); + + } +} + static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { .name = "gfx_v9_0", .early_init = gfx_v9_0_early_init, @@ -6760,7 +6816,9 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 2 + /* gfx_v9_0_ring_emit_vm_flush */ 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ - 7, /* gfx_v9_0_emit_mem_sync */ + 7 + /* gfx_v9_0_emit_mem_sync */ + 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ + 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ .emit_ib = gfx_v9_0_ring_emit_ib_compute, .emit_fence = gfx_v9_0_ring_emit_fence, @@ -6776,6 +6834,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, .emit_mem_sync = gfx_v9_0_emit_mem_sync, + .emit_wave_limit = gfx_v9_0_emit_wave_limit, }; static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index 5648c48be77f..3b7c6c31fce1 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -27,8 +27,6 @@ #include "gmc_v10_0.h" #include "umc_v8_7.h" -#include "hdp/hdp_5_0_0_offset.h" -#include "hdp/hdp_5_0_0_sh_mask.h" #include "athub/athub_2_0_0_sh_mask.h" #include "athub/athub_2_0_0_offset.h" #include "dcn/dcn_2_0_0_offset.h" @@ -312,7 +310,7 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, int r; /* flush hdp cache */ - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); /* For SRIOV run time, driver shouldn't access the register through MMIO * Directly use kiq to do the vm invalidation instead @@ -995,7 +993,6 @@ static int gmc_v10_0_gart_enable(struct amdgpu_device *adev) { int r; bool value; - u32 tmp; if (adev->gart.bo == NULL) { dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); @@ -1014,15 +1011,10 @@ static int gmc_v10_0_gart_enable(struct amdgpu_device *adev) if (r) return r; - tmp = RREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL); - tmp |= HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK; - WREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL, tmp); - - tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); - WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); + adev->hdp.funcs->init_registers(adev); /* Flush HDP after it is initialized */ - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ? false : true; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index e22268f9dba7..3686e777c76c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -31,8 +31,6 @@ #include "amdgpu_atomfirmware.h" #include "amdgpu_gem.h" -#include "hdp/hdp_4_0_offset.h" -#include "hdp/hdp_4_0_sh_mask.h" #include "gc/gc_9_0_sh_mask.h" #include "dce/dce_12_0_offset.h" #include "dce/dce_12_0_sh_mask.h" @@ -241,60 +239,44 @@ static const char *mmhub_client_ids_vega20[][2] = { }; static const char *mmhub_client_ids_arcturus[][2] = { + [0][0] = "DBGU1", + [1][0] = "XDP", [2][0] = "MP1", - [3][0] = "MP0", - [10][0] = "UTCL2", - [13][0] = "OSS", [14][0] = "HDP", - [15][0] = "SDMA0", - [32+15][0] = "SDMA1", - [64+15][0] = "SDMA2", - [96+15][0] = "SDMA3", - [128+15][0] = "SDMA4", - [160+11][0] = "JPEG", - [160+12][0] = "VCN", - [160+13][0] = "VCNU", - [160+15][0] = "SDMA5", - [192+10][0] = "UTCL2", - [192+11][0] = "JPEG1", - [192+12][0] = "VCN1", - [192+13][0] = "VCN1U", - [192+15][0] = "SDMA6", - [224+15][0] = "SDMA7", + [171][0] = "JPEG", + [172][0] = "VCN", + [173][0] = "VCNU", + [203][0] = "JPEG1", + [204][0] = "VCN1", + [205][0] = "VCN1U", + [256][0] = "SDMA0", + [257][0] = "SDMA1", + [258][0] = "SDMA2", + [259][0] = "SDMA3", + [260][0] = "SDMA4", + [261][0] = "SDMA5", + [262][0] = "SDMA6", + [263][0] = "SDMA7", + [384][0] = "OSS", [0][1] = "DBGU1", [1][1] = "XDP", [2][1] = "MP1", - [3][1] = "MP0", - [13][1] = "OSS", [14][1] = "HDP", - [15][1] = "SDMA0", - [32+15][1] = "SDMA1", - [64+15][1] = "SDMA2", - [96+15][1] = "SDMA3", - [128+15][1] = "SDMA4", - [160+11][1] = "JPEG", - [160+12][1] = "VCN", - [160+13][1] = "VCNU", - [160+15][1] = "SDMA5", - [192+11][1] = "JPEG1", - [192+12][1] = "VCN1", - [192+13][1] = "VCN1U", - [192+15][1] = "SDMA6", - [224+15][1] = "SDMA7", -}; - -static const u32 golden_settings_vega10_hdp[] = -{ - 0xf64, 0x0fffffff, 0x00000000, - 0xf65, 0x0fffffff, 0x00000000, - 0xf66, 0x0fffffff, 0x00000000, - 0xf67, 0x0fffffff, 0x00000000, - 0xf68, 0x0fffffff, 0x00000000, - 0xf6a, 0x0fffffff, 0x00000000, - 0xf6b, 0x0fffffff, 0x00000000, - 0xf6c, 0x0fffffff, 0x00000000, - 0xf6d, 0x0fffffff, 0x00000000, - 0xf6e, 0x0fffffff, 0x00000000, + [171][1] = "JPEG", + [172][1] = "VCN", + [173][1] = "VCNU", + [203][1] = "JPEG1", + [204][1] = "VCN1", + [205][1] = "VCN1U", + [256][1] = "SDMA0", + [257][1] = "SDMA1", + [258][1] = "SDMA2", + [259][1] = "SDMA3", + [260][1] = "SDMA4", + [261][1] = "SDMA5", + [262][1] = "SDMA6", + [263][1] = "SDMA7", + [384][1] = "OSS", }; static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = @@ -1571,7 +1553,6 @@ static int gmc_v9_0_hw_init(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; bool value; int r, i; - u32 tmp; /* The sequence of these two function calls matters.*/ gmc_v9_0_init_golden_registers(adev); @@ -1583,31 +1564,13 @@ static int gmc_v9_0_hw_init(void *handle) WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); } - amdgpu_device_program_register_sequence(adev, - golden_settings_vega10_hdp, - ARRAY_SIZE(golden_settings_vega10_hdp)); - if (adev->mmhub.funcs->update_power_gating) adev->mmhub.funcs->update_power_gating(adev, true); - switch (adev->asic_type) { - case CHIP_ARCTURUS: - WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); - break; - default: - break; - } - - WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); - - tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); - WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); - - WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); - WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40)); + adev->hdp.funcs->init_registers(adev); /* After HDP is initialized, flush HDP.*/ - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) value = false; diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c new file mode 100644 index 000000000000..e46621fed5b9 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c @@ -0,0 +1,137 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "amdgpu.h" +#include "amdgpu_atombios.h" +#include "hdp_v4_0.h" +#include "amdgpu_ras.h" + +#include "hdp/hdp_4_0_offset.h" +#include "hdp/hdp_4_0_sh_mask.h" +#include + +/* for Vega20 register name change */ +#define mmHDP_MEM_POWER_CTRL 0x00d4 +#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK 0x00000001L +#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK 0x00000002L +#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK 0x00010000L +#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK 0x00020000L +#define mmHDP_MEM_POWER_CTRL_BASE_IDX 0 + +static void hdp_v4_0_flush_hdp(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ + if (!ring || !ring->funcs->emit_wreg) + WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); + else + amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); +} + +static void hdp_v4_0_invalidate_hdp(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ + if (!ring || !ring->funcs->emit_wreg) + WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); + else + amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( + HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); +} + +static void hdp_v4_0_reset_ras_error_count(struct amdgpu_device *adev) +{ + if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP)) + return; + /*read back hdp ras counter to reset it to 0 */ + RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT); +} + +static void hdp_v4_0_update_clock_gating(struct amdgpu_device *adev, + bool enable) +{ + uint32_t def, data; + + if (adev->asic_type == CHIP_VEGA10 || + adev->asic_type == CHIP_VEGA12 || + adev->asic_type == CHIP_RAVEN) { + def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); + + if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) + data |= HDP_MEM_POWER_LS__LS_ENABLE_MASK; + else + data &= ~HDP_MEM_POWER_LS__LS_ENABLE_MASK; + + if (def != data) + WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS), data); + } else { + def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL)); + + if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) + data |= HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | + HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | + HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | + HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK; + else + data &= ~(HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | + HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | + HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | + HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK); + + if (def != data) + WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL), data); + } +} + +static void hdp_v4_0_get_clockgating_state(struct amdgpu_device *adev, + u32 *flags) +{ + int data; + + /* AMD_CG_SUPPORT_HDP_LS */ + data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); + if (data & HDP_MEM_POWER_LS__LS_ENABLE_MASK) + *flags |= AMD_CG_SUPPORT_HDP_LS; +} + +static void hdp_v4_0_init_registers(struct amdgpu_device *adev) +{ + switch (adev->asic_type) { + case CHIP_ARCTURUS: + WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); + break; + default: + break; + } + + WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); + + WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); + WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40)); +} + +const struct amdgpu_hdp_funcs hdp_v4_0_funcs = { + .flush_hdp = hdp_v4_0_flush_hdp, + .invalidate_hdp = hdp_v4_0_invalidate_hdp, + .reset_ras_error_count = hdp_v4_0_reset_ras_error_count, + .update_clock_gating = hdp_v4_0_update_clock_gating, + .get_clock_gating_state = hdp_v4_0_get_clockgating_state, + .init_registers = hdp_v4_0_init_registers, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.h b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.h new file mode 100644 index 000000000000..d1e6399e8c46 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.h @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __HDP_V4_0_H__ +#define __HDP_V4_0_H__ + +#include "soc15_common.h" + +extern const struct amdgpu_hdp_funcs hdp_v4_0_funcs; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c b/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c new file mode 100644 index 000000000000..7a15e669b68d --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c @@ -0,0 +1,212 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "amdgpu.h" +#include "amdgpu_atombios.h" +#include "hdp_v5_0.h" + +#include "hdp/hdp_5_0_0_offset.h" +#include "hdp/hdp_5_0_0_sh_mask.h" +#include + +static void hdp_v5_0_flush_hdp(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ + if (!ring || !ring->funcs->emit_wreg) + WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); + else + amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); +} + +static void hdp_v5_0_invalidate_hdp(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ + if (!ring || !ring->funcs->emit_wreg) { + WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); + } else { + amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( + HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); + } +} + +static void hdp_v5_0_update_mem_power_gating(struct amdgpu_device *adev, + bool enable) +{ + uint32_t hdp_clk_cntl, hdp_clk_cntl1; + uint32_t hdp_mem_pwr_cntl; + + if (!(adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | + AMD_CG_SUPPORT_HDP_DS | + AMD_CG_SUPPORT_HDP_SD))) + return; + + hdp_clk_cntl = hdp_clk_cntl1 = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); + hdp_mem_pwr_cntl = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL); + + /* Before doing clock/power mode switch, + * forced on IPH & RC clock */ + hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL, + IPH_MEM_CLK_SOFT_OVERRIDE, 1); + hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL, + RC_MEM_CLK_SOFT_OVERRIDE, 1); + WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl); + + /* HDP 5.0 doesn't support dynamic power mode switch, + * disable clock and power gating before any changing */ + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_CTRL_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_LS_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_DS_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_SD_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + RC_MEM_POWER_CTRL_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + RC_MEM_POWER_LS_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + RC_MEM_POWER_DS_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + RC_MEM_POWER_SD_EN, 0); + WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl); + + /* only one clock gating mode (LS/DS/SD) can be enabled */ + if (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS) { + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_LS_EN, enable); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + RC_MEM_POWER_LS_EN, enable); + } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_DS) { + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_DS_EN, enable); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + RC_MEM_POWER_DS_EN, enable); + } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_SD) { + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_SD_EN, enable); + /* RC should not use shut down mode, fallback to ds */ + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + RC_MEM_POWER_DS_EN, enable); + } + + /* confirmed that IPH_MEM_POWER_CTRL_EN and RC_MEM_POWER_CTRL_EN have to + * be set for SRAM LS/DS/SD */ + if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_DS | + AMD_CG_SUPPORT_HDP_SD)) { + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_CTRL_EN, 1); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + RC_MEM_POWER_CTRL_EN, 1); + } + + WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl); + + /* restore IPH & RC clock override after clock/power mode changing */ + WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl1); +} + +static void hdp_v5_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, + bool enable) +{ + uint32_t hdp_clk_cntl; + + if (!(adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG)) + return; + + hdp_clk_cntl = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); + + if (enable) { + hdp_clk_cntl &= + ~(uint32_t) + (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK); + } else { + hdp_clk_cntl |= HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK; + } + + WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl); +} + +static void hdp_v5_0_update_clock_gating(struct amdgpu_device *adev, + bool enable) +{ + hdp_v5_0_update_mem_power_gating(adev, enable); + hdp_v5_0_update_medium_grain_clock_gating(adev, enable); +} + +static void hdp_v5_0_get_clockgating_state(struct amdgpu_device *adev, + u32 *flags) +{ + uint32_t tmp; + + /* AMD_CG_SUPPORT_HDP_MGCG */ + tmp = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); + if (!(tmp & (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK))) + *flags |= AMD_CG_SUPPORT_HDP_MGCG; + + /* AMD_CG_SUPPORT_HDP_LS/DS/SD */ + tmp = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL); + if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK) + *flags |= AMD_CG_SUPPORT_HDP_LS; + else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_DS_EN_MASK) + *flags |= AMD_CG_SUPPORT_HDP_DS; + else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_SD_EN_MASK) + *flags |= AMD_CG_SUPPORT_HDP_SD; +} + +static void hdp_v5_0_init_registers(struct amdgpu_device *adev) +{ + u32 tmp; + + tmp = RREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL); + tmp |= HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK; + WREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL, tmp); +} + +const struct amdgpu_hdp_funcs hdp_v5_0_funcs = { + .flush_hdp = hdp_v5_0_flush_hdp, + .invalidate_hdp = hdp_v5_0_invalidate_hdp, + .update_clock_gating = hdp_v5_0_update_clock_gating, + .get_clock_gating_state = hdp_v5_0_get_clockgating_state, + .init_registers = hdp_v5_0_init_registers, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.h b/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.h new file mode 100644 index 000000000000..2d5ec2b419f3 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.h @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __HDP_V5_0_H__ +#define __HDP_V5_0_H__ + +#include "soc15_common.h" + +extern const struct amdgpu_hdp_funcs hdp_v5_0_funcs; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c index 37d8b6ca4dab..cc957471f31e 100644 --- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c @@ -194,19 +194,29 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev, wptr = le32_to_cpu(*ih->wptr_cpu); - if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { - wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); - /* When a ring buffer overflow happen start parsing interrupt - * from the last not overwritten vector (wptr + 16). Hopefully - * this should allow us to catchup. - */ - dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); - ih->rptr = (wptr + 16) & ih->ptr_mask; - tmp = RREG32(mmIH_RB_CNTL); - tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32(mmIH_RB_CNTL, tmp); - } + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + /* Double check that the overflow wasn't already cleared. */ + wptr = RREG32(mmIH_RB_WPTR); + + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 16). Hopefully + * this should allow us to catchup. + */ + dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; + tmp = RREG32(mmIH_RB_CNTL); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); + WREG32(mmIH_RB_CNTL, tmp); + + +out: return (wptr & ih->ptr_mask); } diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c index 985e454463e1..7f30629f21a2 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c @@ -554,7 +554,7 @@ static int mes_v10_1_allocate_eop_buf(struct amdgpu_device *adev) return r; } - memset(eop, 0, adev->mes.eop_gpu_obj->tbo.mem.size); + memset(eop, 0, adev->mes.eop_gpu_obj->tbo.base.size); amdgpu_bo_kunmap(adev->mes.eop_gpu_obj); amdgpu_bo_unreserve(adev->mes.eop_gpu_obj); diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c index 1961745e89c7..ab9be5ad5a5f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c @@ -531,12 +531,12 @@ mmhub_v2_3_update_medium_grain_light_sleep(struct amdgpu_device *adev, if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)) { data &= ~MM_ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK; - data1 &= !(DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | + data1 &= ~(DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_WRITE_MASK | DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_READ_MASK | DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_RETURN_MASK | DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_REGISTER_MASK); - data2 &= !(DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | + data2 &= ~(DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_WRITE_MASK | DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_READ_MASK | DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_RETURN_MASK | diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c index 7767ccca526b..3ee481557fc9 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c @@ -255,6 +255,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) if (!down_read_trylock(&adev->reset_sem)) return; + amdgpu_virt_fini_data_exchange(adev); atomic_set(&adev->in_gpu_reset, 1); do { diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c index dd5c1e6ce009..48e588d3c409 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c @@ -276,6 +276,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) if (!down_read_trylock(&adev->reset_sem)) return; + amdgpu_virt_fini_data_exchange(adev); atomic_set(&adev->in_gpu_reset, 1); do { diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c index 7ba229e43799..f4e4040bbd25 100644 --- a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c @@ -39,6 +39,53 @@ static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev); +/** + * navi10_ih_init_register_offset - Initialize register offset for ih rings + * + * @adev: amdgpu_device pointer + * + * Initialize register offset ih rings (NAVI10). + */ +static void navi10_ih_init_register_offset(struct amdgpu_device *adev) +{ + struct amdgpu_ih_regs *ih_regs; + + if (adev->irq.ih.ring_size) { + ih_regs = &adev->irq.ih.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR); + ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO); + ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL; + } + + if (adev->irq.ih1.ring_size) { + ih_regs = &adev->irq.ih1.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1; + } + + if (adev->irq.ih2.ring_size) { + ih_regs = &adev->irq.ih2.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2; + } +} + /** * force_update_wptr_for_self_int - Force update the wptr for self interrupt * @@ -82,133 +129,66 @@ force_update_wptr_for_self_int(struct amdgpu_device *adev, } /** - * navi10_ih_enable_interrupts - Enable the interrupt ring buffer + * navi10_ih_toggle_ring_interrupts - toggle the interrupt ring buffer * * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointet + * @enable: true - enable the interrupts, false - disable the interrupts * - * Enable the interrupt ring buffer (NAVI10). + * Toggle the interrupt ring buffer (NAVI10) */ -static void navi10_ih_enable_interrupts(struct amdgpu_device *adev) +static int navi10_ih_toggle_ring_interrupts(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + bool enable) { - u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); + struct amdgpu_ih_regs *ih_regs; + uint32_t tmp; - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return; - } + ih_regs = &ih->ih_regs; + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0)); + /* enable_intr field is only valid in ring0 */ + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0)); + WREG32(ih_regs->ih_rb_cntl, tmp); + + if (enable) { + ih->enabled = true; } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); + /* set rptr, wptr to 0 */ + WREG32(ih_regs->ih_rb_rptr, 0); + WREG32(ih_regs->ih_rb_wptr, 0); + ih->enabled = false; + ih->rptr = 0; } - adev->irq.ih.enabled = true; - - if (adev->irq.ih1.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, - RB_ENABLE, 1); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); - } - adev->irq.ih1.enabled = true; - } - - if (adev->irq.ih2.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, - RB_ENABLE, 1); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); - } - adev->irq.ih2.enabled = true; - } - - if (adev->irq.ih_soft.ring_size) - adev->irq.ih_soft.enabled = true; + return 0; } /** - * navi10_ih_disable_interrupts - Disable the interrupt ring buffer + * navi10_ih_toggle_interrupts - Toggle all the available interrupt ring buffers * * @adev: amdgpu_device pointer + * @enable: enable or disable interrupt ring buffers * - * Disable the interrupt ring buffer (NAVI10). + * Toggle all the available interrupt ring buffers (NAVI10). */ -static void navi10_ih_disable_interrupts(struct amdgpu_device *adev) +static int navi10_ih_toggle_interrupts(struct amdgpu_device *adev, bool enable) { - u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + int i; + int r; - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return; + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + r = navi10_ih_toggle_ring_interrupts(adev, ih[i], enable); + if (r) + return r; } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); - } - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); - adev->irq.ih.enabled = false; - adev->irq.ih.rptr = 0; - - if (adev->irq.ih1.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, - RB_ENABLE, 0); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); - } - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); - adev->irq.ih1.enabled = false; - adev->irq.ih1.rptr = 0; - } - - if (adev->irq.ih2.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, - RB_ENABLE, 0); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); - } - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); - adev->irq.ih2.enabled = false; - adev->irq.ih2.rptr = 0; } + return 0; } static uint32_t navi10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) @@ -253,22 +233,49 @@ static uint32_t navi10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) return ih_doorbell_rtpr; } -static void navi10_ih_reroute_ih(struct amdgpu_device *adev) +/** + * navi10_ih_enable_ring - enable an ih ring buffer + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + * Enable an ih ring buffer (NAVI10) + */ +static int navi10_ih_enable_ring(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { + struct amdgpu_ih_regs *ih_regs; uint32_t tmp; - /* Reroute to IH ring 1 for VMC */ - WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x12); - tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); - WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp); + ih_regs = &ih->ih_regs; - /* Reroute IH ring 1 for UMC */ - WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x1B); - tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); - WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp); + /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ + WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8); + WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff); + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = navi10_ih_rb_cntl(ih, tmp); + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); + if (ih == &adev->irq.ih1) { + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); + } + WREG32(ih_regs->ih_rb_cntl, tmp); + + if (ih == &adev->irq.ih) { + /* set the ih ring 0 writeback address whether it's enabled or not */ + WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr)); + WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF); + } + + /* set rptr, wptr to 0 */ + WREG32(ih_regs->ih_rb_wptr, 0); + WREG32(ih_regs->ih_rb_rptr, 0); + + WREG32(ih_regs->ih_doorbell_rptr, navi10_ih_doorbell_rptr(ih)); + + return 0; } /** @@ -284,36 +291,21 @@ static void navi10_ih_reroute_ih(struct amdgpu_device *adev) */ static int navi10_ih_irq_init(struct amdgpu_device *adev) { - struct amdgpu_ih_ring *ih = &adev->irq.ih; - u32 ih_rb_cntl, ih_chicken; + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + u32 ih_chicken; u32 tmp; + int ret; + int i; /* disable irqs */ - navi10_ih_disable_interrupts(adev); + ret = navi10_ih_toggle_interrupts(adev, false); + if (ret) + return ret; adev->nbio.funcs->ih_control(adev); - /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); - ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, - !!adev->irq.msi_enabled); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); - } - if (adev->irq.ih1.ring_size) - navi10_ih_reroute_ih(adev); - if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) { - if (ih->use_bus_addr) { + if (ih[0]->use_bus_addr) { switch (adev->asic_type) { case CHIP_SIENNA_CICHLID: case CHIP_NAVY_FLOUNDER: @@ -334,77 +326,17 @@ static int navi10_ih_irq_init(struct amdgpu_device *adev) } } - /* set the writeback address whether it's enabled or not */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, - lower_32_bits(ih->wptr_addr)); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, - upper_32_bits(ih->wptr_addr) & 0xFFFF); - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, - navi10_ih_doorbell_rptr(ih)); - - adev->nbio.funcs->ih_doorbell_range(adev, ih->use_doorbell, - ih->doorbell_index); - - ih = &adev->irq.ih1; - if (ih->ring_size) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1, - (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, - WPTR_OVERFLOW_ENABLE, 0); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, - RB_FULL_DRAIN_ENABLE, 1); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + ret = navi10_ih_enable_ring(adev, ih[i]); + if (ret) + return ret; } - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1, - navi10_ih_doorbell_rptr(ih)); - } - - ih = &adev->irq.ih2; - if (ih->ring_size) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2, - (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl); - - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); - } - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2, - navi10_ih_doorbell_rptr(ih)); } + /* update doorbell range for ih ring 0*/ + adev->nbio.funcs->ih_doorbell_range(adev, ih[0]->use_doorbell, + ih[0]->doorbell_index); tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, @@ -418,10 +350,15 @@ static int navi10_ih_irq_init(struct amdgpu_device *adev) pci_set_master(adev->pdev); /* enable interrupts */ - navi10_ih_enable_interrupts(adev); + ret = navi10_ih_toggle_interrupts(adev, true); + if (ret) + return ret; /* enable wptr force update for self int */ force_update_wptr_for_self_int(adev, 0, 8, true); + if (adev->irq.ih_soft.ring_size) + adev->irq.ih_soft.enabled = true; + return 0; } @@ -435,7 +372,7 @@ static int navi10_ih_irq_init(struct amdgpu_device *adev) static void navi10_ih_irq_disable(struct amdgpu_device *adev) { force_update_wptr_for_self_int(adev, 0, 8, false); - navi10_ih_disable_interrupts(adev); + navi10_ih_toggle_interrupts(adev, false); /* Wait and acknowledge irq */ mdelay(1); @@ -455,23 +392,16 @@ static void navi10_ih_irq_disable(struct amdgpu_device *adev) static u32 navi10_ih_get_wptr(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { - u32 wptr, reg, tmp; + u32 wptr, tmp; + struct amdgpu_ih_regs *ih_regs; wptr = le32_to_cpu(*ih->wptr_cpu); + ih_regs = &ih->ih_regs; if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) goto out; - if (ih == &adev->irq.ih) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); - else if (ih == &adev->irq.ih1) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); - else if (ih == &adev->irq.ih2) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); - else - BUG(); - - wptr = RREG32_NO_KIQ(reg); + wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr); if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) goto out; wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); @@ -486,67 +416,13 @@ static u32 navi10_ih_get_wptr(struct amdgpu_device *adev, wptr, ih->rptr, tmp); ih->rptr = tmp; - if (ih == &adev->irq.ih) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); - else if (ih == &adev->irq.ih1) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); - else if (ih == &adev->irq.ih2) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); - else - BUG(); - - tmp = RREG32_NO_KIQ(reg); + tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl); tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32_NO_KIQ(reg, tmp); + WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp); out: return (wptr & ih->ptr_mask); } -/** - * navi10_ih_decode_iv - decode an interrupt vector - * - * @adev: amdgpu_device pointer - * @ih: IH ring buffer to decode - * @entry: IV entry to place decoded information into - * - * Decodes the interrupt vector at the current rptr - * position and also advance the position. - */ -static void navi10_ih_decode_iv(struct amdgpu_device *adev, - struct amdgpu_ih_ring *ih, - struct amdgpu_iv_entry *entry) -{ - /* wptr/rptr are in bytes! */ - u32 ring_index = ih->rptr >> 2; - uint32_t dw[8]; - - dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); - dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); - dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); - dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); - dw[4] = le32_to_cpu(ih->ring[ring_index + 4]); - dw[5] = le32_to_cpu(ih->ring[ring_index + 5]); - dw[6] = le32_to_cpu(ih->ring[ring_index + 6]); - dw[7] = le32_to_cpu(ih->ring[ring_index + 7]); - - entry->client_id = dw[0] & 0xff; - entry->src_id = (dw[0] >> 8) & 0xff; - entry->ring_id = (dw[0] >> 16) & 0xff; - entry->vmid = (dw[0] >> 24) & 0xf; - entry->vmid_src = (dw[0] >> 31); - entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32); - entry->timestamp_src = dw[2] >> 31; - entry->pasid = dw[3] & 0xffff; - entry->pasid_src = dw[3] >> 31; - entry->src_data[0] = dw[4]; - entry->src_data[1] = dw[5]; - entry->src_data[2] = dw[6]; - entry->src_data[3] = dw[7]; - - /* wptr/rptr are in bytes! */ - ih->rptr += 32; -} - /** * navi10_ih_irq_rearm - rearm IRQ if lost * @@ -557,22 +433,15 @@ static void navi10_ih_decode_iv(struct amdgpu_device *adev, static void navi10_ih_irq_rearm(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { - uint32_t reg_rptr = 0; uint32_t v = 0; uint32_t i = 0; + struct amdgpu_ih_regs *ih_regs; - if (ih == &adev->irq.ih) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); - else if (ih == &adev->irq.ih1) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); - else if (ih == &adev->irq.ih2) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); - else - return; + ih_regs = &ih->ih_regs; /* Rearm IRQ / re-write doorbell if doorbell write is lost */ for (i = 0; i < MAX_REARM_RETRY; i++) { - v = RREG32_NO_KIQ(reg_rptr); + v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr); if ((v < ih->ring_size) && (v != ih->rptr)) WDOORBELL32(ih->doorbell_index, ih->rptr); else @@ -591,6 +460,8 @@ static void navi10_ih_irq_rearm(struct amdgpu_device *adev, static void navi10_ih_set_rptr(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { + struct amdgpu_ih_regs *ih_regs; + if (ih->use_doorbell) { /* XXX check if swapping is necessary on BE */ *ih->rptr_cpu = ih->rptr; @@ -598,12 +469,9 @@ static void navi10_ih_set_rptr(struct amdgpu_device *adev, if (amdgpu_sriov_vf(adev)) navi10_ih_irq_rearm(adev, ih); - } else if (ih == &adev->irq.ih) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr); - } else if (ih == &adev->irq.ih1) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, ih->rptr); - } else if (ih == &adev->irq.ih2) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, ih->rptr); + } else { + ih_regs = &ih->ih_regs; + WREG32(ih_regs->ih_rb_rptr, ih->rptr); } } @@ -685,23 +553,8 @@ static int navi10_ih_sw_init(void *handle) adev->irq.ih1.ring_size = 0; adev->irq.ih2.ring_size = 0; - if (adev->asic_type < CHIP_NAVI10) { - r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); - if (r) - return r; - - adev->irq.ih1.use_doorbell = true; - adev->irq.ih1.doorbell_index = - (adev->doorbell_index.ih + 1) << 1; - - r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); - if (r) - return r; - - adev->irq.ih2.use_doorbell = true; - adev->irq.ih2.doorbell_index = - (adev->doorbell_index.ih + 2) << 1; - } + /* initialize ih control registers offset */ + navi10_ih_init_register_offset(adev); r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true); if (r) @@ -717,6 +570,7 @@ static int navi10_ih_sw_fini(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; amdgpu_irq_fini(adev); + amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft); amdgpu_ih_ring_fini(adev, &adev->irq.ih2); amdgpu_ih_ring_fini(adev, &adev->irq.ih1); amdgpu_ih_ring_fini(adev, &adev->irq.ih); @@ -848,7 +702,7 @@ static const struct amd_ip_funcs navi10_ih_ip_funcs = { static const struct amdgpu_ih_funcs navi10_ih_funcs = { .get_wptr = navi10_ih_get_wptr, - .decode_iv = navi10_ih_decode_iv, + .decode_iv = amdgpu_ih_decode_iv_helper, .set_rptr = navi10_ih_set_rptr }; diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c b/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c index b5c3db16c2b0..05ddec7ba7e2 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c @@ -34,6 +34,14 @@ #define smnCPM_CONTROL 0x11180460 #define smnPCIE_CNTL2 0x11180070 #define smnPCIE_LC_CNTL 0x11140280 +#define smnPCIE_LC_CNTL3 0x111402d4 +#define smnPCIE_LC_CNTL6 0x111402ec +#define smnPCIE_LC_CNTL7 0x111402f0 +#define smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2 0x1014008c +#define smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL 0x10123538 +#define smnBIF_CFG_DEV0_EPF0_PCIE_LTR_CAP 0x10140324 +#define smnPSWUSP0_PCIE_LC_CNTL2 0x111402c4 +#define smnNBIF_MGCG_CTRL_LCLK 0x1013a21c #define mmBIF_SDMA2_DOORBELL_RANGE 0x01d6 #define mmBIF_SDMA2_DOORBELL_RANGE_BASE_IDX 2 @@ -80,15 +88,6 @@ static void nbio_v2_3_mc_access_enable(struct amdgpu_device *adev, bool enable) WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0); } -static void nbio_v2_3_hdp_flush(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - else - amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); -} - static u32 nbio_v2_3_get_memsize(struct amdgpu_device *adev) { return RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE); @@ -359,6 +358,111 @@ static void nbio_v2_3_enable_aspm(struct amdgpu_device *adev, WREG32_PCIE(smnPCIE_LC_CNTL, data); } +static void nbio_v2_3_program_ltr(struct amdgpu_device *adev) +{ + uint32_t def, data; + + WREG32_PCIE(smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL, 0x75EB); + + def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP2); + data &= ~RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK; + if (def != data) + WREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP2, data); + + def = data = RREG32_PCIE(smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL); + data &= ~EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK; + if (def != data) + WREG32_PCIE(smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL, data); + + def = data = RREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2); + data |= BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN_MASK; + if (def != data) + WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2, data); +} + +static void nbio_v2_3_program_aspm(struct amdgpu_device *adev) +{ + uint32_t def, data; + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL); + data &= ~PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK; + data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK; + data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL, data); + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL7); + data |= PCIE_LC_CNTL7__LC_NBIF_ASPM_INPUT_EN_MASK; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL7, data); + + def = data = RREG32_PCIE(smnNBIF_MGCG_CTRL_LCLK); + data |= NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_REG_DIS_LCLK_MASK; + if (def != data) + WREG32_PCIE(smnNBIF_MGCG_CTRL_LCLK, data); + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL3); + data |= PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK_MASK; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL3, data); + + def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP3); + data &= ~RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK; + data &= ~RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK; + if (def != data) + WREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP3, data); + + def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP5); + data &= ~RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK; + if (def != data) + WREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP5, data); + + def = data = RREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2); + data &= ~BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN_MASK; + if (def != data) + WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2, data); + + WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_PCIE_LTR_CAP, 0x10011001); + + def = data = RREG32_PCIE(smnPSWUSP0_PCIE_LC_CNTL2); + data |= PSWUSP0_PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK | + PSWUSP0_PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK; + data &= ~PSWUSP0_PCIE_LC_CNTL2__LC_RCV_L0_TO_RCV_L0S_DIS_MASK; + if (def != data) + WREG32_PCIE(smnPSWUSP0_PCIE_LC_CNTL2, data); + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL6); + data |= PCIE_LC_CNTL6__LC_L1_POWERDOWN_MASK | + PCIE_LC_CNTL6__LC_RX_L0S_STANDBY_EN_MASK; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL6, data); + + nbio_v2_3_program_ltr(adev); + + def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP3); + data |= 0x5DE0 << RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT; + data |= 0x0010 << RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT; + if (def != data) + WREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP3, data); + + def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP5); + data |= 0x0010 << RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT; + if (def != data) + WREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP5, data); + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL); + data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK; + data |= 0x9 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT; + data |= 0x1 << PCIE_LC_CNTL__LC_PMI_TO_L1_DIS__SHIFT; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL, data); + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL3); + data &= ~PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK_MASK; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL3, data); +} + const struct amdgpu_nbio_funcs nbio_v2_3_funcs = { .get_hdp_flush_req_offset = nbio_v2_3_get_hdp_flush_req_offset, .get_hdp_flush_done_offset = nbio_v2_3_get_hdp_flush_done_offset, @@ -366,7 +470,6 @@ const struct amdgpu_nbio_funcs nbio_v2_3_funcs = { .get_pcie_data_offset = nbio_v2_3_get_pcie_data_offset, .get_rev_id = nbio_v2_3_get_rev_id, .mc_access_enable = nbio_v2_3_mc_access_enable, - .hdp_flush = nbio_v2_3_hdp_flush, .get_memsize = nbio_v2_3_get_memsize, .sdma_doorbell_range = nbio_v2_3_sdma_doorbell_range, .vcn_doorbell_range = nbio_v2_3_vcn_doorbell_range, @@ -380,4 +483,5 @@ const struct amdgpu_nbio_funcs nbio_v2_3_funcs = { .init_registers = nbio_v2_3_init_registers, .remap_hdp_registers = nbio_v2_3_remap_hdp_registers, .enable_aspm = nbio_v2_3_enable_aspm, + .program_aspm = nbio_v2_3_program_aspm, }; diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c index d2f1fe55d388..83ea063388fd 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c @@ -29,6 +29,15 @@ #include "nbio/nbio_6_1_sh_mask.h" #include "nbio/nbio_6_1_smn.h" #include "vega10_enum.h" +#include + +static void nbio_v6_1_remap_hdp_registers(struct amdgpu_device *adev) +{ + WREG32_SOC15(NBIO, 0, mmREMAP_HDP_MEM_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL); + WREG32_SOC15(NBIO, 0, mmREMAP_HDP_REG_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL); +} static u32 nbio_v6_1_get_rev_id(struct amdgpu_device *adev) { @@ -50,18 +59,6 @@ static void nbio_v6_1_mc_access_enable(struct amdgpu_device *adev, bool enable) WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0); } -static void nbio_v6_1_hdp_flush(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_SOC15_NO_KIQ(NBIO, 0, - mmBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL, - 0); - else - amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( - NBIO, 0, mmBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL), 0); -} - static u32 nbio_v6_1_get_memsize(struct amdgpu_device *adev) { return RREG32_SOC15(NBIO, 0, mmRCC_PF_0_0_RCC_CONFIG_MEMSIZE); @@ -266,7 +263,6 @@ const struct amdgpu_nbio_funcs nbio_v6_1_funcs = { .get_pcie_data_offset = nbio_v6_1_get_pcie_data_offset, .get_rev_id = nbio_v6_1_get_rev_id, .mc_access_enable = nbio_v6_1_mc_access_enable, - .hdp_flush = nbio_v6_1_hdp_flush, .get_memsize = nbio_v6_1_get_memsize, .sdma_doorbell_range = nbio_v6_1_sdma_doorbell_range, .enable_doorbell_aperture = nbio_v6_1_enable_doorbell_aperture, @@ -277,4 +273,5 @@ const struct amdgpu_nbio_funcs nbio_v6_1_funcs = { .get_clockgating_state = nbio_v6_1_get_clockgating_state, .ih_control = nbio_v6_1_ih_control, .init_registers = nbio_v6_1_init_registers, + .remap_hdp_registers = nbio_v6_1_remap_hdp_registers, }; diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c index ae685813c419..3c00666a13e1 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c @@ -60,15 +60,6 @@ static void nbio_v7_0_mc_access_enable(struct amdgpu_device *adev, bool enable) WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0); } -static void nbio_v7_0_hdp_flush(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - else - amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); -} - static u32 nbio_v7_0_get_memsize(struct amdgpu_device *adev) { return RREG32_SOC15(NBIO, 0, mmRCC_CONFIG_MEMSIZE); @@ -292,7 +283,6 @@ const struct amdgpu_nbio_funcs nbio_v7_0_funcs = { .get_pcie_data_offset = nbio_v7_0_get_pcie_data_offset, .get_rev_id = nbio_v7_0_get_rev_id, .mc_access_enable = nbio_v7_0_mc_access_enable, - .hdp_flush = nbio_v7_0_hdp_flush, .get_memsize = nbio_v7_0_get_memsize, .sdma_doorbell_range = nbio_v7_0_sdma_doorbell_range, .vcn_doorbell_range = nbio_v7_0_vcn_doorbell_range, diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_2.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_2.c index aa36022670f9..598ce0e93627 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_2.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_2.c @@ -56,15 +56,6 @@ static void nbio_v7_2_mc_access_enable(struct amdgpu_device *adev, bool enable) WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0); } -static void nbio_v7_2_hdp_flush(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - else - amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); -} - static u32 nbio_v7_2_get_memsize(struct amdgpu_device *adev) { return RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE); @@ -325,7 +316,6 @@ const struct amdgpu_nbio_funcs nbio_v7_2_funcs = { .get_pcie_port_data_offset = nbio_v7_2_get_pcie_port_data_offset, .get_rev_id = nbio_v7_2_get_rev_id, .mc_access_enable = nbio_v7_2_mc_access_enable, - .hdp_flush = nbio_v7_2_hdp_flush, .get_memsize = nbio_v7_2_get_memsize, .sdma_doorbell_range = nbio_v7_2_sdma_doorbell_range, .vcn_doorbell_range = nbio_v7_2_vcn_doorbell_range, diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c index eadc9526d33f..4bc1d1434065 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c @@ -82,15 +82,6 @@ static void nbio_v7_4_mc_access_enable(struct amdgpu_device *adev, bool enable) WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0); } -static void nbio_v7_4_hdp_flush(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - else - amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); -} - static u32 nbio_v7_4_get_memsize(struct amdgpu_device *adev) { return RREG32_SOC15(NBIO, 0, mmRCC_CONFIG_MEMSIZE); @@ -541,7 +532,6 @@ const struct amdgpu_nbio_funcs nbio_v7_4_funcs = { .get_pcie_data_offset = nbio_v7_4_get_pcie_data_offset, .get_rev_id = nbio_v7_4_get_rev_id, .mc_access_enable = nbio_v7_4_mc_access_enable, - .hdp_flush = nbio_v7_4_hdp_flush, .get_memsize = nbio_v7_4_get_memsize, .sdma_doorbell_range = nbio_v7_4_sdma_doorbell_range, .vcn_doorbell_range = nbio_v7_4_vcn_doorbell_range, diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c index 6bee3677394a..160fa5f59805 100644 --- a/drivers/gpu/drm/amd/amdgpu/nv.c +++ b/drivers/gpu/drm/amd/amdgpu/nv.c @@ -38,9 +38,6 @@ #include "gc/gc_10_1_0_offset.h" #include "gc/gc_10_1_0_sh_mask.h" -#include "hdp/hdp_5_0_0_offset.h" -#include "hdp/hdp_5_0_0_sh_mask.h" -#include "smuio/smuio_11_0_0_offset.h" #include "mp/mp_11_0_offset.h" #include "soc15.h" @@ -50,6 +47,7 @@ #include "mmhub_v2_0.h" #include "nbio_v2_3.h" #include "nbio_v7_2.h" +#include "hdp_v5_0.h" #include "nv.h" #include "navi10_ih.h" #include "gfx_v10_0.h" @@ -62,6 +60,8 @@ #include "dce_virtual.h" #include "mes_v10_1.h" #include "mxgpu_nv.h" +#include "smuio_v11_0.h" +#include "smuio_v11_0_6.h" static const struct amd_ip_funcs nv_common_ip_funcs; @@ -203,6 +203,7 @@ static bool nv_read_bios_from_rom(struct amdgpu_device *adev, { u32 *dw_ptr; u32 i, length_dw; + u32 rom_index_offset, rom_data_offset; if (bios == NULL) return false; @@ -215,11 +216,16 @@ static bool nv_read_bios_from_rom(struct amdgpu_device *adev, dw_ptr = (u32 *)bios; length_dw = ALIGN(length_bytes, 4) / 4; + rom_index_offset = + adev->smuio.funcs->get_rom_index_offset(adev); + rom_data_offset = + adev->smuio.funcs->get_rom_data_offset(adev); + /* set rom index to 0 */ - WREG32(SOC15_REG_OFFSET(SMUIO, 0, mmROM_INDEX), 0); + WREG32(rom_index_offset, 0); /* read out the rom data */ for (i = 0; i < length_dw; i++) - dw_ptr[i] = RREG32(SOC15_REG_OFFSET(SMUIO, 0, mmROM_DATA)); + dw_ptr[i] = RREG32(rom_data_offset); return true; } @@ -336,6 +342,38 @@ static int nv_asic_mode1_reset(struct amdgpu_device *adev) return ret; } +static int nv_asic_mode2_reset(struct amdgpu_device *adev) +{ + u32 i; + int ret = 0; + + amdgpu_atombios_scratch_regs_engine_hung(adev, true); + + /* disable BM */ + pci_clear_master(adev->pdev); + + amdgpu_device_cache_pci_state(adev->pdev); + + ret = amdgpu_dpm_mode2_reset(adev); + if (ret) + dev_err(adev->dev, "GPU mode2 reset failed\n"); + + amdgpu_device_load_pci_state(adev->pdev); + + /* wait for asic to come out of reset */ + for (i = 0; i < adev->usec_timeout; i++) { + u32 memsize = adev->nbio.funcs->get_memsize(adev); + + if (memsize != 0xffffffff) + break; + udelay(1); + } + + amdgpu_atombios_scratch_regs_engine_hung(adev, false); + + return ret; +} + static bool nv_asic_supports_baco(struct amdgpu_device *adev) { struct smu_context *smu = &adev->smu; @@ -352,7 +390,9 @@ nv_asic_reset_method(struct amdgpu_device *adev) struct smu_context *smu = &adev->smu; if (amdgpu_reset_method == AMD_RESET_METHOD_MODE1 || - amdgpu_reset_method == AMD_RESET_METHOD_BACO) + amdgpu_reset_method == AMD_RESET_METHOD_MODE2 || + amdgpu_reset_method == AMD_RESET_METHOD_BACO || + amdgpu_reset_method == AMD_RESET_METHOD_PCI) return amdgpu_reset_method; if (amdgpu_reset_method != -1) @@ -360,6 +400,8 @@ nv_asic_reset_method(struct amdgpu_device *adev) amdgpu_reset_method); switch (adev->asic_type) { + case CHIP_VANGOGH: + return AMD_RESET_METHOD_MODE2; case CHIP_SIENNA_CICHLID: case CHIP_NAVY_FLOUNDER: case CHIP_DIMGREY_CAVEFISH: @@ -377,7 +419,16 @@ static int nv_asic_reset(struct amdgpu_device *adev) int ret = 0; struct smu_context *smu = &adev->smu; - if (nv_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { + /* skip reset on vangogh for now */ + if (adev->asic_type == CHIP_VANGOGH) + return 0; + + switch (nv_asic_reset_method(adev)) { + case AMD_RESET_METHOD_PCI: + dev_info(adev->dev, "PCI reset\n"); + ret = amdgpu_device_pci_reset(adev); + break; + case AMD_RESET_METHOD_BACO: dev_info(adev->dev, "BACO reset\n"); ret = smu_baco_enter(smu); @@ -386,9 +437,15 @@ static int nv_asic_reset(struct amdgpu_device *adev) ret = smu_baco_exit(smu); if (ret) return ret; - } else { + break; + case AMD_RESET_METHOD_MODE2: + dev_info(adev->dev, "MODE2 reset\n"); + ret = nv_asic_mode2_reset(adev); + break; + default: dev_info(adev->dev, "MODE1 reset\n"); ret = nv_asic_mode1_reset(adev); + break; } return ret; @@ -423,11 +480,14 @@ static void nv_pcie_gen3_enable(struct amdgpu_device *adev) static void nv_program_aspm(struct amdgpu_device *adev) { - - if (amdgpu_aspm == 0) + if (amdgpu_aspm != 1) return; - /* todo */ + if ((adev->asic_type >= CHIP_SIENNA_CICHLID) && + !(adev->flags & AMD_IS_APU) && + (adev->nbio.funcs->program_aspm)) + adev->nbio.funcs->program_aspm(adev); + } static void nv_enable_doorbell_aperture(struct amdgpu_device *adev, @@ -514,6 +574,12 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) adev->nbio.funcs = &nbio_v2_3_funcs; adev->nbio.hdp_flush_reg = &nbio_v2_3_hdp_flush_reg; } + adev->hdp.funcs = &hdp_v5_0_funcs; + + if (adev->asic_type >= CHIP_SIENNA_CICHLID) + adev->smuio.funcs = &smuio_v11_0_6_funcs; + else + adev->smuio.funcs = &smuio_v11_0_funcs; if (adev->asic_type == CHIP_SIENNA_CICHLID) adev->gmc.xgmi.supported = true; @@ -669,22 +735,6 @@ static uint32_t nv_get_rev_id(struct amdgpu_device *adev) return adev->nbio.funcs->get_rev_id(adev); } -static void nv_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring) -{ - adev->nbio.funcs->hdp_flush(adev, ring); -} - -static void nv_invalidate_hdp(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) { - WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); - } else { - amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( - HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); - } -} - static bool nv_need_full_reset(struct amdgpu_device *adev) { return true; @@ -768,10 +818,10 @@ static int nv_update_umd_stable_pstate(struct amdgpu_device *adev, * The ASPM function is not fully enabled and verified on * Navi yet. Temporarily skip this until ASPM enabled. */ -#if 0 - if (adev->nbio.funcs->enable_aspm) + if ((adev->asic_type >= CHIP_SIENNA_CICHLID) && + !(adev->flags & AMD_IS_APU) && + (adev->nbio.funcs->enable_aspm)) adev->nbio.funcs->enable_aspm(adev, !enter); -#endif return 0; } @@ -788,8 +838,6 @@ static const struct amdgpu_asic_funcs nv_asic_funcs = .set_uvd_clocks = &nv_set_uvd_clocks, .set_vce_clocks = &nv_set_vce_clocks, .get_config_memsize = &nv_get_config_memsize, - .flush_hdp = &nv_flush_hdp, - .invalidate_hdp = &nv_invalidate_hdp, .init_doorbell_index = &nv_init_doorbell_index, .need_full_reset = &nv_need_full_reset, .need_reset_on_init = &nv_need_reset_on_init, @@ -1080,120 +1128,6 @@ static int nv_common_soft_reset(void *handle) return 0; } -static void nv_update_hdp_mem_power_gating(struct amdgpu_device *adev, - bool enable) -{ - uint32_t hdp_clk_cntl, hdp_clk_cntl1; - uint32_t hdp_mem_pwr_cntl; - - if (!(adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | - AMD_CG_SUPPORT_HDP_DS | - AMD_CG_SUPPORT_HDP_SD))) - return; - - hdp_clk_cntl = hdp_clk_cntl1 = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); - hdp_mem_pwr_cntl = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL); - - /* Before doing clock/power mode switch, - * forced on IPH & RC clock */ - hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL, - IPH_MEM_CLK_SOFT_OVERRIDE, 1); - hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL, - RC_MEM_CLK_SOFT_OVERRIDE, 1); - WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl); - - /* HDP 5.0 doesn't support dynamic power mode switch, - * disable clock and power gating before any changing */ - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_CTRL_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_LS_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_DS_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_SD_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - RC_MEM_POWER_CTRL_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - RC_MEM_POWER_LS_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - RC_MEM_POWER_DS_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - RC_MEM_POWER_SD_EN, 0); - WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl); - - /* only one clock gating mode (LS/DS/SD) can be enabled */ - if (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS) { - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_LS_EN, enable); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - RC_MEM_POWER_LS_EN, enable); - } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_DS) { - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_DS_EN, enable); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - RC_MEM_POWER_DS_EN, enable); - } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_SD) { - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_SD_EN, enable); - /* RC should not use shut down mode, fallback to ds */ - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - RC_MEM_POWER_DS_EN, enable); - } - - /* confirmed that IPH_MEM_POWER_CTRL_EN and RC_MEM_POWER_CTRL_EN have to - * be set for SRAM LS/DS/SD */ - if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_DS | - AMD_CG_SUPPORT_HDP_SD)) { - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_CTRL_EN, 1); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - RC_MEM_POWER_CTRL_EN, 1); - } - - WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl); - - /* restore IPH & RC clock override after clock/power mode changing */ - WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl1); -} - -static void nv_update_hdp_clock_gating(struct amdgpu_device *adev, - bool enable) -{ - uint32_t hdp_clk_cntl; - - if (!(adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG)) - return; - - hdp_clk_cntl = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); - - if (enable) { - hdp_clk_cntl &= - ~(uint32_t) - (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK); - } else { - hdp_clk_cntl |= HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK; - } - - WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl); -} - static int nv_common_set_clockgating_state(void *handle, enum amd_clockgating_state state) { @@ -1213,9 +1147,9 @@ static int nv_common_set_clockgating_state(void *handle, state == AMD_CG_STATE_GATE); adev->nbio.funcs->update_medium_grain_light_sleep(adev, state == AMD_CG_STATE_GATE); - nv_update_hdp_mem_power_gating(adev, - state == AMD_CG_STATE_GATE); - nv_update_hdp_clock_gating(adev, + adev->hdp.funcs->update_clock_gating(adev, + state == AMD_CG_STATE_GATE); + adev->smuio.funcs->update_rom_clock_gating(adev, state == AMD_CG_STATE_GATE); break; default: @@ -1234,31 +1168,15 @@ static int nv_common_set_powergating_state(void *handle, static void nv_common_get_clockgating_state(void *handle, u32 *flags) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - uint32_t tmp; if (amdgpu_sriov_vf(adev)) *flags = 0; adev->nbio.funcs->get_clockgating_state(adev, flags); - /* AMD_CG_SUPPORT_HDP_MGCG */ - tmp = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); - if (!(tmp & (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK))) - *flags |= AMD_CG_SUPPORT_HDP_MGCG; + adev->hdp.funcs->get_clock_gating_state(adev, flags); - /* AMD_CG_SUPPORT_HDP_LS/DS/SD */ - tmp = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL); - if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK) - *flags |= AMD_CG_SUPPORT_HDP_LS; - else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_DS_EN_MASK) - *flags |= AMD_CG_SUPPORT_HDP_DS; - else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_SD_EN_MASK) - *flags |= AMD_CG_SUPPORT_HDP_SD; + adev->smuio.funcs->get_clock_gating_state(adev, flags); return; } diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c index d7f92634eba2..4b1cc5e9ee92 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c @@ -92,8 +92,6 @@ static int psp_v10_0_init_microcode(struct psp_context *psp) (uint8_t *)ta_hdr + le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes); - adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version); - adev->psp.ta_dtm_ucode_version = le32_to_cpu(ta_hdr->ta_dtm_ucode_version); adev->psp.ta_dtm_ucode_size = @@ -101,6 +99,16 @@ static int psp_v10_0_init_microcode(struct psp_context *psp) adev->psp.ta_dtm_start_addr = (uint8_t *)adev->psp.ta_hdcp_start_addr + le32_to_cpu(ta_hdr->ta_dtm_offset_bytes); + + adev->psp.ta_securedisplay_ucode_version = + le32_to_cpu(ta_hdr->ta_securedisplay_ucode_version); + adev->psp.ta_securedisplay_ucode_size = + le32_to_cpu(ta_hdr->ta_securedisplay_size_bytes); + adev->psp.ta_securedisplay_start_addr = + (uint8_t *)adev->psp.ta_hdcp_start_addr + + le32_to_cpu(ta_hdr->ta_securedisplay_offset_bytes); + + adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version); } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index bd4248c93c49..c325d6f53a71 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -392,37 +392,6 @@ static int psp_v11_0_bootloader_load_sos(struct psp_context *psp) return ret; } -static void psp_v11_0_reroute_ih(struct psp_context *psp) -{ - struct amdgpu_device *adev = psp->adev; - uint32_t tmp; - - /* Change IH ring for VMC */ - tmp = REG_SET_FIELD(0, IH_CLIENT_CFG_DATA, CREDIT_RETURN_ADDR, 0x1244b); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); - - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, 3); - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_70, tmp); - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, GFX_CTRL_CMD_ID_GBR_IH_SET); - - mdelay(20); - psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64), - 0x80000000, 0x8000FFFF, false); - - /* Change IH ring for UMC */ - tmp = REG_SET_FIELD(0, IH_CLIENT_CFG_DATA, CREDIT_RETURN_ADDR, 0x1216b); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); - - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, 4); - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_70, tmp); - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, GFX_CTRL_CMD_ID_GBR_IH_SET); - - mdelay(20); - psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64), - 0x80000000, 0x8000FFFF, false); -} - static int psp_v11_0_ring_init(struct psp_context *psp, enum psp_ring_type ring_type) { @@ -430,11 +399,6 @@ static int psp_v11_0_ring_init(struct psp_context *psp, struct psp_ring *ring; struct amdgpu_device *adev = psp->adev; - if ((!amdgpu_sriov_vf(adev)) && - !(adev->asic_type >= CHIP_SIENNA_CICHLID && - adev->asic_type <= CHIP_DIMGREY_CAVEFISH)) - psp_v11_0_reroute_ih(psp); - ring = &psp->km_ring; ring->ring_type = ring_type; @@ -726,7 +690,7 @@ static int psp_v11_0_memory_training(struct psp_context *psp, uint32_t ops) } memcpy_toio(adev->mman.aper_base_kaddr, buf, sz); - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); vfree(buf); } diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index ce56e93c6886..c8c22c1d1e65 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -46,7 +46,6 @@ #include "sdma6/sdma6_4_2_2_sh_mask.h" #include "sdma7/sdma7_4_2_2_offset.h" #include "sdma7/sdma7_4_2_2_sh_mask.h" -#include "hdp/hdp_4_0_offset.h" #include "sdma0/sdma0_4_1_default.h" #include "soc15_common.h" diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index b208b81005bb..d345e324837d 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -32,7 +32,6 @@ #include "gc/gc_10_1_0_offset.h" #include "gc/gc_10_1_0_sh_mask.h" -#include "hdp/hdp_5_0_0_offset.h" #include "ivsrcid/sdma0/irqsrcs_sdma0_5_0.h" #include "ivsrcid/sdma1/irqsrcs_sdma1_5_0.h" diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index f1ba36a094da..690a5090475a 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -119,15 +119,7 @@ static int sdma_v5_2_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst) static void sdma_v5_2_destroy_inst_ctx(struct amdgpu_device *adev) { - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - release_firmware(adev->sdma.instance[i].fw); - adev->sdma.instance[i].fw = NULL; - - if (adev->asic_type == CHIP_SIENNA_CICHLID) - break; - } + release_firmware(adev->sdma.instance[0].fw); memset((void *)adev->sdma.instance, 0, sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES); @@ -185,23 +177,10 @@ static int sdma_v5_2_init_microcode(struct amdgpu_device *adev) if (err) goto out; - for (i = 1; i < adev->sdma.num_instances; i++) { - if (adev->asic_type >= CHIP_SIENNA_CICHLID && - adev->asic_type <= CHIP_DIMGREY_CAVEFISH) { - memcpy((void *)&adev->sdma.instance[i], - (void *)&adev->sdma.instance[0], - sizeof(struct amdgpu_sdma_instance)); - } else { - snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma%d.bin", chip_name, i); - err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev); - if (err) - goto out; - - err = sdma_v5_2_init_inst_ctx(&adev->sdma.instance[i]); - if (err) - goto out; - } - } + for (i = 1; i < adev->sdma.num_instances; i++) + memcpy((void *)&adev->sdma.instance[i], + (void *)&adev->sdma.instance[0], + sizeof(struct amdgpu_sdma_instance)); DRM_DEBUG("psp_load == '%s'\n", adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false"); diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c index 3cf0589bfea5..6b5cf7882a12 100644 --- a/drivers/gpu/drm/amd/amdgpu/si.c +++ b/drivers/gpu/drm/amd/amdgpu/si.c @@ -1270,7 +1270,7 @@ static int si_gpu_pci_config_reset(struct amdgpu_device *adev) u32 i; int r = -EINVAL; - dev_info(adev->dev, "GPU pci config reset\n"); + amdgpu_atombios_scratch_regs_engine_hung(adev, true); /* set mclk/sclk to bypass */ si_set_clk_bypass_mode(adev); @@ -1294,20 +1294,6 @@ static int si_gpu_pci_config_reset(struct amdgpu_device *adev) } udelay(1); } - - return r; -} - -static int si_asic_reset(struct amdgpu_device *adev) -{ - int r; - - dev_info(adev->dev, "PCI CONFIG reset\n"); - - amdgpu_atombios_scratch_regs_engine_hung(adev, true); - - r = si_gpu_pci_config_reset(adev); - amdgpu_atombios_scratch_regs_engine_hung(adev, false); return r; @@ -1321,14 +1307,34 @@ static bool si_asic_supports_baco(struct amdgpu_device *adev) static enum amd_reset_method si_asic_reset_method(struct amdgpu_device *adev) { - if (amdgpu_reset_method != AMD_RESET_METHOD_LEGACY && - amdgpu_reset_method != -1) + if (amdgpu_reset_method == AMD_RESET_METHOD_PCI) + return amdgpu_reset_method; + else if (amdgpu_reset_method != AMD_RESET_METHOD_LEGACY && + amdgpu_reset_method != -1) dev_warn(adev->dev, "Specified reset method:%d isn't supported, using AUTO instead.\n", - amdgpu_reset_method); + amdgpu_reset_method); return AMD_RESET_METHOD_LEGACY; } +static int si_asic_reset(struct amdgpu_device *adev) +{ + int r; + + switch (si_asic_reset_method(adev)) { + case AMD_RESET_METHOD_PCI: + dev_info(adev->dev, "PCI reset\n"); + r = amdgpu_device_pci_reset(adev); + break; + default: + dev_info(adev->dev, "PCI CONFIG reset\n"); + r = si_gpu_pci_config_reset(adev); + break; + } + + return r; +} + static u32 si_get_config_memsize(struct amdgpu_device *adev) { return RREG32(mmCONFIG_MEMSIZE); diff --git a/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.c b/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.c new file mode 100644 index 000000000000..3a18dbb55c32 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.c @@ -0,0 +1,77 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "amdgpu.h" +#include "smuio_v11_0_6.h" +#include "smuio/smuio_11_0_6_offset.h" +#include "smuio/smuio_11_0_6_sh_mask.h" + +static u32 smuio_v11_0_6_get_rom_index_offset(struct amdgpu_device *adev) +{ + return SOC15_REG_OFFSET(SMUIO, 0, mmROM_INDEX); +} + +static u32 smuio_v11_0_6_get_rom_data_offset(struct amdgpu_device *adev) +{ + return SOC15_REG_OFFSET(SMUIO, 0, mmROM_DATA); +} + +static void smuio_v11_0_6_update_rom_clock_gating(struct amdgpu_device *adev, bool enable) +{ + u32 def, data; + + /* enable/disable ROM CG is not supported on APU */ + if (adev->flags & AMD_IS_APU) + return; + + def = data = RREG32_SOC15(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0); + + if (enable && (adev->cg_flags & AMD_CG_SUPPORT_ROM_MGCG)) + data &= ~(CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK | + CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK); + else + data |= CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK | + CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK; + + if (def != data) + WREG32_SOC15(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0, data); +} + +static void smuio_v11_0_6_get_clock_gating_state(struct amdgpu_device *adev, u32 *flags) +{ + u32 data; + + /* CGTT_ROM_CLK_CTRL0 is not available for APU */ + if (adev->flags & AMD_IS_APU) + return; + + data = RREG32_SOC15(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0); + if (!(data & CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK)) + *flags |= AMD_CG_SUPPORT_ROM_MGCG; +} + +const struct amdgpu_smuio_funcs smuio_v11_0_6_funcs = { + .get_rom_index_offset = smuio_v11_0_6_get_rom_index_offset, + .get_rom_data_offset = smuio_v11_0_6_get_rom_data_offset, + .update_rom_clock_gating = smuio_v11_0_6_update_rom_clock_gating, + .get_clock_gating_state = smuio_v11_0_6_get_clock_gating_state, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.h b/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.h new file mode 100644 index 000000000000..3c3f4ab0bc9b --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.h @@ -0,0 +1,30 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __SMUIO_V11_0_6_H__ +#define __SMUIO_V11_0_6_H__ + +#include "soc15_common.h" + +extern const struct amdgpu_smuio_funcs smuio_v11_0_6_funcs; + +#endif /* __SMUIO_V11_0_6_H__ */ diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index 0b3516c4eefb..1221aa6b40a9 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -40,8 +40,6 @@ #include "gc/gc_9_0_sh_mask.h" #include "sdma0/sdma0_4_0_offset.h" #include "sdma1/sdma1_4_0_offset.h" -#include "hdp/hdp_4_0_offset.h" -#include "hdp/hdp_4_0_sh_mask.h" #include "nbio/nbio_7_0_default.h" #include "nbio/nbio_7_0_offset.h" #include "nbio/nbio_7_0_sh_mask.h" @@ -59,7 +57,9 @@ #include "nbio_v6_1.h" #include "nbio_v7_0.h" #include "nbio_v7_4.h" +#include "hdp_v4_0.h" #include "vega10_ih.h" +#include "vega20_ih.h" #include "navi10_ih.h" #include "sdma_v4_0.h" #include "uvd_v7_0.h" @@ -83,14 +83,6 @@ #define mmMP0_MISC_LIGHT_SLEEP_CTRL 0x01ba #define mmMP0_MISC_LIGHT_SLEEP_CTRL_BASE_IDX 0 -/* for Vega20 register name change */ -#define mmHDP_MEM_POWER_CTRL 0x00d4 -#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK 0x00000001L -#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK 0x00000002L -#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK 0x00010000L -#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK 0x00020000L -#define mmHDP_MEM_POWER_CTRL_BASE_IDX 0 - /* * Indirect registers accessor */ @@ -241,6 +233,8 @@ static u32 soc15_get_xclk(struct amdgpu_device *adev) { u32 reference_clock = adev->clock.spll.reference_freq; + if (adev->asic_type == CHIP_RENOIR) + return 10000; if (adev->asic_type == CHIP_RAVEN) return reference_clock / 4; @@ -487,7 +481,8 @@ soc15_asic_reset_method(struct amdgpu_device *adev) if (amdgpu_reset_method == AMD_RESET_METHOD_MODE1 || amdgpu_reset_method == AMD_RESET_METHOD_MODE2 || - amdgpu_reset_method == AMD_RESET_METHOD_BACO) + amdgpu_reset_method == AMD_RESET_METHOD_BACO || + amdgpu_reset_method == AMD_RESET_METHOD_PCI) return amdgpu_reset_method; if (amdgpu_reset_method != -1) @@ -532,15 +527,18 @@ static int soc15_asic_reset(struct amdgpu_device *adev) return 0; switch (soc15_asic_reset_method(adev)) { - case AMD_RESET_METHOD_BACO: - dev_info(adev->dev, "BACO reset\n"); - return soc15_asic_baco_reset(adev); - case AMD_RESET_METHOD_MODE2: - dev_info(adev->dev, "MODE2 reset\n"); - return amdgpu_dpm_mode2_reset(adev); - default: - dev_info(adev->dev, "MODE1 reset\n"); - return soc15_asic_mode1_reset(adev); + case AMD_RESET_METHOD_PCI: + dev_info(adev->dev, "PCI reset\n"); + return amdgpu_device_pci_reset(adev); + case AMD_RESET_METHOD_BACO: + dev_info(adev->dev, "BACO reset\n"); + return soc15_asic_baco_reset(adev); + case AMD_RESET_METHOD_MODE2: + dev_info(adev->dev, "MODE2 reset\n"); + return amdgpu_dpm_mode2_reset(adev); + default: + dev_info(adev->dev, "MODE1 reset\n"); + return soc15_asic_mode1_reset(adev); } } @@ -699,6 +697,7 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) adev->nbio.funcs = &nbio_v6_1_funcs; adev->nbio.hdp_flush_reg = &nbio_v6_1_hdp_flush_reg; } + adev->hdp.funcs = &hdp_v4_0_funcs; if (adev->asic_type == CHIP_VEGA20 || adev->asic_type == CHIP_ARCTURUS) adev->df.funcs = &df_v3_6_funcs; @@ -729,12 +728,12 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block); } if (adev->asic_type == CHIP_VEGA20) - amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block); + amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block); else amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block); } else { if (adev->asic_type == CHIP_VEGA20) - amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block); + amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block); else amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block); if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { @@ -787,9 +786,9 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) if (amdgpu_sriov_vf(adev)) { if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block); - amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block); + amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block); } else { - amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block); + amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block); if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block); } @@ -834,35 +833,12 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) return 0; } -static void soc15_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring) -{ - adev->nbio.funcs->hdp_flush(adev, ring); -} - -static void soc15_invalidate_hdp(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); - else - amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( - HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); -} - static bool soc15_need_full_reset(struct amdgpu_device *adev) { /* change this when we implement soft reset */ return true; } -static void vega20_reset_hdp_ras_error_count(struct amdgpu_device *adev) -{ - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP)) - return; - /*read back hdp ras counter to reset it to 0 */ - RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT); -} - static void soc15_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, uint64_t *count1) { @@ -1011,8 +987,6 @@ static const struct amdgpu_asic_funcs soc15_asic_funcs = .set_uvd_clocks = &soc15_set_uvd_clocks, .set_vce_clocks = &soc15_set_vce_clocks, .get_config_memsize = &soc15_get_config_memsize, - .flush_hdp = &soc15_flush_hdp, - .invalidate_hdp = &soc15_invalidate_hdp, .need_full_reset = &soc15_need_full_reset, .init_doorbell_index = &vega10_doorbell_index_init, .get_pcie_usage = &soc15_get_pcie_usage, @@ -1034,9 +1008,6 @@ static const struct amdgpu_asic_funcs vega20_asic_funcs = .set_uvd_clocks = &soc15_set_uvd_clocks, .set_vce_clocks = &soc15_set_vce_clocks, .get_config_memsize = &soc15_get_config_memsize, - .flush_hdp = &soc15_flush_hdp, - .invalidate_hdp = &soc15_invalidate_hdp, - .reset_hdp_ras_error_count = &vega20_reset_hdp_ras_error_count, .need_full_reset = &soc15_need_full_reset, .init_doorbell_index = &vega20_doorbell_index_init, .get_pcie_usage = &vega20_get_pcie_usage, @@ -1294,9 +1265,8 @@ static int soc15_common_late_init(void *handle) if (amdgpu_sriov_vf(adev)) xgpu_ai_mailbox_get_irq(adev); - if (adev->asic_funcs && - adev->asic_funcs->reset_hdp_ras_error_count) - adev->asic_funcs->reset_hdp_ras_error_count(adev); + if (adev->hdp.funcs->reset_ras_error_count) + adev->hdp.funcs->reset_ras_error_count(adev); if (adev->nbio.funcs->ras_late_init) r = adev->nbio.funcs->ras_late_init(adev); @@ -1422,41 +1392,6 @@ static int soc15_common_soft_reset(void *handle) return 0; } -static void soc15_update_hdp_light_sleep(struct amdgpu_device *adev, bool enable) -{ - uint32_t def, data; - - if (adev->asic_type == CHIP_VEGA20 || - adev->asic_type == CHIP_ARCTURUS || - adev->asic_type == CHIP_RENOIR) { - def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL)); - - if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) - data |= HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | - HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | - HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | - HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK; - else - data &= ~(HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | - HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | - HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | - HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK); - - if (def != data) - WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL), data); - } else { - def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); - - if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) - data |= HDP_MEM_POWER_LS__LS_ENABLE_MASK; - else - data &= ~HDP_MEM_POWER_LS__LS_ENABLE_MASK; - - if (def != data) - WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS), data); - } -} - static void soc15_update_drm_clock_gating(struct amdgpu_device *adev, bool enable) { uint32_t def, data; @@ -1517,7 +1452,7 @@ static int soc15_common_set_clockgating_state(void *handle, state == AMD_CG_STATE_GATE); adev->nbio.funcs->update_medium_grain_light_sleep(adev, state == AMD_CG_STATE_GATE); - soc15_update_hdp_light_sleep(adev, + adev->hdp.funcs->update_clock_gating(adev, state == AMD_CG_STATE_GATE); soc15_update_drm_clock_gating(adev, state == AMD_CG_STATE_GATE); @@ -1534,7 +1469,7 @@ static int soc15_common_set_clockgating_state(void *handle, state == AMD_CG_STATE_GATE); adev->nbio.funcs->update_medium_grain_light_sleep(adev, state == AMD_CG_STATE_GATE); - soc15_update_hdp_light_sleep(adev, + adev->hdp.funcs->update_clock_gating(adev, state == AMD_CG_STATE_GATE); soc15_update_drm_clock_gating(adev, state == AMD_CG_STATE_GATE); @@ -1542,7 +1477,7 @@ static int soc15_common_set_clockgating_state(void *handle, state == AMD_CG_STATE_GATE); break; case CHIP_ARCTURUS: - soc15_update_hdp_light_sleep(adev, + adev->hdp.funcs->update_clock_gating(adev, state == AMD_CG_STATE_GATE); break; default: @@ -1561,10 +1496,7 @@ static void soc15_common_get_clockgating_state(void *handle, u32 *flags) adev->nbio.funcs->get_clockgating_state(adev, flags); - /* AMD_CG_SUPPORT_HDP_LS */ - data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); - if (data & HDP_MEM_POWER_LS__LS_ENABLE_MASK) - *flags |= AMD_CG_SUPPORT_HDP_LS; + adev->hdp.funcs->get_clock_gating_state(adev, flags); /* AMD_CG_SUPPORT_DRM_MGCG */ data = RREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_CGTT_CTRL0)); diff --git a/drivers/gpu/drm/amd/amdgpu/ta_secureDisplay_if.h b/drivers/gpu/drm/amd/amdgpu/ta_secureDisplay_if.h new file mode 100644 index 000000000000..5039375bb1d4 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/ta_secureDisplay_if.h @@ -0,0 +1,154 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _TA_SECUREDISPLAY_IF_H +#define _TA_SECUREDISPLAY_IF_H + +/** Secure Display related enumerations */ +/**********************************************************/ + +/** @enum ta_securedisplay_command + * Secure Display Command ID + */ +enum ta_securedisplay_command { + /* Query whether TA is responding used only for validation purpose */ + TA_SECUREDISPLAY_COMMAND__QUERY_TA = 1, + /* Send region of Interest and CRC value to I2C */ + TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC = 2, + /* Maximum Command ID */ + TA_SECUREDISPLAY_COMMAND__MAX_ID = 0x7FFFFFFF, +}; + +/** @enum ta_securedisplay_status + * Secure Display status returns in shared buffer status + */ +enum ta_securedisplay_status { + TA_SECUREDISPLAY_STATUS__SUCCESS = 0x00, /* Success */ + TA_SECUREDISPLAY_STATUS__GENERIC_FAILURE = 0x01, /* Generic Failure */ + TA_SECUREDISPLAY_STATUS__INVALID_PARAMETER = 0x02, /* Invalid Parameter */ + TA_SECUREDISPLAY_STATUS__NULL_POINTER = 0x03, /* Null Pointer*/ + TA_SECUREDISPLAY_STATUS__I2C_WRITE_ERROR = 0x04, /* Fail to Write to I2C */ + TA_SECUREDISPLAY_STATUS__READ_DIO_SCRATCH_ERROR = 0x05, /*Fail Read DIO Scratch Register*/ + TA_SECUREDISPLAY_STATUS__READ_CRC_ERROR = 0x06, /* Fail to Read CRC*/ + + TA_SECUREDISPLAY_STATUS__MAX = 0x7FFFFFFF,/* Maximum Value for status*/ +}; + +/** @enum ta_securedisplay_max_phy + * Physical ID number to use for reading corresponding DIO Scratch register for ROI + */ +enum ta_securedisplay_max_phy { + TA_SECUREDISPLAY_PHY0 = 0, + TA_SECUREDISPLAY_PHY1 = 1, + TA_SECUREDISPLAY_PHY2 = 2, + TA_SECUREDISPLAY_PHY3 = 3, + TA_SECUREDISPLAY_MAX_PHY = 4, +}; + +/** @enum ta_securedisplay_ta_query_cmd_ret + * A predefined specific reteurn value which is 0xAB only used to validate + * communication to Secure Display TA is functional. + * This value is used to validate whether TA is responding successfully + */ +enum ta_securedisplay_ta_query_cmd_ret { + /* This is a value to validate if TA is loaded successfully */ + TA_SECUREDISPLAY_QUERY_CMD_RET = 0xAB, +}; + +/** @enum ta_securedisplay_buffer_size + * I2C Buffer size which contains 8 bytes of ROI (X start, X end, Y start, Y end) + * and 6 bytes of CRC( R,G,B) and 1 byte for physical ID + */ +enum ta_securedisplay_buffer_size { + /* 15 bytes = 8 byte (ROI) + 6 byte(CRC) + 1 byte(phy_id) */ + TA_SECUREDISPLAY_I2C_BUFFER_SIZE = 15, +}; + +/** Input/output structures for Secure Display commands */ +/**********************************************************/ +/** + * Input structures + */ + +/** @struct ta_securedisplay_send_roi_crc_input + * Physical ID to determine which DIO scratch register should be used to get ROI + */ +struct ta_securedisplay_send_roi_crc_input { + uint32_t phy_id; /* Physical ID */ +}; + +/** @union ta_securedisplay_cmd_input + * Input buffer + */ +union ta_securedisplay_cmd_input { + /* send ROI and CRC input buffer format */ + struct ta_securedisplay_send_roi_crc_input send_roi_crc; + uint32_t reserved[4]; +}; + +/** + * Output structures + */ + +/** @struct ta_securedisplay_query_ta_output + * Output buffer format for query TA whether TA is responding used only for validation purpose + */ +struct ta_securedisplay_query_ta_output { + /* return value from TA when it is queried for validation purpose only */ + uint32_t query_cmd_ret; +}; + +/** @struct ta_securedisplay_send_roi_crc_output + * Output buffer format for send ROI CRC command which will pass I2c buffer created inside TA + * and used to write to I2C used only for validation purpose + */ +struct ta_securedisplay_send_roi_crc_output { + uint8_t i2c_buf[TA_SECUREDISPLAY_I2C_BUFFER_SIZE]; /* I2C buffer */ + uint8_t reserved; +}; + +/** @union ta_securedisplay_cmd_output + * Output buffer + */ +union ta_securedisplay_cmd_output { + /* Query TA output buffer format used only for validation purpose*/ + struct ta_securedisplay_query_ta_output query_ta; + /* Send ROI CRC output buffer format used only for validation purpose */ + struct ta_securedisplay_send_roi_crc_output send_roi_crc; + uint32_t reserved[4]; +}; + +/** @struct securedisplay_cmd + * Secure Display Command which is shared buffer memory + */ +struct securedisplay_cmd { + uint32_t cmd_id; /* +0 Bytes Command ID */ + enum ta_securedisplay_status status; /* +4 Bytes Status of Secure Display TA */ + uint32_t reserved[2]; /* +8 Bytes Reserved */ + union ta_securedisplay_cmd_input securedisplay_in_message; /* +16 Bytes Input Buffer */ + union ta_securedisplay_cmd_output securedisplay_out_message;/* +32 Bytes Output Buffer */ + /**@note Total 48 Bytes */ +}; + +#endif //_TA_SECUREDISPLAY_IF_H + diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c index ce3319993b4b..249fcbee7871 100644 --- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c @@ -196,19 +196,30 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev, wptr = le32_to_cpu(*ih->wptr_cpu); - if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { - wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); - /* When a ring buffer overflow happen start parsing interrupt - * from the last not overwritten vector (wptr + 16). Hopefully - * this should allow us to catchup. - */ - dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); - ih->rptr = (wptr + 16) & ih->ptr_mask; - tmp = RREG32(mmIH_RB_CNTL); - tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32(mmIH_RB_CNTL, tmp); - } + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + /* Double check that the overflow wasn't already cleared. */ + wptr = RREG32(mmIH_RB_WPTR); + + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); + + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 16). Hopefully + * this should allow us to catchup. + */ + + dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; + tmp = RREG32(mmIH_RB_CNTL); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); + WREG32(mmIH_RB_CNTL, tmp); + +out: return (wptr & ih->ptr_mask); } diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c index 312ecf6d24a0..7cd67cb2ac5f 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c @@ -36,7 +36,6 @@ #include "vce/vce_4_0_default.h" #include "vce/vce_4_0_sh_mask.h" #include "nbif/nbif_6_1_offset.h" -#include "hdp/hdp_4_0_offset.h" #include "mmhub/mmhub_1_0_offset.h" #include "mmhub/mmhub_1_0_sh_mask.h" #include "ivsrcid/uvd/irqsrcs_uvd_7_0.h" diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c index c734e31a9e65..6117931fa8d7 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c @@ -32,7 +32,6 @@ #include "vcn/vcn_1_0_offset.h" #include "vcn/vcn_1_0_sh_mask.h" -#include "hdp/hdp_4_0_offset.h" #include "mmhub/mmhub_9_1_offset.h" #include "mmhub/mmhub_9_1_sh_mask.h" diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c index e5ae31eb744e..88626d83e07b 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c @@ -38,132 +38,120 @@ static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev); /** - * vega10_ih_enable_interrupts - Enable the interrupt ring buffer + * vega10_ih_init_register_offset - Initialize register offset for ih rings * * @adev: amdgpu_device pointer * - * Enable the interrupt ring buffer (VEGA10). + * Initialize register offset ih rings (VEGA10). */ -static void vega10_ih_enable_interrupts(struct amdgpu_device *adev) +static void vega10_ih_init_register_offset(struct amdgpu_device *adev) { - u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); + struct amdgpu_ih_regs *ih_regs; - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); + if (adev->irq.ih.ring_size) { + ih_regs = &adev->irq.ih.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR); + ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO); + ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL; } - adev->irq.ih.enabled = true; if (adev->irq.ih1.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, - RB_ENABLE, 1); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); - } - adev->irq.ih1.enabled = true; + ih_regs = &adev->irq.ih1.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1; } if (adev->irq.ih2.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, - RB_ENABLE, 1); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); - } - adev->irq.ih2.enabled = true; + ih_regs = &adev->irq.ih2.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2; } - - if (adev->irq.ih_soft.ring_size) - adev->irq.ih_soft.enabled = true; } /** - * vega10_ih_disable_interrupts - Disable the interrupt ring buffer + * vega10_ih_toggle_ring_interrupts - toggle the interrupt ring buffer * * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointet + * @enable: true - enable the interrupts, false - disable the interrupts * - * Disable the interrupt ring buffer (VEGA10). + * Toggle the interrupt ring buffer (VEGA10) */ -static void vega10_ih_disable_interrupts(struct amdgpu_device *adev) +static int vega10_ih_toggle_ring_interrupts(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + bool enable) { - u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); + struct amdgpu_ih_regs *ih_regs; + uint32_t tmp; - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0); + ih_regs = &ih->ih_regs; + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0)); + /* enable_intr field is only valid in ring0 */ + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0)); if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return; + if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { + dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); + return -ETIMEDOUT; } } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); + WREG32(ih_regs->ih_rb_cntl, tmp); } - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); - adev->irq.ih.enabled = false; - adev->irq.ih.rptr = 0; - - if (adev->irq.ih1.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, - RB_ENABLE, 0); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); - } + if (enable) { + ih->enabled = true; + } else { /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); - adev->irq.ih1.enabled = false; - adev->irq.ih1.rptr = 0; + WREG32(ih_regs->ih_rb_rptr, 0); + WREG32(ih_regs->ih_rb_wptr, 0); + ih->enabled = false; + ih->rptr = 0; } - if (adev->irq.ih2.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, - RB_ENABLE, 0); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); + return 0; +} + +/** + * vega10_ih_toggle_interrupts - Toggle all the available interrupt ring buffers + * + * @adev: amdgpu_device pointer + * @enable: enable or disable interrupt ring buffers + * + * Toggle all the available interrupt ring buffers (VEGA10). + */ +static int vega10_ih_toggle_interrupts(struct amdgpu_device *adev, bool enable) +{ + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + int i; + int r; + + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + r = vega10_ih_toggle_ring_interrupts(adev, ih[i], enable); + if (r) + return r; } - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); - adev->irq.ih2.enabled = false; - adev->irq.ih2.rptr = 0; } + + return 0; } static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) @@ -208,6 +196,58 @@ static uint32_t vega10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) return ih_doorbell_rtpr; } +/** + * vega10_ih_enable_ring - enable an ih ring buffer + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + * Enable an ih ring buffer (VEGA10) + */ +static int vega10_ih_enable_ring(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + struct amdgpu_ih_regs *ih_regs; + uint32_t tmp; + + ih_regs = &ih->ih_regs; + + /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ + WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8); + WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff); + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = vega10_ih_rb_cntl(ih, tmp); + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); + if (ih == &adev->irq.ih1) { + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); + } + if (amdgpu_sriov_vf(adev)) { + if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { + dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); + return -ETIMEDOUT; + } + } else { + WREG32(ih_regs->ih_rb_cntl, tmp); + } + + if (ih == &adev->irq.ih) { + /* set the ih ring 0 writeback address whether it's enabled or not */ + WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr)); + WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF); + } + + /* set rptr, wptr to 0 */ + WREG32(ih_regs->ih_rb_wptr, 0); + WREG32(ih_regs->ih_rb_rptr, 0); + + WREG32(ih_regs->ih_doorbell_rptr, vega10_ih_doorbell_rptr(ih)); + + return 0; +} + /** * vega10_ih_irq_init - init and enable the interrupt ring * @@ -221,116 +261,34 @@ static uint32_t vega10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) */ static int vega10_ih_irq_init(struct amdgpu_device *adev) { - struct amdgpu_ih_ring *ih; - u32 ih_rb_cntl, ih_chicken; - int ret = 0; + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + u32 ih_chicken; + int ret; + int i; u32 tmp; /* disable irqs */ - vega10_ih_disable_interrupts(adev); + ret = vega10_ih_toggle_interrupts(adev, false); + if (ret) + return ret; adev->nbio.funcs->ih_control(adev); - ih = &adev->irq.ih; - /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); - ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, - !!adev->irq.msi_enabled); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); - } - - if ((adev->asic_type == CHIP_ARCTURUS && - adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) || - adev->asic_type == CHIP_RENOIR) { + if (adev->asic_type == CHIP_RENOIR) { ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN); if (adev->irq.ih.use_bus_addr) { ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1); - } else { - ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN, - MC_SPACE_FBPA_ENABLE, 1); } WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken); } - /* set the writeback address whether it's enabled or not */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, - lower_32_bits(ih->wptr_addr)); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, - upper_32_bits(ih->wptr_addr) & 0xFFFF); - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, - vega10_ih_doorbell_rptr(ih)); - - ih = &adev->irq.ih1; - if (ih->ring_size) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1, - (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, - WPTR_OVERFLOW_ENABLE, 0); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, - RB_FULL_DRAIN_ENABLE, 1); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + ret = vega10_ih_enable_ring(adev, ih[i]); + if (ret) + return ret; } - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1, - vega10_ih_doorbell_rptr(ih)); - } - - ih = &adev->irq.ih2; - if (ih->ring_size) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2, - (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); - - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); - } - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2, - vega10_ih_doorbell_rptr(ih)); } tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); @@ -345,9 +303,14 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev) pci_set_master(adev->pdev); /* enable interrupts */ - vega10_ih_enable_interrupts(adev); + ret = vega10_ih_toggle_interrupts(adev, true); + if (ret) + return ret; - return ret; + if (adev->irq.ih_soft.ring_size) + adev->irq.ih_soft.enabled = true; + + return 0; } /** @@ -359,7 +322,7 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev) */ static void vega10_ih_irq_disable(struct amdgpu_device *adev) { - vega10_ih_disable_interrupts(adev); + vega10_ih_toggle_interrupts(adev, false); /* Wait and acknowledge irq */ mdelay(1); @@ -379,25 +342,17 @@ static void vega10_ih_irq_disable(struct amdgpu_device *adev) static u32 vega10_ih_get_wptr(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { - u32 wptr, reg, tmp; + u32 wptr, tmp; + struct amdgpu_ih_regs *ih_regs; wptr = le32_to_cpu(*ih->wptr_cpu); + ih_regs = &ih->ih_regs; if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) goto out; /* Double check that the overflow wasn't already cleared. */ - - if (ih == &adev->irq.ih) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); - else if (ih == &adev->irq.ih1) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); - else if (ih == &adev->irq.ih2) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); - else - BUG(); - - wptr = RREG32_NO_KIQ(reg); + wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr); if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) goto out; @@ -413,68 +368,14 @@ static u32 vega10_ih_get_wptr(struct amdgpu_device *adev, wptr, ih->rptr, tmp); ih->rptr = tmp; - if (ih == &adev->irq.ih) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); - else if (ih == &adev->irq.ih1) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); - else if (ih == &adev->irq.ih2) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); - else - BUG(); - - tmp = RREG32_NO_KIQ(reg); + tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl); tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32_NO_KIQ(reg, tmp); + WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp); out: return (wptr & ih->ptr_mask); } -/** - * vega10_ih_decode_iv - decode an interrupt vector - * - * @adev: amdgpu_device pointer - * @ih: IH ring buffer to decode - * @entry: IV entry to place decoded information into - * - * Decodes the interrupt vector at the current rptr - * position and also advance the position. - */ -static void vega10_ih_decode_iv(struct amdgpu_device *adev, - struct amdgpu_ih_ring *ih, - struct amdgpu_iv_entry *entry) -{ - /* wptr/rptr are in bytes! */ - u32 ring_index = ih->rptr >> 2; - uint32_t dw[8]; - - dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); - dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); - dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); - dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); - dw[4] = le32_to_cpu(ih->ring[ring_index + 4]); - dw[5] = le32_to_cpu(ih->ring[ring_index + 5]); - dw[6] = le32_to_cpu(ih->ring[ring_index + 6]); - dw[7] = le32_to_cpu(ih->ring[ring_index + 7]); - - entry->client_id = dw[0] & 0xff; - entry->src_id = (dw[0] >> 8) & 0xff; - entry->ring_id = (dw[0] >> 16) & 0xff; - entry->vmid = (dw[0] >> 24) & 0xf; - entry->vmid_src = (dw[0] >> 31); - entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32); - entry->timestamp_src = dw[2] >> 31; - entry->pasid = dw[3] & 0xffff; - entry->pasid_src = dw[3] >> 31; - entry->src_data[0] = dw[4]; - entry->src_data[1] = dw[5]; - entry->src_data[2] = dw[6]; - entry->src_data[3] = dw[7]; - - /* wptr/rptr are in bytes! */ - ih->rptr += 32; -} - /** * vega10_ih_irq_rearm - rearm IRQ if lost * @@ -485,22 +386,14 @@ static void vega10_ih_decode_iv(struct amdgpu_device *adev, static void vega10_ih_irq_rearm(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { - uint32_t reg_rptr = 0; uint32_t v = 0; uint32_t i = 0; + struct amdgpu_ih_regs *ih_regs; - if (ih == &adev->irq.ih) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); - else if (ih == &adev->irq.ih1) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); - else if (ih == &adev->irq.ih2) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); - else - return; - + ih_regs = &ih->ih_regs; /* Rearm IRQ / re-wwrite doorbell if doorbell write is lost */ for (i = 0; i < MAX_REARM_RETRY; i++) { - v = RREG32_NO_KIQ(reg_rptr); + v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr); if ((v < ih->ring_size) && (v != ih->rptr)) WDOORBELL32(ih->doorbell_index, ih->rptr); else @@ -519,6 +412,8 @@ static void vega10_ih_irq_rearm(struct amdgpu_device *adev, static void vega10_ih_set_rptr(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { + struct amdgpu_ih_regs *ih_regs; + if (ih->use_doorbell) { /* XXX check if swapping is necessary on BE */ *ih->rptr_cpu = ih->rptr; @@ -526,12 +421,9 @@ static void vega10_ih_set_rptr(struct amdgpu_device *adev, if (amdgpu_sriov_vf(adev)) vega10_ih_irq_rearm(adev, ih); - } else if (ih == &adev->irq.ih) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr); - } else if (ih == &adev->irq.ih1) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, ih->rptr); - } else if (ih == &adev->irq.ih2) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, ih->rptr); + } else { + ih_regs = &ih->ih_regs; + WREG32(ih_regs->ih_rb_rptr, ih->rptr); } } @@ -600,19 +492,23 @@ static int vega10_ih_sw_init(void *handle) adev->irq.ih.use_doorbell = true; adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1; - r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); - if (r) - return r; + if (!(adev->flags & AMD_IS_APU)) { + r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); + if (r) + return r; - adev->irq.ih1.use_doorbell = true; - adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1; + adev->irq.ih1.use_doorbell = true; + adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1; - r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); - if (r) - return r; + r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); + if (r) + return r; - adev->irq.ih2.use_doorbell = true; - adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1; + adev->irq.ih2.use_doorbell = true; + adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1; + } + /* initialize ih control registers offset */ + vega10_ih_init_register_offset(adev); r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true); if (r) @@ -628,6 +524,7 @@ static int vega10_ih_sw_fini(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; amdgpu_irq_fini(adev); + amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft); amdgpu_ih_ring_fini(adev, &adev->irq.ih2); amdgpu_ih_ring_fini(adev, &adev->irq.ih1); amdgpu_ih_ring_fini(adev, &adev->irq.ih); @@ -698,15 +595,11 @@ static void vega10_ih_update_clockgating_state(struct amdgpu_device *adev, def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL); field_val = enable ? 0 : 1; /** - * Vega10 does not have IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE - * and IH_BUFFER_MEM_CLK_SOFT_OVERRIDE field. + * Vega10/12 and RAVEN don't have IH_BUFFER_MEM_CLK_SOFT_OVERRIDE field. */ - if (adev->asic_type > CHIP_VEGA10) { - data = REG_SET_FIELD(data, IH_CLK_CTRL, - IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE, field_val); + if (adev->asic_type == CHIP_RENOIR) data = REG_SET_FIELD(data, IH_CLK_CTRL, IH_BUFFER_MEM_CLK_SOFT_OVERRIDE, field_val); - } data = REG_SET_FIELD(data, IH_CLK_CTRL, DBUS_MUX_CLK_SOFT_OVERRIDE, field_val); @@ -759,7 +652,7 @@ const struct amd_ip_funcs vega10_ih_ip_funcs = { static const struct amdgpu_ih_funcs vega10_ih_funcs = { .get_wptr = vega10_ih_get_wptr, - .decode_iv = vega10_ih_decode_iv, + .decode_iv = amdgpu_ih_decode_iv_helper, .set_rptr = vega10_ih_set_rptr }; diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c new file mode 100644 index 000000000000..5a3c867d5881 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c @@ -0,0 +1,703 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include + +#include "amdgpu.h" +#include "amdgpu_ih.h" +#include "soc15.h" + +#include "oss/osssys_4_2_0_offset.h" +#include "oss/osssys_4_2_0_sh_mask.h" + +#include "soc15_common.h" +#include "vega20_ih.h" + +#define MAX_REARM_RETRY 10 + +static void vega20_ih_set_interrupt_funcs(struct amdgpu_device *adev); + +/** + * vega20_ih_init_register_offset - Initialize register offset for ih rings + * + * @adev: amdgpu_device pointer + * + * Initialize register offset ih rings (VEGA20). + */ +static void vega20_ih_init_register_offset(struct amdgpu_device *adev) +{ + struct amdgpu_ih_regs *ih_regs; + + if (adev->irq.ih.ring_size) { + ih_regs = &adev->irq.ih.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR); + ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO); + ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL; + } + + if (adev->irq.ih1.ring_size) { + ih_regs = &adev->irq.ih1.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1; + } + + if (adev->irq.ih2.ring_size) { + ih_regs = &adev->irq.ih2.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2; + } +} + +/** + * vega20_ih_toggle_ring_interrupts - toggle the interrupt ring buffer + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * @enable: true - enable the interrupts, false - disable the interrupts + * + * Toggle the interrupt ring buffer (VEGA20) + */ +static int vega20_ih_toggle_ring_interrupts(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + bool enable) +{ + struct amdgpu_ih_regs *ih_regs; + uint32_t tmp; + + ih_regs = &ih->ih_regs; + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0)); + /* enable_intr field is only valid in ring0 */ + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0)); + if (amdgpu_sriov_vf(adev)) { + if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { + dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); + return -ETIMEDOUT; + } + } else { + WREG32(ih_regs->ih_rb_cntl, tmp); + } + + if (enable) { + ih->enabled = true; + } else { + /* set rptr, wptr to 0 */ + WREG32(ih_regs->ih_rb_rptr, 0); + WREG32(ih_regs->ih_rb_wptr, 0); + ih->enabled = false; + ih->rptr = 0; + } + + return 0; +} + +/** + * vega20_ih_toggle_interrupts - Toggle all the available interrupt ring buffers + * + * @adev: amdgpu_device pointer + * @enable: enable or disable interrupt ring buffers + * + * Toggle all the available interrupt ring buffers (VEGA20). + */ +static int vega20_ih_toggle_interrupts(struct amdgpu_device *adev, bool enable) +{ + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + int i; + int r; + + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + r = vega20_ih_toggle_ring_interrupts(adev, ih[i], enable); + if (r) + return r; + } + } + + return 0; +} + +static uint32_t vega20_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) +{ + int rb_bufsz = order_base_2(ih->ring_size / 4); + + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + MC_SPACE, ih->use_bus_addr ? 1 : 4); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + WPTR_OVERFLOW_CLEAR, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + WPTR_OVERFLOW_ENABLE, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz); + /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register + * value is written to memory + */ + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + WPTR_WRITEBACK_ENABLE, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0); + + return ih_rb_cntl; +} + +static uint32_t vega20_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) +{ + u32 ih_doorbell_rtpr = 0; + + if (ih->use_doorbell) { + ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, + IH_DOORBELL_RPTR, OFFSET, + ih->doorbell_index); + ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, + IH_DOORBELL_RPTR, + ENABLE, 1); + } else { + ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, + IH_DOORBELL_RPTR, + ENABLE, 0); + } + return ih_doorbell_rtpr; +} + +/** + * vega20_ih_enable_ring - enable an ih ring buffer + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + * Enable an ih ring buffer (VEGA20) + */ +static int vega20_ih_enable_ring(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + struct amdgpu_ih_regs *ih_regs; + uint32_t tmp; + + ih_regs = &ih->ih_regs; + + /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ + WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8); + WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff); + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = vega20_ih_rb_cntl(ih, tmp); + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); + if (ih == &adev->irq.ih1) { + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); + } + if (amdgpu_sriov_vf(adev)) { + if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { + dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); + return -ETIMEDOUT; + } + } else { + WREG32(ih_regs->ih_rb_cntl, tmp); + } + + if (ih == &adev->irq.ih) { + /* set the ih ring 0 writeback address whether it's enabled or not */ + WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr)); + WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF); + } + + /* set rptr, wptr to 0 */ + WREG32(ih_regs->ih_rb_wptr, 0); + WREG32(ih_regs->ih_rb_rptr, 0); + + WREG32(ih_regs->ih_doorbell_rptr, vega20_ih_doorbell_rptr(ih)); + + return 0; +} + +/** + * vega20_ih_reroute_ih - reroute VMC/UTCL2 ih to an ih ring + * + * @adev: amdgpu_device pointer + * + * Reroute VMC and UMC interrupts on primary ih ring to + * ih ring 1 so they won't lose when bunches of page faults + * interrupts overwhelms the interrupt handler(VEGA20) + */ +static void vega20_ih_reroute_ih(struct amdgpu_device *adev) +{ + uint32_t tmp; + + /* vega20 ih reroute will go through psp + * this function is only used for arcturus + */ + if (adev->asic_type == CHIP_ARCTURUS) { + /* Reroute to IH ring 1 for VMC */ + WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x12); + tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); + tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1); + tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp); + + /* Reroute IH ring 1 for UTCL2 */ + WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x1B); + tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); + tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp); + } +} + +/** + * vega20_ih_irq_init - init and enable the interrupt ring + * + * @adev: amdgpu_device pointer + * + * Allocate a ring buffer for the interrupt controller, + * enable the RLC, disable interrupts, enable the IH + * ring buffer and enable it (VI). + * Called at device load and reume. + * Returns 0 for success, errors for failure. + */ +static int vega20_ih_irq_init(struct amdgpu_device *adev) +{ + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + u32 ih_chicken; + int ret; + int i; + u32 tmp; + + /* disable irqs */ + ret = vega20_ih_toggle_interrupts(adev, false); + if (ret) + return ret; + + adev->nbio.funcs->ih_control(adev); + + if (adev->asic_type == CHIP_ARCTURUS && + adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { + ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN); + if (adev->irq.ih.use_bus_addr) { + ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN, + MC_SPACE_GPA_ENABLE, 1); + } + WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken); + } + + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + if (i == 1) + vega20_ih_reroute_ih(adev); + ret = vega20_ih_enable_ring(adev, ih[i]); + if (ret) + return ret; + } + } + + tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); + tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, + CLIENT18_IS_STORM_CLIENT, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp); + + tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL); + tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp); + + pci_set_master(adev->pdev); + + /* enable interrupts */ + ret = vega20_ih_toggle_interrupts(adev, true); + if (ret) + return ret; + + if (adev->irq.ih_soft.ring_size) + adev->irq.ih_soft.enabled = true; + + return 0; +} + +/** + * vega20_ih_irq_disable - disable interrupts + * + * @adev: amdgpu_device pointer + * + * Disable interrupts on the hw (VEGA20). + */ +static void vega20_ih_irq_disable(struct amdgpu_device *adev) +{ + vega20_ih_toggle_interrupts(adev, false); + + /* Wait and acknowledge irq */ + mdelay(1); +} + +/** + * vega20_ih_get_wptr - get the IH ring buffer wptr + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + * Get the IH ring buffer wptr from either the register + * or the writeback memory buffer (VEGA20). Also check for + * ring buffer overflow and deal with it. + * Returns the value of the wptr. + */ +static u32 vega20_ih_get_wptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + u32 wptr, tmp; + struct amdgpu_ih_regs *ih_regs; + + wptr = le32_to_cpu(*ih->wptr_cpu); + ih_regs = &ih->ih_regs; + + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + /* Double check that the overflow wasn't already cleared. */ + wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr); + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); + + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 32). Hopefully + * this should allow us to catchup. + */ + tmp = (wptr + 32) & ih->ptr_mask; + dev_warn(adev->dev, "IH ring buffer overflow " + "(0x%08X, 0x%08X, 0x%08X)\n", + wptr, ih->rptr, tmp); + ih->rptr = tmp; + + tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); + WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp); + +out: + return (wptr & ih->ptr_mask); +} + +/** + * vega20_ih_irq_rearm - rearm IRQ if lost + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + */ +static void vega20_ih_irq_rearm(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + uint32_t v = 0; + uint32_t i = 0; + struct amdgpu_ih_regs *ih_regs; + + ih_regs = &ih->ih_regs; + + /* Rearm IRQ / re-wwrite doorbell if doorbell write is lost */ + for (i = 0; i < MAX_REARM_RETRY; i++) { + v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr); + if ((v < ih->ring_size) && (v != ih->rptr)) + WDOORBELL32(ih->doorbell_index, ih->rptr); + else + break; + } +} + +/** + * vega20_ih_set_rptr - set the IH ring buffer rptr + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + * Set the IH ring buffer rptr. + */ +static void vega20_ih_set_rptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + struct amdgpu_ih_regs *ih_regs; + + if (ih->use_doorbell) { + /* XXX check if swapping is necessary on BE */ + *ih->rptr_cpu = ih->rptr; + WDOORBELL32(ih->doorbell_index, ih->rptr); + + if (amdgpu_sriov_vf(adev)) + vega20_ih_irq_rearm(adev, ih); + } else { + ih_regs = &ih->ih_regs; + WREG32(ih_regs->ih_rb_rptr, ih->rptr); + } +} + +/** + * vega20_ih_self_irq - dispatch work for ring 1 and 2 + * + * @adev: amdgpu_device pointer + * @source: irq source + * @entry: IV with WPTR update + * + * Update the WPTR from the IV and schedule work to handle the entries. + */ +static int vega20_ih_self_irq(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + uint32_t wptr = cpu_to_le32(entry->src_data[0]); + + switch (entry->ring_id) { + case 1: + *adev->irq.ih1.wptr_cpu = wptr; + schedule_work(&adev->irq.ih1_work); + break; + case 2: + *adev->irq.ih2.wptr_cpu = wptr; + schedule_work(&adev->irq.ih2_work); + break; + default: break; + } + return 0; +} + +static const struct amdgpu_irq_src_funcs vega20_ih_self_irq_funcs = { + .process = vega20_ih_self_irq, +}; + +static void vega20_ih_set_self_irq_funcs(struct amdgpu_device *adev) +{ + adev->irq.self_irq.num_types = 0; + adev->irq.self_irq.funcs = &vega20_ih_self_irq_funcs; +} + +static int vega20_ih_early_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + vega20_ih_set_interrupt_funcs(adev); + vega20_ih_set_self_irq_funcs(adev); + return 0; +} + +static int vega20_ih_sw_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + int r; + + r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_IH, 0, + &adev->irq.self_irq); + if (r) + return r; + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true); + if (r) + return r; + + adev->irq.ih.use_doorbell = true; + adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1; + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); + if (r) + return r; + + adev->irq.ih1.use_doorbell = true; + adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1; + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); + if (r) + return r; + + adev->irq.ih2.use_doorbell = true; + adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1; + + /* initialize ih control registers offset */ + vega20_ih_init_register_offset(adev); + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true); + if (r) + return r; + + r = amdgpu_irq_init(adev); + + return r; +} + +static int vega20_ih_sw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + amdgpu_irq_fini(adev); + amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft); + amdgpu_ih_ring_fini(adev, &adev->irq.ih2); + amdgpu_ih_ring_fini(adev, &adev->irq.ih1); + amdgpu_ih_ring_fini(adev, &adev->irq.ih); + + return 0; +} + +static int vega20_ih_hw_init(void *handle) +{ + int r; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + r = vega20_ih_irq_init(adev); + if (r) + return r; + + return 0; +} + +static int vega20_ih_hw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + vega20_ih_irq_disable(adev); + + return 0; +} + +static int vega20_ih_suspend(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return vega20_ih_hw_fini(adev); +} + +static int vega20_ih_resume(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return vega20_ih_hw_init(adev); +} + +static bool vega20_ih_is_idle(void *handle) +{ + /* todo */ + return true; +} + +static int vega20_ih_wait_for_idle(void *handle) +{ + /* todo */ + return -ETIMEDOUT; +} + +static int vega20_ih_soft_reset(void *handle) +{ + /* todo */ + + return 0; +} + +static void vega20_ih_update_clockgating_state(struct amdgpu_device *adev, + bool enable) +{ + uint32_t data, def, field_val; + + if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) { + def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL); + field_val = enable ? 0 : 1; + data = REG_SET_FIELD(data, IH_CLK_CTRL, + IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + IH_BUFFER_MEM_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + DBUS_MUX_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + DYN_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + REG_CLK_SOFT_OVERRIDE, field_val); + if (def != data) + WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data); + } +} + +static int vega20_ih_set_clockgating_state(void *handle, + enum amd_clockgating_state state) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + vega20_ih_update_clockgating_state(adev, + state == AMD_CG_STATE_GATE); + return 0; + +} + +static int vega20_ih_set_powergating_state(void *handle, + enum amd_powergating_state state) +{ + return 0; +} + +const struct amd_ip_funcs vega20_ih_ip_funcs = { + .name = "vega20_ih", + .early_init = vega20_ih_early_init, + .late_init = NULL, + .sw_init = vega20_ih_sw_init, + .sw_fini = vega20_ih_sw_fini, + .hw_init = vega20_ih_hw_init, + .hw_fini = vega20_ih_hw_fini, + .suspend = vega20_ih_suspend, + .resume = vega20_ih_resume, + .is_idle = vega20_ih_is_idle, + .wait_for_idle = vega20_ih_wait_for_idle, + .soft_reset = vega20_ih_soft_reset, + .set_clockgating_state = vega20_ih_set_clockgating_state, + .set_powergating_state = vega20_ih_set_powergating_state, +}; + +static const struct amdgpu_ih_funcs vega20_ih_funcs = { + .get_wptr = vega20_ih_get_wptr, + .decode_iv = amdgpu_ih_decode_iv_helper, + .set_rptr = vega20_ih_set_rptr +}; + +static void vega20_ih_set_interrupt_funcs(struct amdgpu_device *adev) +{ + adev->irq.ih_funcs = &vega20_ih_funcs; +} + +const struct amdgpu_ip_block_version vega20_ih_ip_block = +{ + .type = AMD_IP_BLOCK_TYPE_IH, + .major = 4, + .minor = 2, + .rev = 0, + .funcs = &vega20_ih_ip_funcs, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_ih.h b/drivers/gpu/drm/amd/amdgpu/vega20_ih.h new file mode 100644 index 000000000000..7af6d8758ee3 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/vega20_ih.h @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __VEGA20_IH_H__ +#define __VEGA20_IH_H__ + +extern const struct amd_ip_funcs vega20_ih_ip_funcs; +extern const struct amdgpu_ip_block_version vega20_ih_ip_block; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index d56b474b3a21..eafb76aebd00 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -642,11 +642,21 @@ static int vi_read_register(struct amdgpu_device *adev, u32 se_num, return -EINVAL; } -static int vi_gpu_pci_config_reset(struct amdgpu_device *adev) +/** + * vi_asic_pci_config_reset - soft reset GPU + * + * @adev: amdgpu_device pointer + * + * Use PCI Config method to reset the GPU. + * + * Returns 0 for success. + */ +static int vi_asic_pci_config_reset(struct amdgpu_device *adev) { u32 i; + int r = -EINVAL; - dev_info(adev->dev, "GPU pci config reset\n"); + amdgpu_atombios_scratch_regs_engine_hung(adev, true); /* disable BM */ pci_clear_master(adev->pdev); @@ -661,29 +671,11 @@ static int vi_gpu_pci_config_reset(struct amdgpu_device *adev) /* enable BM */ pci_set_master(adev->pdev); adev->has_hw_reset = true; - return 0; + r = 0; + break; } udelay(1); } - return -EINVAL; -} - -/** - * vi_asic_pci_config_reset - soft reset GPU - * - * @adev: amdgpu_device pointer - * - * Use PCI Config method to reset the GPU. - * - * Returns 0 for success. - */ -static int vi_asic_pci_config_reset(struct amdgpu_device *adev) -{ - int r; - - amdgpu_atombios_scratch_regs_engine_hung(adev, true); - - r = vi_gpu_pci_config_reset(adev); amdgpu_atombios_scratch_regs_engine_hung(adev, false); diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig index e8fb10c41f16..f02c938f75da 100644 --- a/drivers/gpu/drm/amd/amdkfd/Kconfig +++ b/drivers/gpu/drm/amd/amdkfd/Kconfig @@ -7,6 +7,8 @@ config HSA_AMD bool "HSA kernel driver for AMD GPU devices" depends on DRM_AMDGPU && (X86_64 || ARM64 || PPC64) imply AMD_IOMMU_V2 if X86_64 + select HMM_MIRROR select MMU_NOTIFIER + select DRM_AMDGPU_USERPTR help Enable this if you want to use HSA features on AMD GPU devices. diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h index 16262e5d93f5..7351dd195274 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -243,11 +243,11 @@ get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd) static inline void dqm_lock(struct device_queue_manager *dqm) { mutex_lock(&dqm->lock_hidden); - dqm->saved_flags = memalloc_nofs_save(); + dqm->saved_flags = memalloc_noreclaim_save(); } static inline void dqm_unlock(struct device_queue_manager *dqm) { - memalloc_nofs_restore(dqm->saved_flags); + memalloc_noreclaim_restore(dqm->saved_flags); mutex_unlock(&dqm->lock_hidden); } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c index 241bd6ff79f4..74a460be077b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c @@ -44,6 +44,25 @@ static bool event_interrupt_isr_v9(struct kfd_dev *dev, client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry); pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry); + /* Only handle clients we care about */ + if (client_id != SOC15_IH_CLIENTID_GRBM_CP && + client_id != SOC15_IH_CLIENTID_SDMA0 && + client_id != SOC15_IH_CLIENTID_SDMA1 && + client_id != SOC15_IH_CLIENTID_SDMA2 && + client_id != SOC15_IH_CLIENTID_SDMA3 && + client_id != SOC15_IH_CLIENTID_SDMA4 && + client_id != SOC15_IH_CLIENTID_SDMA5 && + client_id != SOC15_IH_CLIENTID_SDMA6 && + client_id != SOC15_IH_CLIENTID_SDMA7 && + client_id != SOC15_IH_CLIENTID_VMC && + client_id != SOC15_IH_CLIENTID_VMC1 && + client_id != SOC15_IH_CLIENTID_UTCL2 && + client_id != SOC15_IH_CLIENTID_SE0SH && + client_id != SOC15_IH_CLIENTID_SE1SH && + client_id != SOC15_IH_CLIENTID_SE2SH && + client_id != SOC15_IH_CLIENTID_SE3SH) + return false; + /* This is a known issue for gfx9. Under non HWS, pasid is not set * in the interrupt payload, so we need to find out the pasid on our * own. @@ -96,17 +115,30 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev, vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry); context_id = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry); - if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) - kfd_signal_event_interrupt(pasid, context_id, 32); - else if (source_id == SOC15_INTSRC_SDMA_TRAP) - kfd_signal_event_interrupt(pasid, context_id & 0xfffffff, 28); - else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) - kfd_signal_event_interrupt(pasid, context_id & 0xffffff, 24); - else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE) - kfd_signal_hw_exception_event(pasid); - else if (client_id == SOC15_IH_CLIENTID_VMC || - client_id == SOC15_IH_CLIENTID_VMC1 || - client_id == SOC15_IH_CLIENTID_UTCL2) { + if (client_id == SOC15_IH_CLIENTID_GRBM_CP || + client_id == SOC15_IH_CLIENTID_SE0SH || + client_id == SOC15_IH_CLIENTID_SE1SH || + client_id == SOC15_IH_CLIENTID_SE2SH || + client_id == SOC15_IH_CLIENTID_SE3SH) { + if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) + kfd_signal_event_interrupt(pasid, context_id, 32); + else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) + kfd_signal_event_interrupt(pasid, context_id & 0xffffff, 24); + else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE) + kfd_signal_hw_exception_event(pasid); + } else if (client_id == SOC15_IH_CLIENTID_SDMA0 || + client_id == SOC15_IH_CLIENTID_SDMA1 || + client_id == SOC15_IH_CLIENTID_SDMA2 || + client_id == SOC15_IH_CLIENTID_SDMA3 || + client_id == SOC15_IH_CLIENTID_SDMA4 || + client_id == SOC15_IH_CLIENTID_SDMA5 || + client_id == SOC15_IH_CLIENTID_SDMA6 || + client_id == SOC15_IH_CLIENTID_SDMA7) { + if (source_id == SOC15_INTSRC_SDMA_TRAP) + kfd_signal_event_interrupt(pasid, context_id & 0xfffffff, 28); + } else if (client_id == SOC15_IH_CLIENTID_VMC || + client_id == SOC15_IH_CLIENTID_VMC1 || + client_id == SOC15_IH_CLIENTID_UTCL2) { struct kfd_vm_fault_info info = {0}; uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index a3fc23873819..0be72789ccbc 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -497,8 +497,6 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr, dev->node_props.num_sdma_queues_per_engine); sysfs_show_32bit_prop(buffer, offs, "num_cp_queues", dev->node_props.num_cp_queues); - sysfs_show_64bit_prop(buffer, offs, "unique_id", - dev->node_props.unique_id); if (dev->gpu) { log_max_watch_addr = @@ -529,6 +527,9 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr, dev->node_props.capability); sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version", dev->gpu->sdma_fw_version); + sysfs_show_64bit_prop(buffer, offs, "unique_id", + amdgpu_amdkfd_get_unique_id(dev->gpu->kgd)); + } return sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_ccompute", @@ -1340,7 +1341,6 @@ int kfd_topology_add_device(struct kfd_dev *gpu) dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ? amdgpu_amdkfd_get_num_gws(dev->gpu->kgd) : 0; dev->node_props.num_cp_queues = get_cp_queues_num(dev->gpu->dqm); - dev->node_props.unique_id = amdgpu_amdkfd_get_unique_id(dev->gpu->kgd); kfd_fill_mem_clk_max_info(dev); kfd_fill_iolink_non_crat_info(dev); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h index 326d9b26b7aa..416fd910e12e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h @@ -57,7 +57,6 @@ struct kfd_node_properties { uint64_t hive_id; - uint64_t unique_id; uint32_t cpu_cores_count; uint32_t simd_count; uint32_t mem_banks_count; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 961abf1cf040..3e1fd1e7d09f 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -60,7 +60,6 @@ #include #include -#include #include #include #include @@ -938,7 +937,49 @@ static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_ } #endif +#if defined(CONFIG_DRM_AMD_DC_DCN) +static void event_mall_stutter(struct work_struct *work) +{ + struct vblank_workqueue *vblank_work = container_of(work, struct vblank_workqueue, mall_work); + struct amdgpu_display_manager *dm = vblank_work->dm; + + mutex_lock(&dm->dc_lock); + + if (vblank_work->enable) + dm->active_vblank_irq_count++; + else + dm->active_vblank_irq_count--; + + + dc_allow_idle_optimizations( + dm->dc, dm->active_vblank_irq_count == 0 ? true : false); + + DRM_DEBUG_DRIVER("Allow idle optimizations (MALL): %d\n", dm->active_vblank_irq_count == 0); + + + mutex_unlock(&dm->dc_lock); +} + +static struct vblank_workqueue *vblank_create_workqueue(struct amdgpu_device *adev, struct dc *dc) +{ + + int max_caps = dc->caps.max_links; + struct vblank_workqueue *vblank_work; + int i = 0; + + vblank_work = kcalloc(max_caps, sizeof(*vblank_work), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(vblank_work)) { + kfree(vblank_work); + return NULL; + } + + for (i = 0; i < max_caps; i++) + INIT_WORK(&vblank_work[i].mall_work, event_mall_stutter); + + return vblank_work; +} +#endif static int amdgpu_dm_init(struct amdgpu_device *adev) { struct dc_init_data init_data; @@ -958,6 +999,9 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) mutex_init(&adev->dm.dc_lock); mutex_init(&adev->dm.audio_lock); +#if defined(CONFIG_DRM_AMD_DC_DCN) + spin_lock_init(&adev->dm.vblank_lock); +#endif if(amdgpu_dm_irq_init(adev)) { DRM_ERROR("amdgpu: failed to initialize DM IRQ support.\n"); @@ -1016,8 +1060,6 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) init_data.flags.power_down_display_on_boot = true; - init_data.soc_bounding_box = adev->dm.soc_bounding_box; - /* Display Core create. */ adev->dm.dc = dc_create(&init_data); @@ -1074,6 +1116,17 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) amdgpu_dm_init_color_mod(); +#if defined(CONFIG_DRM_AMD_DC_DCN) + if (adev->dm.dc->caps.max_links > 0) { + adev->dm.vblank_workqueue = vblank_create_workqueue(adev, adev->dm.dc); + + if (!adev->dm.vblank_workqueue) + DRM_ERROR("amdgpu: failed to initialize vblank_workqueue.\n"); + else + DRM_DEBUG_DRIVER("amdgpu: vblank_workqueue init done %p.\n", adev->dm.vblank_workqueue); + } +#endif + #ifdef CONFIG_DRM_AMD_DC_HDCP if (adev->dm.dc->caps.max_links > 0 && adev->asic_type >= CHIP_RAVEN) { adev->dm.hdcp_workqueue = hdcp_create_workqueue(adev, &init_params.cp_psp, adev->dm.dc); @@ -1131,7 +1184,7 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev) #ifdef CONFIG_DRM_AMD_DC_HDCP if (adev->dm.hdcp_workqueue) { - hdcp_destroy(adev->dm.hdcp_workqueue); + hdcp_destroy(&adev->dev->kobj, adev->dm.hdcp_workqueue); adev->dm.hdcp_workqueue = NULL; } @@ -1778,6 +1831,11 @@ static int dm_suspend(void *handle) if (amdgpu_in_reset(adev)) { mutex_lock(&dm->dc_lock); + +#if defined(CONFIG_DRM_AMD_DC_DCN) + dc_allow_idle_optimizations(adev->dm.dc, false); +#endif + dm->cached_dc_state = dc_copy_state(dm->dc->current_state); dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, false); @@ -1934,7 +1992,7 @@ static void dm_gpureset_commit_state(struct dc_state *dc_state, dc_commit_updates_for_stream( dm->dc, bundle->surface_updates, dc_state->stream_status->plane_count, - dc_state->streams[k], &bundle->stream_update); + dc_state->streams[k], &bundle->stream_update, dc_state); } cleanup: @@ -1965,7 +2023,8 @@ static void dm_set_dpms_off(struct dc_link *link) stream_update.stream = stream_state; dc_commit_updates_for_stream(stream_state->ctx->dc, NULL, 0, - stream_state, &stream_update); + stream_state, &stream_update, + stream_state->ctx->dc->current_state); mutex_unlock(&adev->dm.dc_lock); } @@ -3719,10 +3778,53 @@ static const struct drm_encoder_funcs amdgpu_dm_encoder_funcs = { }; +static void get_min_max_dc_plane_scaling(struct drm_device *dev, + struct drm_framebuffer *fb, + int *min_downscale, int *max_upscale) +{ + struct amdgpu_device *adev = drm_to_adev(dev); + struct dc *dc = adev->dm.dc; + /* Caps for all supported planes are the same on DCE and DCN 1 - 3 */ + struct dc_plane_cap *plane_cap = &dc->caps.planes[0]; + + switch (fb->format->format) { + case DRM_FORMAT_P010: + case DRM_FORMAT_NV12: + case DRM_FORMAT_NV21: + *max_upscale = plane_cap->max_upscale_factor.nv12; + *min_downscale = plane_cap->max_downscale_factor.nv12; + break; + + case DRM_FORMAT_XRGB16161616F: + case DRM_FORMAT_ARGB16161616F: + case DRM_FORMAT_XBGR16161616F: + case DRM_FORMAT_ABGR16161616F: + *max_upscale = plane_cap->max_upscale_factor.fp16; + *min_downscale = plane_cap->max_downscale_factor.fp16; + break; + + default: + *max_upscale = plane_cap->max_upscale_factor.argb8888; + *min_downscale = plane_cap->max_downscale_factor.argb8888; + break; + } + + /* + * A factor of 1 in the plane_cap means to not allow scaling, ie. use a + * scaling factor of 1.0 == 1000 units. + */ + if (*max_upscale == 1) + *max_upscale = 1000; + + if (*min_downscale == 1) + *min_downscale = 1000; +} + + static int fill_dc_scaling_info(const struct drm_plane_state *state, struct dc_scaling_info *scaling_info) { - int scale_w, scale_h; + int scale_w, scale_h, min_downscale, max_upscale; memset(scaling_info, 0, sizeof(*scaling_info)); @@ -3754,17 +3856,25 @@ static int fill_dc_scaling_info(const struct drm_plane_state *state, /* DRM doesn't specify clipping on destination output. */ scaling_info->clip_rect = scaling_info->dst_rect; - /* TODO: Validate scaling per-format with DC plane caps */ + /* Validate scaling per-format with DC plane caps */ + if (state->plane && state->plane->dev && state->fb) { + get_min_max_dc_plane_scaling(state->plane->dev, state->fb, + &min_downscale, &max_upscale); + } else { + min_downscale = 250; + max_upscale = 16000; + } + scale_w = scaling_info->dst_rect.width * 1000 / scaling_info->src_rect.width; - if (scale_w < 250 || scale_w > 16000) + if (scale_w < min_downscale || scale_w > max_upscale) return -EINVAL; scale_h = scaling_info->dst_rect.height * 1000 / scaling_info->src_rect.height; - if (scale_h < 250 || scale_h > 16000) + if (scale_h < min_downscale || scale_h > max_upscale) return -EINVAL; /* @@ -5321,6 +5431,10 @@ static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); struct amdgpu_device *adev = drm_to_adev(crtc->dev); struct dm_crtc_state *acrtc_state = to_dm_crtc_state(crtc->state); +#if defined(CONFIG_DRM_AMD_DC_DCN) + struct amdgpu_display_manager *dm = &adev->dm; + unsigned long flags; +#endif int rc = 0; if (enable) { @@ -5336,7 +5450,23 @@ static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable) return rc; irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst; - return dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY; + + if (!dc_interrupt_set(adev->dm.dc, irq_source, enable)) + return -EBUSY; + + if (amdgpu_in_reset(adev)) + return 0; + +#if defined(CONFIG_DRM_AMD_DC_DCN) + spin_lock_irqsave(&dm->vblank_lock, flags); + dm->vblank_workqueue->dm = dm; + dm->vblank_workqueue->otg_inst = acrtc->otg_inst; + dm->vblank_workqueue->enable = enable; + spin_unlock_irqrestore(&dm->vblank_lock, flags); + schedule_work(&dm->vblank_workqueue->mall_work); +#endif + + return 0; } static int dm_enable_vblank(struct drm_crtc *crtc) @@ -5353,7 +5483,6 @@ static void dm_disable_vblank(struct drm_crtc *crtc) static const struct drm_crtc_funcs amdgpu_dm_crtc_funcs = { .reset = dm_crtc_reset_state, .destroy = amdgpu_dm_crtc_destroy, - .gamma_set = drm_atomic_helper_legacy_gamma_set, .set_config = drm_atomic_helper_set_config, .page_flip = drm_atomic_helper_page_flip, .atomic_duplicate_state = dm_crtc_duplicate_state, @@ -6328,12 +6457,51 @@ static void dm_plane_helper_cleanup_fb(struct drm_plane *plane, static int dm_plane_helper_check_state(struct drm_plane_state *state, struct drm_crtc_state *new_crtc_state) { - int max_downscale = 0; - int max_upscale = INT_MAX; + struct drm_framebuffer *fb = state->fb; + int min_downscale, max_upscale; + int min_scale = 0; + int max_scale = INT_MAX; + + /* Plane enabled? Validate viewport and get scaling factors from plane caps. */ + if (fb && state->crtc) { + /* Validate viewport to cover the case when only the position changes */ + if (state->plane->type != DRM_PLANE_TYPE_CURSOR) { + int viewport_width = state->crtc_w; + int viewport_height = state->crtc_h; + + if (state->crtc_x < 0) + viewport_width += state->crtc_x; + else if (state->crtc_x + state->crtc_w > new_crtc_state->mode.crtc_hdisplay) + viewport_width = new_crtc_state->mode.crtc_hdisplay - state->crtc_x; + + if (state->crtc_y < 0) + viewport_height += state->crtc_y; + else if (state->crtc_y + state->crtc_h > new_crtc_state->mode.crtc_vdisplay) + viewport_height = new_crtc_state->mode.crtc_vdisplay - state->crtc_y; + + /* If completely outside of screen, viewport_width and/or viewport_height will be negative, + * which is still OK to satisfy the condition below, thereby also covering these cases + * (when plane is completely outside of screen). + * x2 for width is because of pipe-split. + */ + if (viewport_width < MIN_VIEWPORT_SIZE*2 || viewport_height < MIN_VIEWPORT_SIZE) + return -EINVAL; + } + + /* Get min/max allowed scaling factors from plane caps. */ + get_min_max_dc_plane_scaling(state->crtc->dev, fb, + &min_downscale, &max_upscale); + /* + * Convert to drm convention: 16.16 fixed point, instead of dc's + * 1.0 == 1000. Also drm scaling is src/dst instead of dc's + * dst/src, so min_scale = 1.0 / max_upscale, etc. + */ + min_scale = (1000 << 16) / max_upscale; + max_scale = (1000 << 16) / min_downscale; + } - /* TODO: These should be checked against DC plane caps */ return drm_atomic_helper_check_plane_state( - state, new_crtc_state, max_downscale, max_upscale, true, true); + state, new_crtc_state, min_scale, max_scale, true, true); } static int dm_plane_atomic_check(struct drm_plane *plane, @@ -7548,7 +7716,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, struct drm_crtc *pcrtc, bool wait_for_vblank) { - int i; + uint32_t i; uint64_t timestamp_ns; struct drm_plane *plane; struct drm_plane_state *old_plane_state, *new_plane_state; @@ -7589,7 +7757,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, amdgpu_dm_commit_cursors(state); /* update planes when needed */ - for_each_oldnew_plane_in_state_reverse(state, plane, old_plane_state, new_plane_state, i) { + for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { struct drm_crtc *crtc = new_plane_state->crtc; struct drm_crtc_state *new_crtc_state; struct drm_framebuffer *fb = new_plane_state->fb; @@ -7812,7 +7980,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, bundle->surface_updates, planes_count, acrtc_state->stream, - &bundle->stream_update); + &bundle->stream_update, + dc_state); /** * Enable or disable the interrupts on the backend. @@ -8148,13 +8317,13 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); struct dm_connector_state *dm_old_con_state = to_dm_connector_state(old_con_state); struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - struct dc_surface_update surface_updates[MAX_SURFACES]; + struct dc_surface_update dummy_updates[MAX_SURFACES]; struct dc_stream_update stream_update; struct dc_info_packet hdr_packet; struct dc_stream_status *status = NULL; bool abm_changed, hdr_changed, scaling_changed; - memset(&surface_updates, 0, sizeof(surface_updates)); + memset(&dummy_updates, 0, sizeof(dummy_updates)); memset(&stream_update, 0, sizeof(stream_update)); if (acrtc) { @@ -8211,15 +8380,16 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) * To fix this, DC should permit updating only stream properties. */ for (j = 0; j < status->plane_count; j++) - surface_updates[j].surface = status->plane_states[j]; + dummy_updates[j].surface = status->plane_states[0]; mutex_lock(&dm->dc_lock); dc_commit_updates_for_stream(dm->dc, - surface_updates, + dummy_updates, status->plane_count, dm_new_crtc_state->stream, - &stream_update); + &stream_update, + dc_state); mutex_unlock(&dm->dc_lock); } @@ -9588,6 +9758,10 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector, amdgpu_dm_connector->max_vfreq = range->max_vfreq; amdgpu_dm_connector->pixel_clock_mhz = range->pixel_clock_mhz * 10; + + connector->display_info.monitor_range.min_vfreq = range->min_vfreq; + connector->display_info.monitor_range.max_vfreq = range->max_vfreq; + break; } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index 1182dafcef02..8bfe901cf237 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -58,10 +58,10 @@ /* Forward declarations */ struct amdgpu_device; struct drm_device; -struct amdgpu_dm_irq_handler_data; struct dc; struct amdgpu_bo; struct dmub_srv; +struct dc_plane_state; struct common_irq_params { struct amdgpu_device *adev; @@ -92,6 +92,20 @@ struct dm_compressor_info { uint64_t gpu_addr; }; +/** + * struct vblank_workqueue - Works to be executed in a separate thread during vblank + * @mall_work: work for mall stutter + * @dm: amdgpu display manager device + * @otg_inst: otg instance of which vblank is being set + * @enable: true if enable vblank + */ +struct vblank_workqueue { + struct work_struct mall_work; + struct amdgpu_display_manager *dm; + int otg_inst; + bool enable; +}; + /** * struct amdgpu_dm_backlight_caps - Information about backlight * @@ -243,6 +257,15 @@ struct amdgpu_display_manager { */ struct mutex audio_lock; + /** + * @vblank_work_lock: + * + * Guards access to deferred vblank work state. + */ +#if defined(CONFIG_DRM_AMD_DC_DCN) + spinlock_t vblank_lock; +#endif + /** * @audio_component: * @@ -321,6 +344,10 @@ struct amdgpu_display_manager { struct hdcp_workqueue *hdcp_workqueue; #endif +#if defined(CONFIG_DRM_AMD_DC_DCN) + struct vblank_workqueue *vblank_workqueue; +#endif + struct drm_atomic_state *cached_state; struct dc_state *cached_dc_state; @@ -336,6 +363,13 @@ struct amdgpu_display_manager { */ const struct gpu_info_soc_bounding_box_v1_0 *soc_bounding_box; + /** + * @active_vblank_irq_count: + * + * number of currently active vblank irqs + */ + uint32_t active_vblank_irq_count; + /** * @mst_encoders: * @@ -412,11 +446,6 @@ struct amdgpu_dm_connector { extern const struct amdgpu_ip_block_version dm_ip_block; -struct amdgpu_framebuffer; -struct amdgpu_display_manager; -struct dc_validation_set; -struct dc_plane_state; - struct dm_plane_state { struct drm_plane_state base; struct dc_plane_state *dc_state; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c index 11459fb09a37..360952129b6d 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c @@ -691,7 +691,7 @@ static ssize_t dp_phy_test_pattern_debugfs_write(struct file *f, const char __us return size; } -/** +/* * Returns the DMCUB tracebuffer contents. * Example usage: cat /sys/kernel/debug/dri/0/amdgpu_dm_dmub_tracebuffer */ @@ -735,7 +735,7 @@ static int dmub_tracebuffer_show(struct seq_file *m, void *data) return 0; } -/** +/* * Returns the DMCUB firmware state contents. * Example usage: cat /sys/kernel/debug/dri/0/amdgpu_dm_dmub_fw_state */ @@ -1063,7 +1063,7 @@ static int dp_dsc_fec_support_show(struct seq_file *m, void *data) * echo 0 > /sys/kernel/debug/dri/0/DP-X/trigger_hotplug * */ -static ssize_t dp_trigger_hotplug(struct file *f, const char __user *buf, +static ssize_t trigger_hotplug(struct file *f, const char __user *buf, size_t size, loff_t *pos) { struct amdgpu_dm_connector *aconnector = file_inode(f)->i_private; @@ -2214,9 +2214,9 @@ static const struct file_operations dp_dsc_slice_bpg_offset_debugfs_fops = { .llseek = default_llseek }; -static const struct file_operations dp_trigger_hotplug_debugfs_fops = { +static const struct file_operations trigger_hotplug_debugfs_fops = { .owner = THIS_MODULE, - .write = dp_trigger_hotplug, + .write = trigger_hotplug, .llseek = default_llseek }; @@ -2270,7 +2270,6 @@ static const struct { const struct file_operations *fops; } dp_debugfs_entries[] = { {"link_settings", &dp_link_settings_debugfs_fops}, - {"trigger_hotplug", &dp_trigger_hotplug_debugfs_fops}, {"phy_settings", &dp_phy_settings_debugfs_fop}, {"test_pattern", &dp_phy_test_pattern_fops}, #ifdef CONFIG_DRM_AMD_DC_HDCP @@ -2367,6 +2366,9 @@ void connector_debugfs_init(struct amdgpu_dm_connector *connector) debugfs_create_file("output_bpc", 0644, dir, connector, &output_bpc_fops); + debugfs_create_file("trigger_hotplug", 0644, dir, connector, + &trigger_hotplug_debugfs_fops); + connector->debugfs_dpcd_address = 0; connector->debugfs_dpcd_size = 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index c2cd184f0bbd..0cdbfcd475ec 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -376,7 +376,7 @@ static void event_cpirq(struct work_struct *work) } -void hdcp_destroy(struct hdcp_workqueue *hdcp_work) +void hdcp_destroy(struct kobject *kobj, struct hdcp_workqueue *hdcp_work) { int i = 0; @@ -385,6 +385,7 @@ void hdcp_destroy(struct hdcp_workqueue *hdcp_work) cancel_delayed_work_sync(&hdcp_work[i].watchdog_timer_dwork); } + sysfs_remove_bin_file(kobj, &hdcp_work[0].attr); kfree(hdcp_work->srm); kfree(hdcp_work->srm_temp); kfree(hdcp_work); @@ -449,11 +450,12 @@ static void update_config(void *handle, struct cp_psp_stream_config *config) link->mode = mod_hdcp_signal_type_to_operation_mode(aconnector->dc_sink->sink_signal); display->controller = CONTROLLER_ID_D0 + config->otg_inst; - display->dig_fe = config->stream_enc_inst; - link->dig_be = config->link_enc_inst; + display->dig_fe = config->dig_fe; + link->dig_be = config->dig_be; link->ddc_line = aconnector->dc_link->ddc_hw_inst + 1; link->dp.rev = aconnector->dc_link->dpcd_caps.dpcd_rev.raw; - link->dp.mst_supported = config->mst_supported; + link->dp.assr_enabled = config->assr_enabled; + link->dp.mst_enabled = config->mst_enabled; display->adjust.disable = 1; link->adjust.auth_delay = 3; link->adjust.hdcp1.disable = 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h index 5159b3a5e5b0..09294ff122fe 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h @@ -69,7 +69,7 @@ void hdcp_update_display(struct hdcp_workqueue *hdcp_work, void hdcp_reset_display(struct hdcp_workqueue *work, unsigned int link_index); void hdcp_handle_cpirq(struct hdcp_workqueue *work, unsigned int link_index); -void hdcp_destroy(struct hdcp_workqueue *work); +void hdcp_destroy(struct kobject *kobj, struct hdcp_workqueue *work); struct hdcp_workqueue *hdcp_create_workqueue(struct amdgpu_device *adev, struct cp_psp *cp_psp, struct dc *dc); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index f6f487e9fe2d..5159399f8239 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -25,7 +25,6 @@ #include #include -#include #include #include @@ -527,11 +526,11 @@ bool dm_helpers_submit_i2c( bool dm_helpers_dp_write_dsc_enable( struct dc_context *ctx, const struct dc_stream_state *stream, - bool enable -) + bool enable) { uint8_t enable_dsc = enable ? 1 : 0; struct amdgpu_dm_connector *aconnector; + uint8_t ret; if (!stream) return false; @@ -542,13 +541,13 @@ bool dm_helpers_dp_write_dsc_enable( if (!aconnector->dsc_aux) return false; - return (drm_dp_dpcd_write(aconnector->dsc_aux, DP_DSC_ENABLE, &enable_dsc, 1) >= 0); + ret = drm_dp_dpcd_write(aconnector->dsc_aux, DP_DSC_ENABLE, &enable_dsc, 1); } if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT) return dm_helpers_dp_write_dpcd(ctx, stream->link, DP_DSC_ENABLE, &enable_dsc, 1); - return false; + return (ret > 0); } bool dm_helpers_is_dp_sink_present(struct dc_link *link) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c index 26ed70e5538a..e0000c180ed1 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c @@ -662,6 +662,20 @@ static int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev, __func__); } +static int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + unsigned int crtc_id, + enum amdgpu_interrupt_state state) +{ + return dm_irq_state( + adev, + source, + crtc_id, + state, + IRQ_TYPE_VLINE0, + __func__); +} + static int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned int crtc_id, @@ -681,6 +695,11 @@ static const struct amdgpu_irq_src_funcs dm_crtc_irq_funcs = { .process = amdgpu_dm_irq_handler, }; +static const struct amdgpu_irq_src_funcs dm_vline0_irq_funcs = { + .set = amdgpu_dm_set_vline0_irq_state, + .process = amdgpu_dm_irq_handler, +}; + static const struct amdgpu_irq_src_funcs dm_vupdate_irq_funcs = { .set = amdgpu_dm_set_vupdate_irq_state, .process = amdgpu_dm_irq_handler, @@ -702,6 +721,9 @@ void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev) adev->crtc_irq.num_types = adev->mode_info.num_crtc; adev->crtc_irq.funcs = &dm_crtc_irq_funcs; + adev->vline0_irq.num_types = adev->mode_info.num_crtc; + adev->vline0_irq.funcs = &dm_vline0_irq_funcs; + adev->vupdate_irq.num_types = adev->mode_info.num_crtc; adev->vupdate_irq.funcs = &dm_vupdate_irq_funcs; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index f2d8cf34be46..41b09ab22233 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -23,7 +23,6 @@ * */ -#include #include #include #include diff --git a/drivers/gpu/drm/amd/display/dc/Makefile b/drivers/gpu/drm/amd/display/dc/Makefile index bf8fe0471b8f..5bf2f2375b40 100644 --- a/drivers/gpu/drm/amd/display/dc/Makefile +++ b/drivers/gpu/drm/amd/display/dc/Makefile @@ -69,5 +69,7 @@ AMD_DISPLAY_FILES += $(AMD_DISPLAY_CORE) AMD_DISPLAY_FILES += $(AMD_DM_REG_UPDATE) DC_DMUB += dc_dmub_srv.o +DC_EDID += dc_edid_parser.o AMD_DISPLAY_DMUB = $(addprefix $(AMDDALPATH)/dc/,$(DC_DMUB)) -AMD_DISPLAY_FILES += $(AMD_DISPLAY_DMUB) +AMD_DISPLAY_EDID = $(addprefix $(AMDDALPATH)/dc/,$(DC_EDID)) +AMD_DISPLAY_FILES += $(AMD_DISPLAY_DMUB) $(AMD_DISPLAY_EDID) diff --git a/drivers/gpu/drm/amd/display/dc/basics/conversion.c b/drivers/gpu/drm/amd/display/dc/basics/conversion.c index 24ed03d8cda7..6767fab55c26 100644 --- a/drivers/gpu/drm/amd/display/dc/basics/conversion.c +++ b/drivers/gpu/drm/amd/display/dc/basics/conversion.c @@ -73,12 +73,9 @@ uint16_t fixed_point_to_int_frac( return result; } -/** -* convert_float_matrix -* This converts a double into HW register spec defined format S2D13. -* @param : -* @return None -*/ +/* + * convert_float_matrix - This converts a double into HW register spec defined format S2D13. + */ void convert_float_matrix( uint16_t *matrix, struct fixed31_32 *flt, diff --git a/drivers/gpu/drm/amd/display/dc/basics/dc_common.c b/drivers/gpu/drm/amd/display/dc/basics/dc_common.c index b2fc4f8e6482..ad04ef98e652 100644 --- a/drivers/gpu/drm/amd/display/dc/basics/dc_common.c +++ b/drivers/gpu/drm/amd/display/dc/basics/dc_common.c @@ -49,20 +49,24 @@ bool is_rgb_cspace(enum dc_color_space output_color_space) } } -bool is_lower_pipe_tree_visible(struct pipe_ctx *pipe_ctx) +bool is_child_pipe_tree_visible(struct pipe_ctx *pipe_ctx) { if (pipe_ctx->plane_state && pipe_ctx->plane_state->visible) return true; - if (pipe_ctx->bottom_pipe && is_lower_pipe_tree_visible(pipe_ctx->bottom_pipe)) + if (pipe_ctx->bottom_pipe && is_child_pipe_tree_visible(pipe_ctx->bottom_pipe)) + return true; + if (pipe_ctx->next_odm_pipe && is_child_pipe_tree_visible(pipe_ctx->next_odm_pipe)) return true; return false; } -bool is_upper_pipe_tree_visible(struct pipe_ctx *pipe_ctx) +bool is_parent_pipe_tree_visible(struct pipe_ctx *pipe_ctx) { if (pipe_ctx->plane_state && pipe_ctx->plane_state->visible) return true; - if (pipe_ctx->top_pipe && is_upper_pipe_tree_visible(pipe_ctx->top_pipe)) + if (pipe_ctx->top_pipe && is_parent_pipe_tree_visible(pipe_ctx->top_pipe)) + return true; + if (pipe_ctx->prev_odm_pipe && is_parent_pipe_tree_visible(pipe_ctx->prev_odm_pipe)) return true; return false; } @@ -71,9 +75,13 @@ bool is_pipe_tree_visible(struct pipe_ctx *pipe_ctx) { if (pipe_ctx->plane_state && pipe_ctx->plane_state->visible) return true; - if (pipe_ctx->top_pipe && is_upper_pipe_tree_visible(pipe_ctx->top_pipe)) + if (pipe_ctx->top_pipe && is_parent_pipe_tree_visible(pipe_ctx->top_pipe)) return true; - if (pipe_ctx->bottom_pipe && is_lower_pipe_tree_visible(pipe_ctx->bottom_pipe)) + if (pipe_ctx->bottom_pipe && is_child_pipe_tree_visible(pipe_ctx->bottom_pipe)) + return true; + if (pipe_ctx->prev_odm_pipe && is_parent_pipe_tree_visible(pipe_ctx->prev_odm_pipe)) + return true; + if (pipe_ctx->next_odm_pipe && is_child_pipe_tree_visible(pipe_ctx->next_odm_pipe)) return true; return false; } diff --git a/drivers/gpu/drm/amd/display/dc/basics/dc_common.h b/drivers/gpu/drm/amd/display/dc/basics/dc_common.h index 7c0cbf47e8ce..b061497480b8 100644 --- a/drivers/gpu/drm/amd/display/dc/basics/dc_common.h +++ b/drivers/gpu/drm/amd/display/dc/basics/dc_common.h @@ -30,9 +30,9 @@ bool is_rgb_cspace(enum dc_color_space output_color_space); -bool is_lower_pipe_tree_visible(struct pipe_ctx *pipe_ctx); +bool is_child_pipe_tree_visible(struct pipe_ctx *pipe_ctx); -bool is_upper_pipe_tree_visible(struct pipe_ctx *pipe_ctx); +bool is_parent_pipe_tree_visible(struct pipe_ctx *pipe_ctx); bool is_pipe_tree_visible(struct pipe_ctx *pipe_ctx); diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c index 23a373ca94b5..c67d21a5ee52 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c @@ -911,11 +911,11 @@ static enum bp_result get_ss_info_from_tbl( * ver 2.1 can co-exist with SS_Info table. Expect ASIC_InternalSS_Info ver 3.1, * there is only one entry for each signal /ss id. However, there is * no planning of supporting multiple spread Sprectum entry for EverGreen - * @param [in] this - * @param [in] signal, ASSignalType to be converted to info index - * @param [in] index, number of entries that match the converted info index - * @param [out] ss_info, sprectrum information structure, - * @return Bios parser result code + * @dcb: pointer to the DC BIOS + * @signal: ASSignalType to be converted to info index + * @index: number of entries that match the converted info index + * @ss_info: sprectrum information structure, + * return: Bios parser result code */ static enum bp_result bios_parser_get_spread_spectrum_info( struct dc_bios *dcb, @@ -985,10 +985,10 @@ static enum bp_result get_ss_info_from_internal_ss_info_tbl_V2_1( * There can not be more than 1 entry for ASIC_InternalSS_Info Ver 2.1 or * SS_Info. * - * @param this - * @param id, spread sprectrum info index - * @param pSSinfo, sprectrum information structure, - * @return Bios parser result code + * @bp: pointer to the BIOS parser + * @id: spread sprectrum info index + * @ss_info: sprectrum information structure, + * return: BIOS parser result code */ static enum bp_result get_ss_info_from_tbl( struct bios_parser *bp, @@ -1011,9 +1011,10 @@ static enum bp_result get_ss_info_from_tbl( * from the VBIOS * There will not be multiple entry for Ver 2.1 * - * @param id, spread sprectrum info index - * @param pSSinfo, sprectrum information structure, - * @return Bios parser result code + * @bp: pointer to the Bios parser + * @id: spread sprectrum info index + * @info: sprectrum information structure, + * return: Bios parser result code */ static enum bp_result get_ss_info_from_internal_ss_info_tbl_V2_1( struct bios_parser *bp, @@ -1076,9 +1077,10 @@ static enum bp_result get_ss_info_from_internal_ss_info_tbl_V2_1( * of entries that matches the id * for, the SS_Info table, there should not be more than 1 entry match. * - * @param [in] id, spread sprectrum id - * @param [out] pSSinfo, sprectrum information structure, - * @return Bios parser result code + * @bp: pointer to the Bios parser + * @id: spread sprectrum id + * @ss_info: sprectrum information structure, + * return: Bios parser result code */ static enum bp_result get_ss_info_from_ss_info_table( struct bios_parser *bp, @@ -1451,16 +1453,14 @@ static enum bp_result get_embedded_panel_info_v1_3( } /** - * bios_parser_get_encoder_cap_info + * bios_parser_get_encoder_cap_info - get encoder capability + * information of input object id * - * @brief - * Get encoder capability information of input object id - * - * @param object_id, Object id - * @param object_id, encoder cap information structure - * - * @return Bios parser result code + * @dcb: pointer to the DC BIOS + * @object_id: object id + * @info: encoder cap information structure * + * return: Bios parser result code */ static enum bp_result bios_parser_get_encoder_cap_info( struct dc_bios *dcb, @@ -1490,17 +1490,12 @@ static enum bp_result bios_parser_get_encoder_cap_info( } /** - * get_encoder_cap_record + * get_encoder_cap_record - Get encoder cap record for the object * - * @brief - * Get encoder cap record for the object - * - * @param object, ATOM object - * - * @return atom encoder cap record - * - * @note - * search all records to find the ATOM_ENCODER_CAP_RECORD_V2 record + * @bp: pointer to the BIOS parser + * @object: ATOM object + * return: atom encoder cap record + * note: search all records to find the ATOM_ENCODER_CAP_RECORD_V2 record */ static ATOM_ENCODER_CAP_RECORD_V2 *get_encoder_cap_record( struct bios_parser *bp, @@ -1557,8 +1552,9 @@ static uint32_t get_ss_entry_number_from_ss_info_tbl( * Get Number of SpreadSpectrum Entry from the ASIC_InternalSS_Info table from * the VBIOS that match the SSid (to be converted from signal) * - * @param[in] signal, ASSignalType to be converted to SSid - * @return number of SS Entry that match the signal + * @dcb: pointer to the DC BIOS + * @signal: ASSignalType to be converted to SSid + * return: number of SS Entry that match the signal */ static uint32_t bios_parser_get_ss_entry_number( struct dc_bios *dcb, @@ -1608,10 +1604,10 @@ static uint32_t bios_parser_get_ss_entry_number( * get_ss_entry_number_from_ss_info_tbl * Get Number of spread spectrum entry from the SS_Info table from the VBIOS. * - * @note There can only be one entry for each id for SS_Info Table - * - * @param [in] id, spread spectrum id - * @return number of SS Entry that match the id + * @bp: pointer to the BIOS parser + * @id: spread spectrum id + * return: number of SS Entry that match the id + * note: There can only be one entry for each id for SS_Info Table */ static uint32_t get_ss_entry_number_from_ss_info_tbl( struct bios_parser *bp, @@ -1679,8 +1675,9 @@ static uint32_t get_ss_entry_number_from_ss_info_tbl( * There can not be more than 1 entry for ASIC_InternalSS_Info Ver 2.1 or * SS_Info. * - * @param id, spread sprectrum info index - * @return Bios parser result code + * @bp: pointer to the BIOS parser + * @id: spread sprectrum info index + * return: Bios parser result code */ static uint32_t get_ss_entry_number(struct bios_parser *bp, uint32_t id) { @@ -1696,8 +1693,9 @@ static uint32_t get_ss_entry_number(struct bios_parser *bp, uint32_t id) * Ver 2.1 from the VBIOS * There will not be multiple entry for Ver 2.1 * - * @param id, spread sprectrum info index - * @return number of SS Entry that match the id + * @bp: pointer to the BIOS parser + * @id: spread sprectrum info index + * return: number of SS Entry that match the id */ static uint32_t get_ss_entry_number_from_internal_ss_info_tbl_v2_1( struct bios_parser *bp, @@ -1731,8 +1729,9 @@ static uint32_t get_ss_entry_number_from_internal_ss_info_tbl_v2_1( * Get Number of SpreadSpectrum Entry from the ASIC_InternalSS_Info table of * the VBIOS that matches id * - * @param[in] id, spread sprectrum id - * @return number of SS Entry that match the id + * @bp: pointer to the BIOS parser + * @id: spread sprectrum id + * return: number of SS Entry that match the id */ static uint32_t get_ss_entry_number_from_internal_ss_info_tbl_V3_1( struct bios_parser *bp, @@ -1767,10 +1766,11 @@ static uint32_t get_ss_entry_number_from_internal_ss_info_tbl_V3_1( * bios_parser_get_gpio_pin_info * Get GpioPin information of input gpio id * - * @param gpio_id, GPIO ID - * @param info, GpioPin information structure - * @return Bios parser result code - * @note + * @dcb: pointer to the DC BIOS + * @gpio_id: GPIO ID + * @info: GpioPin information structure + * return: Bios parser result code + * note: * to get the GPIO PIN INFO, we need: * 1. get the GPIO_ID from other object table, see GetHPDInfo() * 2. in DATA_TABLE.GPIO_Pin_LUT, search all records, to get the registerA @@ -2197,13 +2197,10 @@ static uint32_t get_support_mask_for_device_id(struct device_id device_id) } /** - * bios_parser_set_scratch_critical_state - * - * @brief - * update critical state bit in VBIOS scratch register - * - * @param - * bool - to set or reset state + * bios_parser_set_scratch_critical_state - update critical state + * bit in VBIOS scratch register + * @dcb: pointer to the DC BIOS + * @state: set or reset state */ static void bios_parser_set_scratch_critical_state( struct dc_bios *dcb, @@ -2222,7 +2219,7 @@ static void bios_parser_set_scratch_critical_state( * bios_parser *bp - [in]BIOS parser handler to get master data table * integrated_info *info - [out] store and output integrated info * - * @return + * return: * enum bp_result - BP_RESULT_OK if information is available, * BP_RESULT_BADBIOSTABLE otherwise. */ @@ -2372,7 +2369,7 @@ static enum bp_result get_integrated_info_v8( * bios_parser *bp - [in]BIOS parser handler to get master data table * integrated_info *info - [out] store and output integrated info * - * @return + * return: * enum bp_result - BP_RESULT_OK if information is available, * BP_RESULT_BADBIOSTABLE otherwise. */ @@ -2509,7 +2506,7 @@ static enum bp_result get_integrated_info_v9( * bios_parser *bp - [in]BIOS parser handler to get master data table * integrated_info *info - [out] store and output integrated info * - * @return + * return: * enum bp_result - BP_RESULT_OK if information is available, * BP_RESULT_BADBIOSTABLE otherwise. */ @@ -2585,7 +2582,7 @@ static struct integrated_info *bios_parser_create_integrated_info( return NULL; } -enum bp_result update_slot_layout_info( +static enum bp_result update_slot_layout_info( struct dc_bios *dcb, unsigned int i, struct slot_layout_info *slot_layout_info, @@ -2689,7 +2686,7 @@ enum bp_result update_slot_layout_info( } -enum bp_result get_bracket_layout_record( +static enum bp_result get_bracket_layout_record( struct dc_bios *dcb, unsigned int bracket_layout_id, struct slot_layout_info *slot_layout_info) diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c index 670c26583817..9f9fda3118d1 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c @@ -485,10 +485,11 @@ static struct atom_hpd_int_record *get_hpd_record( * bios_parser_get_gpio_pin_info * Get GpioPin information of input gpio id * - * @param gpio_id, GPIO ID - * @param info, GpioPin information structure - * @return Bios parser result code - * @note + * @dcb: pointer to the DC BIOS + * @gpio_id: GPIO ID + * @info: GpioPin information structure + * return: Bios parser result code + * note: * to get the GPIO PIN INFO, we need: * 1. get the GPIO_ID from other object table, see GetHPDInfo() * 2. in DATA_TABLE.GPIO_Pin_LUT, search all records, @@ -801,11 +802,11 @@ static enum bp_result get_ss_info_v4_2( * ver 3.1, * there is only one entry for each signal /ss id. However, there is * no planning of supporting multiple spread Sprectum entry for EverGreen - * @param [in] this - * @param [in] signal, ASSignalType to be converted to info index - * @param [in] index, number of entries that match the converted info index - * @param [out] ss_info, sprectrum information structure, - * @return Bios parser result code + * @dcb: pointer to the DC BIOS + * @signal: ASSignalType to be converted to info index + * @index: number of entries that match the converted info index + * @ss_info: sprectrum information structure, + * return: Bios parser result code */ static enum bp_result bios_parser_get_spread_spectrum_info( struct dc_bios *dcb, @@ -1196,13 +1197,11 @@ static bool bios_parser_is_accelerated_mode( } /** - * bios_parser_set_scratch_critical_state + * bios_parser_set_scratch_critical_state - update critical state bit + * in VBIOS scratch register * - * @brief - * update critical state bit in VBIOS scratch register - * - * @param - * bool - to set or reset state + * @dcb: pointer to the DC BIO + * @state: set or reset state */ static void bios_parser_set_scratch_critical_state( struct dc_bios *dcb, diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c index 070459e3e407..afc10b954ffa 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c @@ -245,6 +245,23 @@ static enum bp_result encoder_control_digx_v3( cntl->enable_dp_audio); params.ucLaneNum = (uint8_t)(cntl->lanes_number); + switch (cntl->color_depth) { + case COLOR_DEPTH_888: + params.ucBitPerColor = PANEL_8BIT_PER_COLOR; + break; + case COLOR_DEPTH_101010: + params.ucBitPerColor = PANEL_10BIT_PER_COLOR; + break; + case COLOR_DEPTH_121212: + params.ucBitPerColor = PANEL_12BIT_PER_COLOR; + break; + case COLOR_DEPTH_161616: + params.ucBitPerColor = PANEL_16BIT_PER_COLOR; + break; + default: + break; + } + if (EXEC_BIOS_CMD_TABLE(DIGxEncoderControl, params)) result = BP_RESULT_OK; @@ -274,6 +291,23 @@ static enum bp_result encoder_control_digx_v4( cntl->enable_dp_audio)); params.ucLaneNum = (uint8_t)(cntl->lanes_number); + switch (cntl->color_depth) { + case COLOR_DEPTH_888: + params.ucBitPerColor = PANEL_8BIT_PER_COLOR; + break; + case COLOR_DEPTH_101010: + params.ucBitPerColor = PANEL_10BIT_PER_COLOR; + break; + case COLOR_DEPTH_121212: + params.ucBitPerColor = PANEL_12BIT_PER_COLOR; + break; + case COLOR_DEPTH_161616: + params.ucBitPerColor = PANEL_16BIT_PER_COLOR; + break; + default: + break; + } + if (EXEC_BIOS_CMD_TABLE(DIGxEncoderControl, params)) result = BP_RESULT_OK; @@ -1057,6 +1091,19 @@ static enum bp_result set_pixel_clock_v5( * driver choose program it itself, i.e. here we program it * to 888 by default. */ + if (bp_params->signal_type == SIGNAL_TYPE_HDMI_TYPE_A) + switch (bp_params->color_depth) { + case TRANSMITTER_COLOR_DEPTH_30: + /* yes this is correct, the atom define is wrong */ + clk.sPCLKInput.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_32BPP; + break; + case TRANSMITTER_COLOR_DEPTH_36: + /* yes this is correct, the atom define is wrong */ + clk.sPCLKInput.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_30BPP; + break; + default: + break; + } if (EXEC_BIOS_CMD_TABLE(SetPixelClock, clk)) result = BP_RESULT_OK; @@ -1135,6 +1182,20 @@ static enum bp_result set_pixel_clock_v6( * driver choose program it itself, i.e. here we pass required * target rate that includes deep color. */ + if (bp_params->signal_type == SIGNAL_TYPE_HDMI_TYPE_A) + switch (bp_params->color_depth) { + case TRANSMITTER_COLOR_DEPTH_30: + clk.sPCLKInput.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_30BPP_V6; + break; + case TRANSMITTER_COLOR_DEPTH_36: + clk.sPCLKInput.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_36BPP_V6; + break; + case TRANSMITTER_COLOR_DEPTH_48: + clk.sPCLKInput.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_48BPP; + break; + default: + break; + } if (EXEC_BIOS_CMD_TABLE(SetPixelClock, clk)) result = BP_RESULT_OK; diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c index 48b4ef03fc8f..5b77251e0590 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c @@ -114,18 +114,14 @@ bool dal_cmd_table_helper_controller_id_to_atom( } /** -* translate_transmitter_bp_to_atom -* -* @brief -* Translate the Transmitter to the corresponding ATOM BIOS value -* -* @param -* input transmitter -* output digitalTransmitter -* // =00: Digital Transmitter1 ( UNIPHY linkAB ) -* // =01: Digital Transmitter2 ( UNIPHY linkCD ) -* // =02: Digital Transmitter3 ( UNIPHY linkEF ) -*/ + * translate_transmitter_bp_to_atom - Translate the Transmitter to the + * corresponding ATOM BIOS value + * @t: transmitter + * returns: output digitalTransmitter + * // =00: Digital Transmitter1 ( UNIPHY linkAB ) + * // =01: Digital Transmitter2 ( UNIPHY linkCD ) + * // =02: Digital Transmitter3 ( UNIPHY linkEF ) + */ uint8_t dal_cmd_table_helper_transmitter_bp_to_atom( enum transmitter t) { diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c index 7736c92d55c4..455ee2be15a3 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c @@ -128,18 +128,14 @@ bool dal_cmd_table_helper_controller_id_to_atom2( } /** -* translate_transmitter_bp_to_atom -* -* @brief -* Translate the Transmitter to the corresponding ATOM BIOS value -* -* @param -* input transmitter -* output digitalTransmitter -* // =00: Digital Transmitter1 ( UNIPHY linkAB ) -* // =01: Digital Transmitter2 ( UNIPHY linkCD ) -* // =02: Digital Transmitter3 ( UNIPHY linkEF ) -*/ + * translate_transmitter_bp_to_atom2 - Translate the Transmitter to the + * corresponding ATOM BIOS value + * @t: transmitter + * returns: digitalTransmitter + * // =00: Digital Transmitter1 ( UNIPHY linkAB ) + * // =01: Digital Transmitter2 ( UNIPHY linkCD ) + * // =02: Digital Transmitter3 ( UNIPHY linkEF ) + */ uint8_t dal_cmd_table_helper_transmitter_bp_to_atom2( enum transmitter t) { diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c index ef41b287cbe2..e633f8a51edb 100644 --- a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c +++ b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c @@ -106,7 +106,6 @@ static void calculate_bandwidth( bool lpt_enabled; enum bw_defines sclk_message; enum bw_defines yclk_message; - enum bw_defines v_filter_init_mode[maximum_number_of_surfaces]; enum bw_defines tiling_mode[maximum_number_of_surfaces]; enum bw_defines surface_type[maximum_number_of_surfaces]; enum bw_defines voltage; @@ -792,12 +791,8 @@ static void calculate_bandwidth( data->v_filter_init[i] = bw_add(data->v_filter_init[i], bw_int_to_fixed(1)); } if (data->stereo_mode[i] == bw_def_top_bottom) { - v_filter_init_mode[i] = bw_def_manual; data->v_filter_init[i] = bw_min2(data->v_filter_init[i], bw_int_to_fixed(4)); } - else { - v_filter_init_mode[i] = bw_def_auto; - } if (data->stereo_mode[i] == bw_def_top_bottom) { data->num_lines_at_frame_start = bw_int_to_fixed(1); } @@ -2730,7 +2725,7 @@ void bw_calcs_init(struct bw_calcs_dceip *bw_dceip, } -/** +/* * Compare calculated (required) clocks against the clocks available at * maximum voltage (max Performance Level). */ @@ -3001,13 +2996,12 @@ static bool all_displays_in_sync(const struct pipe_ctx pipe[], return true; } -/** +/* * Return: * true - Display(s) configuration supported. * In this case 'calcs_output' contains data for HW programming * false - Display(s) configuration not supported (not enough bandwidth). */ - bool bw_calcs(struct dc_context *ctx, const struct bw_calcs_dceip *dceip, const struct bw_calcs_vbios *vbios, @@ -3028,7 +3022,7 @@ bool bw_calcs(struct dc_context *ctx, calcs_output->all_displays_in_sync = false; if (data->number_of_displays != 0) { - uint8_t yclk_lvl, sclk_lvl; + uint8_t yclk_lvl; struct bw_fixed high_sclk = vbios->high_sclk; struct bw_fixed mid1_sclk = vbios->mid1_sclk; struct bw_fixed mid2_sclk = vbios->mid2_sclk; @@ -3049,7 +3043,6 @@ bool bw_calcs(struct dc_context *ctx, calculate_bandwidth(dceip, vbios, data); yclk_lvl = data->y_clk_level; - sclk_lvl = data->sclk_level; calcs_output->nbp_state_change_enable = data->nbp_state_change_enable; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c index 75b8240ed059..e133edc587d3 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c @@ -187,17 +187,6 @@ static void ramp_up_dispclk_with_dpp( clk_mgr->base.clks.max_supported_dppclk_khz = new_clocks->max_supported_dppclk_khz; } -static bool is_mpo_enabled(struct dc_state *context) -{ - int i; - - for (i = 0; i < context->stream_count; i++) { - if (context->stream_status[i].plane_count > 1) - return true; - } - return false; -} - static void rv1_update_clocks(struct clk_mgr *clk_mgr_base, struct dc_state *context, bool safe_to_lower) @@ -295,22 +284,9 @@ static void rv1_update_clocks(struct clk_mgr *clk_mgr_base, if (pp_smu->set_hard_min_fclk_by_freq && pp_smu->set_hard_min_dcfclk_by_freq && pp_smu->set_min_deep_sleep_dcfclk) { - // Only increase clocks when display is active and MPO is enabled - if (display_count && is_mpo_enabled(context)) { - pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, - ((new_clocks->fclk_khz / 1000) * 101) / 100); - pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, - ((new_clocks->dcfclk_khz / 1000) * 101) / 100); - pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, - (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000); - } else { - pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, - new_clocks->fclk_khz / 1000); - pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, - new_clocks->dcfclk_khz / 1000); - pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, - (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000); - } + pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, new_clocks->fclk_khz / 1000); + pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, new_clocks->dcfclk_khz / 1000); + pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000); } } } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c index f2114bc910bf..ec9dc265cde0 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c @@ -257,8 +257,7 @@ void dcn2_update_clocks(struct clk_mgr *clk_mgr_base, if (update_dppclk || update_dispclk) dcn20_update_clocks_update_dentist(clk_mgr); // always update dtos unless clock is lowered and not safe to lower - if (new_clocks->dppclk_khz >= dc->current_state->bw_ctx.bw.dcn.clk.dppclk_khz) - dcn20_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower); + dcn20_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower); } } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c index ab98c259ef69..c7e5a64e06af 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c @@ -146,15 +146,15 @@ static noinline void dcn3_build_wm_range_table(struct clk_mgr_internal *clk_mgr) clk_mgr->base.bw_params->wm_table.nv_entries[WM_C].pmfw_breakdown.max_uclk = 0xFFFF; /* Set D - MALL - SR enter and exit times adjusted for MALL */ -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].valid = true; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.pstate_latency_us = pstate_latency_us; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.sr_exit_time_us = 2; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.sr_enter_plus_exit_time_us = 4; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.wm_type = WATERMARKS_MALL; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.min_dcfclk = 0; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_dcfclk = 0xFFFF; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.min_uclk = min_uclk_mhz; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_uclk = 0xFFFF; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].valid = true; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.pstate_latency_us = pstate_latency_us; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.sr_exit_time_us = 2; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.sr_enter_plus_exit_time_us = 4; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.wm_type = WATERMARKS_MALL; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.min_dcfclk = 0; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_dcfclk = 0xFFFF; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.min_uclk = min_uclk_mhz; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_uclk = 0xFFFF; } void dcn3_init_clocks(struct clk_mgr *clk_mgr_base) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/dcn301_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/dcn301_smu.c index cfa8e02cf103..68942bbc7472 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/dcn301_smu.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/dcn301_smu.c @@ -103,7 +103,7 @@ int dcn301_smu_send_msg_with_param( /* Trigger the message transaction by writing the message ID */ REG_WRITE(MP1_SMN_C2PMSG_67, msg_id); - result = dcn301_smu_wait_for_response(clk_mgr, 10, 1000); + result = dcn301_smu_wait_for_response(clk_mgr, 10, 200000); ASSERT(result == VBIOSSMC_Result_OK); diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c index 991b9c5beaa3..aadb801447a7 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c @@ -75,7 +75,8 @@ int vg_get_active_display_cnt_wa( const struct dc_link *link = dc->links[i]; /* abusing the fact that the dig and phy are coupled to see if the phy is enabled */ - if (link->link_enc->funcs->is_dig_enabled(link->link_enc)) + if (link->link_enc->funcs->is_dig_enabled && + link->link_enc->funcs->is_dig_enabled(link->link_enc)) display_count++; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 6cf1a5a2a5ec..8f8a13c7cf73 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -175,6 +175,8 @@ static bool create_links( connectors_num = bios->funcs->get_connectors_number(bios); + DC_LOG_DC("BIOS object table - number of connectors: %d", connectors_num); + if (connectors_num > ENUM_ID_COUNT) { dm_error( "DC: Number of connectors %d exceeds maximum of %d!\n", @@ -193,6 +195,8 @@ static bool create_links( struct link_init_data link_init_params = {0}; struct dc_link *link; + DC_LOG_DC("BIOS object table - printing link object info for connector number: %d, link_index: %d", i, dc->link_count); + link_init_params.ctx = dc->ctx; /* next BIOS object table connector */ link_init_params.connector_index = i; @@ -201,30 +205,14 @@ static bool create_links( link = link_create(&link_init_params); if (link) { - bool should_destory_link = false; - - if (link->connector_signal == SIGNAL_TYPE_EDP) { - if (dc->config.edp_not_connected) { - if (!IS_DIAG_DC(dc->ctx->dce_environment)) - should_destory_link = true; - } else { - enum dc_connection_type type; - dc_link_detect_sink(link, &type); - if (type == dc_connection_none) - should_destory_link = true; - } - } - - if (dc->config.force_enum_edp || !should_destory_link) { dc->links[dc->link_count] = link; link->dc = dc; ++dc->link_count; - } else { - link_destroy(&link); - } } } + DC_LOG_DC("BIOS object table - end"); + for (i = 0; i < num_virtual_links; i++) { struct dc_link *link = kzalloc(sizeof(*link), GFP_KERNEL); struct encoder_init_data enc_init = {0}; @@ -284,20 +272,16 @@ static void dc_perf_trace_destroy(struct dc_perf_trace **perf_trace) } /** - ***************************************************************************** - * Function: dc_stream_adjust_vmin_vmax + * dc_stream_adjust_vmin_vmax: * - * @brief - * Looks up the pipe context of dc_stream_state and updates the - * vertical_total_min and vertical_total_max of the DRR, Dynamic Refresh - * Rate, which is a power-saving feature that targets reducing panel - * refresh rate while the screen is static + * Looks up the pipe context of dc_stream_state and updates the + * vertical_total_min and vertical_total_max of the DRR, Dynamic Refresh + * Rate, which is a power-saving feature that targets reducing panel + * refresh rate while the screen is static * - * @param [in] dc: dc reference - * @param [in] stream: Initial dc stream state - * @param [in] adjust: Updated parameters for vertical_total_min and - * vertical_total_max - ***************************************************************************** + * @dc: dc reference + * @stream: Initial dc stream state + * @adjust: Updated parameters for vertical_total_min and vertical_total_max */ bool dc_stream_adjust_vmin_vmax(struct dc *dc, struct dc_stream_state *stream, @@ -355,6 +339,7 @@ bool dc_stream_get_crtc_position(struct dc *dc, * @dc: DC Object * @stream: The stream to configure CRC on. * @enable: Enable CRC if true, disable otherwise. + * @crc_window: CRC window (x/y start/end) information * @continuous: Capture CRC on every frame if true. Otherwise, only capture * once. * @@ -420,7 +405,9 @@ bool dc_stream_configure_crc(struct dc *dc, struct dc_stream_state *stream, * dc_stream_get_crc() - Get CRC values for the given stream. * @dc: DC object * @stream: The DC stream state of the stream to get CRCs from. - * @r_cr, g_y, b_cb: CRC values for the three channels are stored here. + * @r_cr: CRC value for the first of the 3 channels stored here. + * @g_y: CRC value for the second of the 3 channels stored here. + * @b_cb: CRC value for the third of the 3 channels stored here. * * dc_stream_configure_crc needs to be called beforehand to enable CRCs. * Return false if stream is not found, or if CRCs are not enabled. @@ -707,7 +694,6 @@ static bool dc_construct(struct dc *dc, } dc->dcn_ip = dcn_ip; - dc->soc_bounding_box = init_params->soc_bounding_box; #endif if (!dc_construct_ctx(dc, init_params)) { @@ -757,6 +743,10 @@ static bool dc_construct(struct dc *dc, if (!dc->res_pool) goto fail; + /* set i2c speed if not done by the respective dcnxxx__resource.c */ + if (dc->caps.i2c_speed_in_khz_hdcp == 0) + dc->caps.i2c_speed_in_khz_hdcp = dc->caps.i2c_speed_in_khz; + dc->clk_mgr = dc_clk_mgr_create(dc->ctx, dc->res_pool->pp_smu, dc->res_pool->dccg); if (!dc->clk_mgr) goto fail; @@ -764,8 +754,6 @@ static bool dc_construct(struct dc *dc, dc->clk_mgr->force_smu_not_present = init_params->force_smu_not_present; #endif - dc->debug.force_ignore_link_settings = init_params->force_ignore_link_settings; - if (dc->res_pool->funcs->update_bw_bounding_box) dc->res_pool->funcs->update_bw_bounding_box(dc, dc->clk_mgr->bw_params); @@ -803,7 +791,8 @@ static void disable_all_writeback_pipes_for_stream( stream->writeback_info[i].wb_enabled = false; } -void apply_ctx_interdependent_lock(struct dc *dc, struct dc_state *context, struct dc_stream_state *stream, bool lock) +static void apply_ctx_interdependent_lock(struct dc *dc, struct dc_state *context, + struct dc_stream_state *stream, bool lock) { int i = 0; @@ -964,19 +953,15 @@ struct dc *dc_create(const struct dc_init_data *init_params) struct dc *dc = kzalloc(sizeof(*dc), GFP_KERNEL); unsigned int full_pipe_count; - if (NULL == dc) - goto alloc_fail; + if (!dc) + return NULL; if (init_params->dce_environment == DCE_ENV_VIRTUAL_HW) { - if (false == dc_construct_ctx(dc, init_params)) { - dc_destruct(dc); - goto construct_fail; - } + if (!dc_construct_ctx(dc, init_params)) + goto destruct_dc; } else { - if (false == dc_construct(dc, init_params)) { - dc_destruct(dc); - goto construct_fail; - } + if (!dc_construct(dc, init_params)) + goto destruct_dc; full_pipe_count = dc->res_pool->pipe_count; if (dc->res_pool->underlay_pipe_index != NO_UNDERLAY_PIPE) @@ -1007,15 +992,36 @@ struct dc *dc_create(const struct dc_init_data *init_params) return dc; -construct_fail: +destruct_dc: + dc_destruct(dc); kfree(dc); - -alloc_fail: return NULL; } +static void detect_edp_presence(struct dc *dc) +{ + struct dc_link *edp_link = get_edp_link(dc); + bool edp_sink_present = true; + + if (!edp_link) + return; + + if (dc->config.edp_not_connected) { + edp_sink_present = false; + } else { + enum dc_connection_type type; + dc_link_detect_sink(edp_link, &type); + if (type == dc_connection_none) + edp_sink_present = false; + } + + edp_link->edp_sink_present = edp_sink_present; +} + void dc_hardware_init(struct dc *dc) { + + detect_edp_presence(dc); if (dc->ctx->dce_environment != DCE_ENV_VIRTUAL_HW) dc->hwss.init_hw(dc); } @@ -1493,7 +1499,7 @@ bool dc_commit_state(struct dc *dc, struct dc_state *context) enum dc_status result = DC_ERROR_UNEXPECTED; int i; - if (false == context_changed(dc, context)) + if (!context_changed(dc, context)) return DC_OK; DC_LOG_DC("%s: %d streams\n", @@ -1540,7 +1546,7 @@ bool dc_acquire_release_mpc_3dlut( if (found_pipe_idx) { if (acquire && pool->funcs->acquire_post_bldn_3dlut) ret = pool->funcs->acquire_post_bldn_3dlut(res_ctx, pool, mpcc_id, lut, shaper); - else if (acquire == false && pool->funcs->release_post_bldn_3dlut) + else if (!acquire && pool->funcs->release_post_bldn_3dlut) ret = pool->funcs->release_post_bldn_3dlut(res_ctx, pool, lut, shaper); } } @@ -2016,7 +2022,7 @@ static enum surface_update_type check_update_surfaces_for_stream( return overall_type; } -/** +/* * dc_check_update_surfaces_for_stream() - Determine update type (fast, med, or full) * * See :c:type:`enum surface_update_type ` for explanation of update types @@ -2270,6 +2276,9 @@ static void copy_stream_update_to_stream(struct dc *dc, if (update->dither_option) stream->dither_option = *update->dither_option; + + if (update->pending_test_pattern) + stream->test_pattern = *update->pending_test_pattern; /* update current stream with writeback info */ if (update->wb_update) { int i; @@ -2366,6 +2375,15 @@ static void commit_planes_do_stream_update(struct dc *dc, } } + if (stream_update->pending_test_pattern) { + dc_link_dp_set_test_pattern(stream->link, + stream->test_pattern.type, + stream->test_pattern.color_space, + stream->test_pattern.p_link_settings, + stream->test_pattern.p_custom_pattern, + stream->test_pattern.cust_pattern_size); + } + /* Full fe update*/ if (update_type == UPDATE_TYPE_FAST) continue; @@ -2679,7 +2697,8 @@ void dc_commit_updates_for_stream(struct dc *dc, struct dc_surface_update *srf_updates, int surface_count, struct dc_stream_state *stream, - struct dc_stream_update *stream_update) + struct dc_stream_update *stream_update, + struct dc_state *state) { const struct dc_stream_status *stream_status; enum surface_update_type update_type; @@ -2698,12 +2717,6 @@ void dc_commit_updates_for_stream(struct dc *dc, if (update_type >= UPDATE_TYPE_FULL) { - struct dc_plane_state *new_planes[MAX_SURFACES]; - - memset(new_planes, 0, sizeof(new_planes)); - - for (i = 0; i < surface_count; i++) - new_planes[i] = srf_updates[i].surface; /* initialize scratch memory for building context */ context = dc_create_state(dc); @@ -2712,21 +2725,15 @@ void dc_commit_updates_for_stream(struct dc *dc, return; } - dc_resource_state_copy_construct( - dc->current_state, context); + dc_resource_state_copy_construct(state, context); - /*remove old surfaces from context */ - if (!dc_rem_all_planes_for_stream(dc, stream, context)) { - DC_ERROR("Failed to remove streams for new validate context!\n"); - return; + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *new_pipe = &context->res_ctx.pipe_ctx[i]; + struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i]; + + if (new_pipe->plane_state && new_pipe->plane_state != old_pipe->plane_state) + new_pipe->plane_state->force_full_update = true; } - - /* add surface to context */ - if (!dc_add_all_planes_for_stream(dc, stream, new_planes, surface_count, context)) { - DC_ERROR("Failed to add streams for new validate context!\n"); - return; - } - } @@ -2830,7 +2837,7 @@ enum dc_irq_source dc_interrupt_to_irq_source( return dal_irq_service_to_irq_source(dc->res_pool->irqs, src_id, ext_id); } -/** +/* * dc_interrupt_set() - Enable/disable an AMD hw interrupt source */ bool dc_interrupt_set(struct dc *dc, enum dc_irq_source src, bool enable) @@ -2964,7 +2971,7 @@ static bool link_add_remote_sink_helper(struct dc_link *dc_link, struct dc_sink return true; } -/** +/* * dc_link_add_remote_sink() - Create a sink and attach it to an existing link * * EDID length is in bytes @@ -3027,7 +3034,7 @@ fail_add_sink: return NULL; } -/** +/* * dc_link_remove_remote_sink() - Remove a remote sink from a dc_link * * Note that this just removes the struct dc_sink - it doesn't @@ -3154,9 +3161,11 @@ void dc_lock_memory_clock_frequency(struct dc *dc) core_link_enable_stream(dc->current_state, &dc->current_state->res_ctx.pipe_ctx[i]); } -bool dc_is_plane_eligible_for_idle_optimizaitons(struct dc *dc, - struct dc_plane_state *plane) +bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc, struct dc_plane_state *plane, + struct dc_cursor_attributes *cursor_attr) { + if (dc->hwss.does_plane_fit_in_mall && dc->hwss.does_plane_fit_in_mall(dc, plane, cursor_attr)) + return true; return false; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index f4a2088ab179..fa5059f71727 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -203,9 +203,21 @@ static bool program_hpd_filter(const struct dc_link *link) return result; } +bool dc_link_wait_for_t12(struct dc_link *link) +{ + if (link->connector_signal == SIGNAL_TYPE_EDP && link->dc->hwss.edp_wait_for_T12) { + link->dc->hwss.edp_wait_for_T12(link); + + return true; + } + + return false; +} + /** * dc_link_detect_sink() - Determine if there is a sink connected * + * @link: pointer to the dc link * @type: Returned connection type * Does not detect downstream devices, such as MST sinks * or display connected through active dongles @@ -342,7 +354,7 @@ static enum signal_type get_basic_signal_type(struct graphics_object_id encoder, return SIGNAL_TYPE_NONE; } -/** +/* * dc_link_is_dp_sink_present() - Check if there is a native DP * or passive DP-HDMI dongle connected */ @@ -596,8 +608,6 @@ static void query_hdcp_capability(enum signal_type signal, struct dc_link *link) dc_process_hdcp_msg(signal, link, &msg22); if (signal == SIGNAL_TYPE_DISPLAY_PORT || signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { - enum hdcp_message_status status = HDCP_MESSAGE_UNSUPPORTED; - msg14.data = &link->hdcp_caps.bcaps.raw; msg14.length = sizeof(link->hdcp_caps.bcaps.raw); msg14.msg_id = HDCP_MESSAGE_ID_READ_BCAPS; @@ -605,7 +615,7 @@ static void query_hdcp_capability(enum signal_type signal, struct dc_link *link) msg14.link = HDCP_LINK_PRIMARY; msg14.max_retries = 5; - status = dc_process_hdcp_msg(signal, link, &msg14); + dc_process_hdcp_msg(signal, link, &msg14); } } @@ -830,7 +840,7 @@ static bool wait_for_entering_dp_alt_mode(struct dc_link *link) return false; } -/** +/* * dc_link_detect() - Detect if a sink is attached to a given link * * link->local_sink is created or destroyed as needed. @@ -1065,9 +1075,6 @@ static bool dc_link_detect_helper(struct dc_link *link, break; } - if (link->local_sink->edid_caps.panel_patch.disable_fec) - link->ctx->dc->debug.disable_fec = true; - // Check if edid is the same if ((prev_sink) && (edid_status == EDID_THE_SAME || edid_status == EDID_OK)) @@ -1366,13 +1373,17 @@ static bool dc_link_construct(struct dc_link *link, struct dc_context *dc_ctx = init_params->ctx; struct encoder_init_data enc_init_data = { 0 }; struct panel_cntl_init_data panel_cntl_init_data = { 0 }; - struct integrated_info info = {{{ 0 }}}; + struct integrated_info *info; struct dc_bios *bios = init_params->dc->ctx->dc_bios; const struct dc_vbios_funcs *bp_funcs = bios->funcs; struct bp_disp_connector_caps_info disp_connect_caps_info = { 0 }; DC_LOGGER_INIT(dc_ctx->logger); + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + goto create_fail; + link->irq_source_hpd = DC_IRQ_SOURCE_INVALID; link->irq_source_hpd_rx = DC_IRQ_SOURCE_INVALID; @@ -1390,10 +1401,12 @@ static bool dc_link_construct(struct dc_link *link, link->link_id = bios->funcs->get_connector_id(bios, init_params->connector_index); + DC_LOG_DC("BIOS object table - link_id: %d", link->link_id.id); if (bios->funcs->get_disp_connector_caps_info) { bios->funcs->get_disp_connector_caps_info(bios, link->link_id, &disp_connect_caps_info); link->is_internal_display = disp_connect_caps_info.INTERNAL_DISPLAY; + DC_LOG_DC("BIOS object table - is_internal_display: %d", link->is_internal_display); } if (link->link_id.type != OBJECT_TYPE_CONNECTOR) { @@ -1408,10 +1421,14 @@ static bool dc_link_construct(struct dc_link *link, link->hpd_gpio = get_hpd_gpio(link->ctx->dc_bios, link->link_id, link->ctx->gpio_service); + if (link->hpd_gpio) { dal_gpio_open(link->hpd_gpio, GPIO_MODE_INTERRUPT); dal_gpio_unlock_pin(link->hpd_gpio); link->irq_source_hpd = dal_irq_get_source(link->hpd_gpio); + + DC_LOG_DC("BIOS object table - hpd_gpio id: %d", link->hpd_gpio->id); + DC_LOG_DC("BIOS object table - hpd_gpio en: %d", link->hpd_gpio->en); } switch (link->link_id.id) { @@ -1470,6 +1487,11 @@ static bool dc_link_construct(struct dc_link *link, goto ddc_create_fail; } + if (!link->ddc->ddc_pin) { + DC_ERROR("Failed to get I2C info for connector!\n"); + goto ddc_create_fail; + } + link->ddc_hw_inst = dal_ddc_get_line(dal_ddc_service_get_ddc_pin(link->ddc)); @@ -1508,6 +1530,8 @@ static bool dc_link_construct(struct dc_link *link, goto link_enc_create_fail; } + DC_LOG_DC("BIOS object table - DP_IS_USB_C: %d", link->link_enc->features.flags.bits.DP_IS_USB_C); + link->link_enc_hw_inst = link->link_enc->transmitter; for (i = 0; i < 4; i++) { @@ -1530,16 +1554,20 @@ static bool dc_link_construct(struct dc_link *link, if (link->device_tag.dev_id.device_type == DEVICE_TYPE_LCD && link->connector_signal == SIGNAL_TYPE_RGB) continue; + + DC_LOG_DC("BIOS object table - device_tag.acpi_device: %d", link->device_tag.acpi_device); + DC_LOG_DC("BIOS object table - device_tag.dev_id.device_type: %d", link->device_tag.dev_id.device_type); + DC_LOG_DC("BIOS object table - device_tag.dev_id.enum_id: %d", link->device_tag.dev_id.enum_id); break; } if (bios->integrated_info) - info = *bios->integrated_info; + memcpy(info, bios->integrated_info, sizeof(*info)); /* Look for channel mapping corresponding to connector and device tag */ for (i = 0; i < MAX_NUMBER_OF_EXT_DISPLAY_PATH; i++) { struct external_display_path *path = - &info.ext_disp_conn_info.path[i]; + &info->ext_disp_conn_info.path[i]; if (path->device_connector_id.enum_id == link->link_id.enum_id && path->device_connector_id.id == link->link_id.id && @@ -1548,10 +1576,14 @@ static bool dc_link_construct(struct dc_link *link, path->device_acpi_enum == link->device_tag.acpi_device) { link->ddi_channel_mapping = path->channel_mapping; link->chip_caps = path->caps; + DC_LOG_DC("BIOS object table - ddi_channel_mapping: 0x%04X", link->ddi_channel_mapping.raw); + DC_LOG_DC("BIOS object table - chip_caps: %d", link->chip_caps); } else if (path->device_tag == link->device_tag.dev_id.raw_device_tag) { link->ddi_channel_mapping = path->channel_mapping; link->chip_caps = path->caps; + DC_LOG_DC("BIOS object table - ddi_channel_mapping: 0x%04X", link->ddi_channel_mapping.raw); + DC_LOG_DC("BIOS object table - chip_caps: %d", link->chip_caps); } break; } @@ -1570,6 +1602,7 @@ static bool dc_link_construct(struct dc_link *link, link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED; + DC_LOG_DC("BIOS object table - %s finished successfully.\n", __func__); return true; device_tag_fail: link->link_enc->funcs->destroy(&link->link_enc); @@ -1586,6 +1619,9 @@ create_fail: link->hpd_gpio = NULL; } + DC_LOG_DC("BIOS object table - %s failed.\n", __func__); + kfree(info); + return false; } @@ -3133,17 +3169,17 @@ static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off) { struct cp_psp *cp_psp = &pipe_ctx->stream->ctx->cp_psp; if (cp_psp && cp_psp->funcs.update_stream_config) { - struct cp_psp_stream_config config; - - memset(&config, 0, sizeof(config)); + struct cp_psp_stream_config config = {0}; + enum dp_panel_mode panel_mode = + dp_get_panel_mode(pipe_ctx->stream->link); config.otg_inst = (uint8_t) pipe_ctx->stream_res.tg->inst; - /*stream_enc_inst*/ - config.stream_enc_inst = (uint8_t) pipe_ctx->stream_res.stream_enc->stream_enc_inst; - config.link_enc_inst = pipe_ctx->stream->link->link_enc_hw_inst; + config.dig_fe = (uint8_t) pipe_ctx->stream_res.stream_enc->stream_enc_inst; + config.dig_be = pipe_ctx->stream->link->link_enc_hw_inst; config.dpms_off = dpms_off; config.dm_stream_ctx = pipe_ctx->stream->dm_stream_context; - config.mst_supported = (pipe_ctx->stream->signal == + config.assr_enabled = (panel_mode == DP_PANEL_MODE_EDP); + config.mst_enabled = (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST); cp_psp->funcs.update_stream_config(cp_psp->handle, &config); } @@ -3396,10 +3432,7 @@ void core_link_set_avmute(struct pipe_ctx *pipe_ctx, bool enable) } /** - ***************************************************************************** - * Function: dc_link_enable_hpd_filter - * - * @brief + * dc_link_enable_hpd_filter: * If enable is true, programs HPD filter on associated HPD line using * delay_on_disconnect/delay_on_connect values dependent on * link->connector_signal @@ -3407,9 +3440,8 @@ void core_link_set_avmute(struct pipe_ctx *pipe_ctx, bool enable) * If enable is false, programs HPD filter on associated HPD line with no * delays on connect or disconnect * - * @param [in] link: pointer to the dc link - * @param [in] enable: boolean specifying whether to enable hbd - ***************************************************************************** + * @link: pointer to the dc link + * @enable: boolean specifying whether to enable hbd */ void dc_link_enable_hpd_filter(struct dc_link *link, bool enable) { @@ -3635,7 +3667,7 @@ uint32_t dc_link_bandwidth_kbps( link_bw_kbps *= 8; /* 8 bits per byte*/ link_bw_kbps *= link_setting->lane_count; - if (dc_link_is_fec_supported(link) && !link->dc->debug.disable_fec) { + if (dc_link_should_enable_fec(link)) { /* Account for FEC overhead. * We have to do it based on caps, * and not based on FEC being set ready, @@ -3656,8 +3688,8 @@ uint32_t dc_link_bandwidth_kbps( * but the difference is minimal and is in a safe direction, * which all works well around potential ambiguity of DP 1.4a spec. */ - link_bw_kbps = mul_u64_u32_shr(BIT_ULL(32) * 970LL / 1000, - link_bw_kbps, 32); + long long fec_link_bw_kbps = link_bw_kbps * 970LL; + link_bw_kbps = (uint32_t)(div64_s64(fec_link_bw_kbps, 1000LL)); } return link_bw_kbps; @@ -3687,3 +3719,19 @@ bool dc_link_is_fec_supported(const struct dc_link *link) !IS_FPGA_MAXIMUS_DC(link->ctx->dce_environment)); } +bool dc_link_should_enable_fec(const struct dc_link *link) +{ + bool is_fec_disable = false; + bool ret = false; + + if ((link->connector_signal != SIGNAL_TYPE_DISPLAY_PORT_MST && + link->local_sink && + link->local_sink->edid_caps.panel_patch.disable_fec) || + link->connector_signal == SIGNAL_TYPE_EDP) // Disable FEC for eDP + is_fec_disable = true; + + if (dc_link_is_fec_supported(link) && !link->dc->debug.disable_fec && !is_fec_disable) + ret = true; + + return ret; +} diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c index c5936e064360..ae6484ab567b 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c @@ -37,12 +37,16 @@ #include "dc_link_ddc.h" #include "dce/dce_aux.h" -/*DP to Dual link DVI converter*/ +#define DC_LOGGER_INIT(logger) + +static const uint8_t DP_VGA_DONGLE_BRANCH_DEV_NAME[] = "DpVga"; +/* DP to Dual link DVI converter */ static const uint8_t DP_DVI_CONVERTER_ID_4[] = "m2DVIa"; static const uint8_t DP_DVI_CONVERTER_ID_5[] = "3393N2"; #define AUX_POWER_UP_WA_DELAY 500 #define I2C_OVER_AUX_DEFER_WA_DELAY 70 +#define DPVGA_DONGLE_AUX_DEFER_WA_DELAY 40 #define I2C_OVER_AUX_DEFER_WA_DELAY_1MS 1 /* CV smart dongle slave address for retrieving supported HDTV modes*/ @@ -194,6 +198,10 @@ static void ddc_service_construct( if (BP_RESULT_OK != dcb->funcs->get_i2c_info(dcb, init_data->id, &i2c_info)) { ddc_service->ddc_pin = NULL; } else { + DC_LOGGER_INIT(ddc_service->ctx->logger); + DC_LOG_DC("BIOS object table - i2c_line: %d", i2c_info.i2c_line); + DC_LOG_DC("BIOS object table - i2c_engine_id: %d", i2c_info.i2c_engine_id); + hw_info.ddc_channel = i2c_info.i2c_line; if (ddc_service->link != NULL) hw_info.hw_supported = i2c_info.i2c_hw_assist; @@ -286,6 +294,15 @@ static uint32_t defer_delay_converter_wa( { struct dc_link *link = ddc->link; + if (link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_VGA_CONVERTER && + link->dpcd_caps.branch_dev_id == DP_BRANCH_DEVICE_ID_0080E1 && + !memcmp(link->dpcd_caps.branch_dev_name, + DP_VGA_DONGLE_BRANCH_DEV_NAME, + sizeof(link->dpcd_caps.branch_dev_name))) + + return defer_delay > DPVGA_DONGLE_AUX_DEFER_WA_DELAY ? + defer_delay : DPVGA_DONGLE_AUX_DEFER_WA_DELAY; + if (link->dpcd_caps.branch_dev_id == DP_BRANCH_DEVICE_ID_0080E1 && !memcmp(link->dpcd_caps.branch_dev_name, DP_DVI_CONVERTER_ID_4, diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index 1e4794e2825c..c1391bfb7a9b 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -126,9 +126,7 @@ static void dpcd_set_training_pattern( static enum dc_dp_training_pattern decide_cr_training_pattern( const struct dc_link_settings *link_settings) { - enum dc_dp_training_pattern pattern = DP_TRAINING_PATTERN_SEQUENCE_1; - - return pattern; + return DP_TRAINING_PATTERN_SEQUENCE_1; } static enum dc_dp_training_pattern decide_eq_training_pattern(struct dc_link *link, @@ -3710,7 +3708,7 @@ bool detect_dp_sink_caps(struct dc_link *link) /* TODO save sink caps in link->sink */ } -enum dc_link_rate linkRateInKHzToLinkRateMultiplier(uint32_t link_rate_in_khz) +static enum dc_link_rate linkRateInKHzToLinkRateMultiplier(uint32_t link_rate_in_khz) { enum dc_link_rate link_rate; // LinkRate is normally stored as a multiplier of 0.27 Gbps per lane. Do the translation. @@ -4348,7 +4346,7 @@ void dp_set_fec_ready(struct dc_link *link, bool ready) struct link_encoder *link_enc = link->link_enc; uint8_t fec_config = 0; - if (!dc_link_is_fec_supported(link) || link->dc->debug.disable_fec) + if (!dc_link_should_enable_fec(link)) return; if (link_enc->funcs->fec_set_ready && @@ -4383,7 +4381,7 @@ void dp_set_fec_enable(struct dc_link *link, bool enable) { struct link_encoder *link_enc = link->link_enc; - if (!dc_link_is_fec_supported(link) || link->dc->debug.disable_fec) + if (!dc_link_should_enable_fec(link)) return; if (link_enc->funcs->fec_set_enable && @@ -4409,25 +4407,40 @@ void dp_set_fec_enable(struct dc_link *link, bool enable) void dpcd_set_source_specific_data(struct dc_link *link) { if (!link->dc->vendor_signature.is_valid) { - enum dc_status result_write_min_hblank = DC_NOT_SUPPORTED; - struct dpcd_amd_signature amd_signature; - amd_signature.AMD_IEEE_TxSignature_byte1 = 0x0; - amd_signature.AMD_IEEE_TxSignature_byte2 = 0x0; - amd_signature.AMD_IEEE_TxSignature_byte3 = 0x1A; - amd_signature.device_id_byte1 = - (uint8_t)(link->ctx->asic_id.chip_id); - amd_signature.device_id_byte2 = - (uint8_t)(link->ctx->asic_id.chip_id >> 8); - memset(&amd_signature.zero, 0, 4); - amd_signature.dce_version = - (uint8_t)(link->ctx->dce_version); - amd_signature.dal_version_byte1 = 0x0; // needed? where to get? - amd_signature.dal_version_byte2 = 0x0; // needed? where to get? + enum dc_status __maybe_unused result_write_min_hblank = DC_NOT_SUPPORTED; + struct dpcd_amd_signature amd_signature = {0}; + struct dpcd_amd_device_id amd_device_id = {0}; - core_link_write_dpcd(link, DP_SOURCE_OUI, + amd_device_id.device_id_byte1 = + (uint8_t)(link->ctx->asic_id.chip_id); + amd_device_id.device_id_byte2 = + (uint8_t)(link->ctx->asic_id.chip_id >> 8); + amd_device_id.dce_version = + (uint8_t)(link->ctx->dce_version); + amd_device_id.dal_version_byte1 = 0x0; // needed? where to get? + amd_device_id.dal_version_byte2 = 0x0; // needed? where to get? + + core_link_read_dpcd(link, DP_SOURCE_OUI, (uint8_t *)(&amd_signature), sizeof(amd_signature)); + if (!((amd_signature.AMD_IEEE_TxSignature_byte1 == 0x0) && + (amd_signature.AMD_IEEE_TxSignature_byte2 == 0x0) && + (amd_signature.AMD_IEEE_TxSignature_byte3 == 0x1A))) { + + amd_signature.AMD_IEEE_TxSignature_byte1 = 0x0; + amd_signature.AMD_IEEE_TxSignature_byte2 = 0x0; + amd_signature.AMD_IEEE_TxSignature_byte3 = 0x1A; + + core_link_write_dpcd(link, DP_SOURCE_OUI, + (uint8_t *)(&amd_signature), + sizeof(amd_signature)); + } + + core_link_write_dpcd(link, DP_SOURCE_OUI+0x03, + (uint8_t *)(&amd_device_id), + sizeof(amd_device_id)); + if (link->ctx->dce_version >= DCN_VERSION_2_0 && link->dc->caps.min_horizontal_blanking_period != 0) { diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 07c22556480b..0c26c2ade782 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -1117,7 +1117,7 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx) * We also need to make sure pipe_ctx->plane_res.scl_data.h_active uses the * original h_border_left value in its calculation. */ -int shift_border_left_to_dst(struct pipe_ctx *pipe_ctx) +static int shift_border_left_to_dst(struct pipe_ctx *pipe_ctx) { int store_h_border_left = pipe_ctx->stream->timing.h_border_left; @@ -1128,8 +1128,8 @@ int shift_border_left_to_dst(struct pipe_ctx *pipe_ctx) return store_h_border_left; } -void restore_border_left_from_dst(struct pipe_ctx *pipe_ctx, - int store_h_border_left) +static void restore_border_left_from_dst(struct pipe_ctx *pipe_ctx, + int store_h_border_left) { pipe_ctx->stream->dst.x -= store_h_border_left; pipe_ctx->stream->timing.h_border_left = store_h_border_left; @@ -1153,8 +1153,8 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx) calculate_viewport(pipe_ctx); - if (pipe_ctx->plane_res.scl_data.viewport.height < 12 || - pipe_ctx->plane_res.scl_data.viewport.width < 12) { + if (pipe_ctx->plane_res.scl_data.viewport.height < MIN_VIEWPORT_SIZE || + pipe_ctx->plane_res.scl_data.viewport.width < MIN_VIEWPORT_SIZE) { if (store_h_border_left) { restore_border_left_from_dst(pipe_ctx, store_h_border_left); @@ -1697,7 +1697,7 @@ static bool are_stream_backends_same( return true; } -/** +/* * dc_is_stream_unchanged() - Compare two stream states for equivalence. * * Checks if there a difference between the two states @@ -1718,7 +1718,7 @@ bool dc_is_stream_unchanged( return true; } -/** +/* * dc_is_stream_scaling_unchanged() - Compare scaling rectangles of two streams. */ bool dc_is_stream_scaling_unchanged( @@ -1833,7 +1833,7 @@ static struct audio *find_first_free_audio( return 0; } -/** +/* * dc_add_stream_to_ctx() - Add a new dc_stream_state to a dc_state. */ enum dc_status dc_add_stream_to_ctx( @@ -1860,7 +1860,7 @@ enum dc_status dc_add_stream_to_ctx( return res; } -/** +/* * dc_remove_stream_from_ctx() - Remove a stream from a dc_state. */ enum dc_status dc_remove_stream_from_ctx( @@ -2075,6 +2075,20 @@ static int acquire_resource_from_hw_enabled_state( return -1; } +static void mark_seamless_boot_stream( + const struct dc *dc, + struct dc_stream_state *stream) +{ + struct dc_bios *dcb = dc->ctx->dc_bios; + + /* TODO: Check Linux */ + if (dc->config.allow_seamless_boot_optimization && + !dcb->funcs->is_accelerated_mode(dcb)) { + if (dc_validate_seamless_boot_timing(dc, stream->sink, &stream->timing)) + stream->apply_seamless_boot_optimization = true; + } +} + enum dc_status resource_map_pool_resources( const struct dc *dc, struct dc_state *context, @@ -2085,22 +2099,20 @@ enum dc_status resource_map_pool_resources( struct dc_context *dc_ctx = dc->ctx; struct pipe_ctx *pipe_ctx = NULL; int pipe_idx = -1; - struct dc_bios *dcb = dc->ctx->dc_bios; calculate_phy_pix_clks(stream); - /* TODO: Check Linux */ - if (dc->config.allow_seamless_boot_optimization && - !dcb->funcs->is_accelerated_mode(dcb)) { - if (dc_validate_seamless_boot_timing(dc, stream->sink, &stream->timing)) - stream->apply_seamless_boot_optimization = true; - } + mark_seamless_boot_stream(dc, stream); - if (stream->apply_seamless_boot_optimization) + if (stream->apply_seamless_boot_optimization) { pipe_idx = acquire_resource_from_hw_enabled_state( &context->res_ctx, pool, stream); + if (pipe_idx < 0) + /* hw resource was assigned to other stream */ + stream->apply_seamless_boot_optimization = false; + } if (pipe_idx < 0) /* acquire new resources */ diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c index c103f858375d..25fa712a7847 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c @@ -244,7 +244,7 @@ struct dc_stream_status *dc_stream_get_status( } #ifndef TRIM_FSFT -/** +/* * dc_optimize_timing_for_fsft() - dc to optimize timing */ bool dc_optimize_timing_for_fsft( @@ -260,8 +260,7 @@ bool dc_optimize_timing_for_fsft( } #endif - -/** +/* * dc_stream_set_cursor_attributes() - Update cursor attributes and set cursor surface address */ bool dc_stream_set_cursor_attributes( diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c index 3d7d27435f15..e6b9c6a71841 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c @@ -115,7 +115,7 @@ struct dc_plane_state *dc_create_plane_state(struct dc *dc) return plane_state; } -/** +/* ***************************************************************************** * Function: dc_plane_get_status * diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 3aedadb34548..4eee3a55fa30 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -42,12 +42,13 @@ #include "inc/hw/dmcu.h" #include "dml/display_mode_lib.h" -#define DC_VER "3.2.116" +#define DC_VER "3.2.122" #define MAX_SURFACES 3 #define MAX_PLANES 6 #define MAX_STREAMS 6 #define MAX_SINKS_PER_LINK 4 +#define MIN_VIEWPORT_SIZE 12 /******************************************************************************* * Display Core Interfaces @@ -171,6 +172,9 @@ struct dc_caps { bool dmcub_support; uint32_t num_of_internal_disp; enum dp_protocol_version max_dp_protocol_version; + unsigned int mall_size_per_mem_channel; + unsigned int mall_size_total; + unsigned int cursor_cache_size; struct dc_plane_cap planes[MAX_PLANES]; struct dc_color_caps color; }; @@ -481,7 +485,6 @@ struct dc_debug_options { bool performance_trace; bool az_endpoint_mute_only; bool always_use_regamma; - bool p010_mpo_support; bool recovery_enabled; bool avoid_vbios_exec_table; bool scl_reset_length10; @@ -499,6 +502,9 @@ struct dc_debug_options { bool dmcub_emulation; #if defined(CONFIG_DRM_AMD_DC_DCN) bool disable_idle_power_optimizations; + unsigned int mall_size_override; + unsigned int mall_additional_timer_percent; + bool mall_error_as_fatal; #endif bool dmub_command_table; /* for testing only */ struct dc_bw_validation_profile bw_val_profile; @@ -521,7 +527,6 @@ struct dc_debug_options { bool usbc_combo_phy_reset_wa; bool disable_dsc; bool enable_dram_clock_change_one_display_vactive; - bool force_ignore_link_settings; union mem_low_power_enable_options enable_mem_low_power; }; @@ -633,7 +638,6 @@ struct dc { const char *build_id; struct vm_helper *vm_helper; - const struct gpu_info_soc_bounding_box_v1_0 *soc_bounding_box; }; enum frame_buffer_mode { @@ -671,16 +675,10 @@ struct dc_init_data { struct dc_config flags; uint64_t log_mask; - /** - * gpu_info FW provided soc bounding box struct or 0 if not - * available in FW - */ - const struct gpu_info_soc_bounding_box_v1_0 *soc_bounding_box; struct dpcd_vendor_signature vendor_signature; #if defined(CONFIG_DRM_AMD_DC_DCN) bool force_smu_not_present; #endif - bool force_ignore_link_settings; }; struct dc_callback_init { @@ -1269,8 +1267,8 @@ enum dc_status dc_set_clock(struct dc *dc, enum dc_clock_type clock_type, uint32 void dc_get_clock(struct dc *dc, enum dc_clock_type clock_type, struct dc_clock_config *clock_cfg); #if defined(CONFIG_DRM_AMD_DC_DCN) -bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc, - struct dc_plane_state *plane); +bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc, struct dc_plane_state *plane, + struct dc_cursor_attributes *cursor_attr); void dc_allow_idle_optimizations(struct dc *dc, bool allow); diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h index 80a2191a3115..cc6fb838420e 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h @@ -451,6 +451,9 @@ struct dpcd_amd_signature { uint8_t AMD_IEEE_TxSignature_byte1; uint8_t AMD_IEEE_TxSignature_byte2; uint8_t AMD_IEEE_TxSignature_byte3; +}; + +struct dpcd_amd_device_id { uint8_t device_id_byte1; uint8_t device_id_byte2; uint8_t zero[4]; diff --git a/drivers/gpu/drm/amd/display/dc/dc_edid_parser.c b/drivers/gpu/drm/amd/display/dc/dc_edid_parser.c new file mode 100644 index 000000000000..0db5b49e9d5e --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dc_edid_parser.c @@ -0,0 +1,80 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include "dce/dce_dmcu.h" +#include "dc_edid_parser.h" + +bool dc_edid_parser_send_cea(struct dc *dc, + int offset, + int total_length, + uint8_t *data, + int length) +{ + struct dmcu *dmcu = dc->res_pool->dmcu; + + if (dmcu && + dmcu->funcs->is_dmcu_initialized(dmcu) && + dmcu->funcs->send_edid_cea) { + return dmcu->funcs->send_edid_cea(dmcu, + offset, + total_length, + data, + length); + } + + return false; +} + +bool dc_edid_parser_recv_cea_ack(struct dc *dc, int *offset) +{ + struct dmcu *dmcu = dc->res_pool->dmcu; + + if (dmcu && + dmcu->funcs->is_dmcu_initialized(dmcu) && + dmcu->funcs->recv_edid_cea_ack) { + return dmcu->funcs->recv_edid_cea_ack(dmcu, offset); + } + + return false; +} + +bool dc_edid_parser_recv_amd_vsdb(struct dc *dc, + int *version, + int *min_frame_rate, + int *max_frame_rate) +{ + struct dmcu *dmcu = dc->res_pool->dmcu; + + if (dmcu && + dmcu->funcs->is_dmcu_initialized(dmcu) && + dmcu->funcs->recv_amd_vsdb) { + return dmcu->funcs->recv_amd_vsdb(dmcu, + version, + min_frame_rate, + max_frame_rate); + } + + return false; +} diff --git a/drivers/gpu/drm/amd/display/dc/dc_edid_parser.h b/drivers/gpu/drm/amd/display/dc/dc_edid_parser.h new file mode 100644 index 000000000000..da67ec06f0a2 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dc_edid_parser.h @@ -0,0 +1,44 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef _DC_EDID_PARSER_H_ +#define _DC_EDID_PARSER_H_ + +#include "core_types.h" + +bool dc_edid_parser_send_cea(struct dc *dc, + int offset, + int total_length, + uint8_t *data, + int length); + +bool dc_edid_parser_recv_cea_ack(struct dc *dc, int *offset); + +bool dc_edid_parser_recv_amd_vsdb(struct dc *dc, + int *version, + int *min_frame_rate, + int *max_frame_rate); + +#endif /* _DC_EDID_PARSER_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c index 57edb25fc381..a612ba6dc389 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_helper.c +++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c @@ -34,6 +34,7 @@ #include "dc.h" #include "dc_dmub_srv.h" +#include "reg_helper.h" static inline void submit_dmub_read_modify_write( struct dc_reg_helper_state *offload, diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h index 701aa7178a89..b41e6367b15e 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h @@ -71,6 +71,7 @@ struct dc_plane_address { union { struct{ PHYSICAL_ADDRESS_LOC addr; + PHYSICAL_ADDRESS_LOC cursor_cache_addr; PHYSICAL_ADDRESS_LOC meta_addr; union large_integer dcc_const_color; } grph; diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h index 6d9a60c9dcc0..e189f16bc026 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_link.h +++ b/drivers/gpu/drm/amd/display/dc/dc_link.h @@ -103,6 +103,8 @@ struct dc_link { bool lttpr_non_transparent_mode; bool is_internal_display; + bool edp_sink_present; + /* caps is the same as reported_link_cap. link_traing use * reported_link_cap. Will clean up. TODO */ @@ -259,6 +261,13 @@ enum dc_status dc_link_reallocate_mst_payload(struct dc_link *link); bool dc_link_handle_hpd_rx_irq(struct dc_link *dc_link, union hpd_irq_data *hpd_irq_dpcd_data, bool *out_link_loss); +/* + * On eDP links this function call will stall until T12 has elapsed. + * If the panel is not in power off state, this function will return + * immediately. + */ +bool dc_link_wait_for_t12(struct dc_link *link); + enum dc_status read_hpd_rx_irq_data( struct dc_link *link, union hpd_irq_data *irq_data); @@ -369,5 +378,6 @@ uint32_t dc_bandwidth_in_kbps_from_timing( const struct dc_crtc_timing *timing); bool dc_link_is_fec_supported(const struct dc_link *link); +bool dc_link_should_enable_fec(const struct dc_link *link); #endif /* DC_LINK_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index e243c01b9672..80b67b860091 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -130,6 +130,14 @@ union stream_update_flags { uint32_t raw; }; +struct test_pattern { + enum dp_test_pattern type; + enum dp_test_pattern_color_space color_space; + struct link_training_settings const *p_link_settings; + unsigned char const *p_custom_pattern; + unsigned int cust_pattern_size; +}; + struct dc_stream_state { // sink is deprecated, new code should not reference // this pointer @@ -227,6 +235,8 @@ struct dc_stream_state { uint32_t stream_id; bool is_dsc_enabled; + + struct test_pattern test_pattern; union stream_update_flags update_flags; }; @@ -261,6 +271,7 @@ struct dc_stream_update { struct dc_dsc_config *dsc_config; struct dc_transfer_func *func_shaper; struct dc_3dlut *lut3d_func; + struct test_pattern *pending_test_pattern; }; bool dc_is_stream_unchanged( @@ -283,7 +294,8 @@ void dc_commit_updates_for_stream(struct dc *dc, struct dc_surface_update *srf_updates, int surface_count, struct dc_stream_state *stream, - struct dc_stream_update *stream_update); + struct dc_stream_update *stream_update, + struct dc_state *state); /* * Log the current stream state. */ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c index 2a2a0fdb9253..7866cf2a668f 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c @@ -868,7 +868,7 @@ void dce_aud_wall_dto_setup( } #if defined(CONFIG_DRM_AMD_DC_SI) -void dce60_aud_wall_dto_setup( +static void dce60_aud_wall_dto_setup( struct audio *audio, enum signal_type signal, const struct audio_crtc_info *crtc_info, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c index cda5fd0464bc..d51b5fe91287 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c @@ -388,12 +388,6 @@ static enum aux_channel_operation_result get_channel_status( } } -enum i2caux_engine_type get_engine_type( - const struct dce_aux *engine) -{ - return I2CAUX_ENGINE_TYPE_AUX; -} - static bool acquire( struct dce_aux *engine, struct ddc *ddc) @@ -582,7 +576,7 @@ int dce_aux_transfer_raw(struct ddc_service *ddc, *operation_result = get_channel_status(aux_engine, &returned_bytes); if (*operation_result == AUX_CHANNEL_OPERATION_SUCCEEDED) { - int bytes_replied = 0; + int __maybe_unused bytes_replied = 0; bytes_replied = read_channel_reply(aux_engine, payload->length, payload->data, payload->reply, &status); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h index 382465862f29..277484cf853e 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h @@ -124,7 +124,6 @@ struct dce110_aux_registers { AUX_SF(AUX_SW_CONTROL, AUX_SW_GO, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_AUTOINCREMENT_DISABLE, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_DATA_RW, mask_sh),\ - AUX_SF(AUX_SW_DATA, AUX_SW_AUTOINCREMENT_DISABLE, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_INDEX, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_DATA, mask_sh),\ AUX_SF(AUX_SW_STATUS, AUX_SW_REPLY_BYTE_COUNT, mask_sh),\ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c index fb733f573715..dec58b3c42e4 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c @@ -113,20 +113,19 @@ static const struct spread_spectrum_data *get_ss_data_entry( } /** - * Function: calculate_fb_and_fractional_fb_divider + * calculate_fb_and_fractional_fb_divider - Calculates feedback and fractional + * feedback dividers values * - * * DESCRIPTION: Calculates feedback and fractional feedback dividers values + * @calc_pll_cs: Pointer to clock source information + * @target_pix_clk_100hz: Desired frequency in 100 Hz + * @ref_divider: Reference divider (already known) + * @post_divider: Post Divider (already known) + * @feedback_divider_param: Pointer where to store + * calculated feedback divider value + * @fract_feedback_divider_param: Pointer where to store + * calculated fract feedback divider value * - *PARAMETERS: - * targetPixelClock Desired frequency in 100 Hz - * ref_divider Reference divider (already known) - * postDivider Post Divider (already known) - * feedback_divider_param Pointer where to store - * calculated feedback divider value - * fract_feedback_divider_param Pointer where to store - * calculated fract feedback divider value - * - *RETURNS: + * return: * It fills the locations pointed by feedback_divider_param * and fract_feedback_divider_param * It returns - true if feedback divider not 0 @@ -175,22 +174,22 @@ static bool calculate_fb_and_fractional_fb_divider( } /** -*calc_fb_divider_checking_tolerance -* -*DESCRIPTION: Calculates Feedback and Fractional Feedback divider values -* for passed Reference and Post divider, checking for tolerance. -*PARAMETERS: -* pll_settings Pointer to structure -* ref_divider Reference divider (already known) -* postDivider Post Divider (already known) -* tolerance Tolerance for Calculated Pixel Clock to be within -* -*RETURNS: -* It fills the PLLSettings structure with PLL Dividers values -* if calculated values are within required tolerance -* It returns - true if error is within tolerance -* - false if error is not within tolerance -*/ + * calc_fb_divider_checking_tolerance - Calculates Feedback and + * Fractional Feedback divider values + * for passed Reference and Post divider, + * checking for tolerance. + * @calc_pll_cs: Pointer to clock source information + * @pll_settings: Pointer to PLL settings + * @ref_divider: Reference divider (already known) + * @post_divider: Post Divider (already known) + * @tolerance: Tolerance for Calculated Pixel Clock to be within + * + * return: + * It fills the PLLSettings structure with PLL Dividers values + * if calculated values are within required tolerance + * It returns - true if error is within tolerance + * - false if error is not within tolerance + */ static bool calc_fb_divider_checking_tolerance( struct calc_pll_clock_source *calc_pll_cs, struct pll_settings *pll_settings, @@ -241,7 +240,7 @@ static bool calc_fb_divider_checking_tolerance( pll_settings->calculated_pix_clk_100hz = actual_calculated_clock_100hz; pll_settings->vco_freq = - actual_calculated_clock_100hz * post_divider / 10; + div_u64((u64)actual_calculated_clock_100hz * post_divider, 10); return true; } return false; @@ -460,7 +459,7 @@ static bool pll_adjust_pix_clk( return false; } -/** +/* * Calculate PLL Dividers for given Clock Value. * First will call VBIOS Adjust Exec table to check if requested Pixel clock * will be Adjusted based on usage. @@ -871,6 +870,20 @@ static bool dce110_program_pix_clk( bp_pc_params.flags.SET_EXTERNAL_REF_DIV_SRC = pll_settings->use_external_clk; + switch (pix_clk_params->color_depth) { + case COLOR_DEPTH_101010: + bp_pc_params.color_depth = TRANSMITTER_COLOR_DEPTH_30; + break; + case COLOR_DEPTH_121212: + bp_pc_params.color_depth = TRANSMITTER_COLOR_DEPTH_36; + break; + case COLOR_DEPTH_161616: + bp_pc_params.color_depth = TRANSMITTER_COLOR_DEPTH_48; + break; + default: + break; + } + if (clk_src->bios->funcs->set_pixel_clock( clk_src->bios, &bp_pc_params) != BP_RESULT_OK) return false; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c index f3ed8b619caf..ddc789daf3b1 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c @@ -57,6 +57,9 @@ #define MCP_SYNC_PHY_LOCK 0x90 #define MCP_SYNC_PHY_UNLOCK 0x91 #define MCP_BL_SET_PWM_FRAC 0x6A /* Enable or disable Fractional PWM */ +#define MCP_SEND_EDID_CEA 0xA0 +#define EDID_CEA_CMD_ACK 1 +#define EDID_CEA_CMD_NACK 2 #define MASTER_COMM_CNTL_REG__MASTER_COMM_INTERRUPT_MASK 0x00000001L // PSP FW version @@ -65,13 +68,17 @@ //Register access policy version #define mmMP0_SMN_C2PMSG_91 0x1609B +#if defined(CONFIG_DRM_AMD_DC_DCN) +static const uint32_t abm_gain_stepsize = 0x0060; +#endif + static bool dce_dmcu_init(struct dmcu *dmcu) { // Do nothing return true; } -bool dce_dmcu_load_iram(struct dmcu *dmcu, +static bool dce_dmcu_load_iram(struct dmcu *dmcu, unsigned int start_offset, const char *src, unsigned int bytes) @@ -807,6 +814,120 @@ static bool dcn20_unlock_phy(struct dmcu *dmcu) return true; } +static bool dcn10_send_edid_cea(struct dmcu *dmcu, + int offset, + int total_length, + uint8_t *data, + int length) +{ + struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu); + uint32_t header, data1, data2; + + /* If microcontroller is not running, do nothing */ + if (dmcu->dmcu_state != DMCU_RUNNING) + return false; + + if (length > 8 || length <= 0) + return false; + + header = ((uint32_t)offset & 0xFFFF) << 16 | (total_length & 0xFFFF); + data1 = (((uint32_t)data[0]) << 24) | (((uint32_t)data[1]) << 16) | + (((uint32_t)data[2]) << 8) | ((uint32_t)data[3]); + data2 = (((uint32_t)data[4]) << 24) | (((uint32_t)data[5]) << 16) | + (((uint32_t)data[6]) << 8) | ((uint32_t)data[7]); + + /* waitDMCUReadyForCmd */ + REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000); + + /* setDMCUParam_Cmd */ + REG_UPDATE(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0, MCP_SEND_EDID_CEA); + + REG_WRITE(MASTER_COMM_DATA_REG1, header); + REG_WRITE(MASTER_COMM_DATA_REG2, data1); + REG_WRITE(MASTER_COMM_DATA_REG3, data2); + + /* notifyDMCUMsg */ + REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1); + + /* waitDMCUReadyForCmd */ + REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000); + + return true; +} + +static bool dcn10_get_scp_results(struct dmcu *dmcu, + uint32_t *cmd, + uint32_t *data1, + uint32_t *data2, + uint32_t *data3) +{ + struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu); + + /* If microcontroller is not running, do nothing */ + if (dmcu->dmcu_state != DMCU_RUNNING) + return false; + + *cmd = REG_READ(SLAVE_COMM_CMD_REG); + *data1 = REG_READ(SLAVE_COMM_DATA_REG1); + *data2 = REG_READ(SLAVE_COMM_DATA_REG2); + *data3 = REG_READ(SLAVE_COMM_DATA_REG3); + + /* clear SCP interrupt */ + REG_UPDATE(SLAVE_COMM_CNTL_REG, SLAVE_COMM_INTERRUPT, 0); + + return true; +} + +static bool dcn10_recv_amd_vsdb(struct dmcu *dmcu, + int *version, + int *min_frame_rate, + int *max_frame_rate) +{ + uint32_t data[4]; + int cmd, ack, len; + + if (!dcn10_get_scp_results(dmcu, &data[0], &data[1], &data[2], &data[3])) + return false; + + cmd = data[0] & 0x3FF; + len = (data[0] >> 10) & 0x3F; + ack = data[1]; + + if (cmd != MCP_SEND_EDID_CEA || ack != EDID_CEA_CMD_ACK || len != 12) + return false; + + if ((data[2] & 0xFF)) { + *version = (data[2] >> 8) & 0xFF; + *min_frame_rate = (data[3] >> 16) & 0xFFFF; + *max_frame_rate = data[3] & 0xFFFF; + return true; + } + + return false; +} + +static bool dcn10_recv_edid_cea_ack(struct dmcu *dmcu, int *offset) +{ + uint32_t data[4]; + int cmd, ack; + + if (!dcn10_get_scp_results(dmcu, + &data[0], &data[1], &data[2], &data[3])) + return false; + + cmd = data[0] & 0x3FF; + ack = data[1]; + + if (cmd != MCP_SEND_EDID_CEA) + return false; + + if (ack == EDID_CEA_CMD_ACK) + return true; + + *offset = data[2]; /* nack */ + return false; +} + #endif //(CONFIG_DRM_AMD_DC_DCN) static const struct dmcu_funcs dce_funcs = { @@ -829,6 +950,9 @@ static const struct dmcu_funcs dcn10_funcs = { .get_psr_state = dcn10_get_dmcu_psr_state, .set_psr_wait_loop = dcn10_psr_wait_loop, .get_psr_wait_loop = dcn10_get_psr_wait_loop, + .send_edid_cea = dcn10_send_edid_cea, + .recv_amd_vsdb = dcn10_recv_amd_vsdb, + .recv_edid_cea_ack = dcn10_recv_edid_cea_ack, .is_dmcu_initialized = dcn10_is_dmcu_initialized }; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h index 93e7f34d4775..ff726b35ef6a 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h @@ -40,6 +40,10 @@ SR(MASTER_COMM_DATA_REG3), \ SR(MASTER_COMM_CMD_REG), \ SR(MASTER_COMM_CNTL_REG), \ + SR(SLAVE_COMM_DATA_REG1), \ + SR(SLAVE_COMM_DATA_REG2), \ + SR(SLAVE_COMM_DATA_REG3), \ + SR(SLAVE_COMM_CMD_REG), \ SR(DMCU_IRAM_RD_CTRL), \ SR(DMCU_IRAM_RD_DATA), \ SR(DMCU_INTERRUPT_TO_UC_EN_MASK), \ @@ -112,6 +116,7 @@ DMCU_SF(MASTER_COMM_CMD_REG, \ MASTER_COMM_CMD_REG_BYTE0, mask_sh), \ DMCU_SF(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, mask_sh), \ + DMCU_SF(SLAVE_COMM_CNTL_REG, SLAVE_COMM_INTERRUPT, mask_sh), \ DMCU_SF(DMCU_INTERRUPT_TO_UC_EN_MASK, \ STATIC_SCREEN1_INT_TO_UC_EN, mask_sh), \ DMCU_SF(DMCU_INTERRUPT_TO_UC_EN_MASK, \ @@ -179,6 +184,7 @@ type UC_IN_RESET; \ type MASTER_COMM_CMD_REG_BYTE0; \ type MASTER_COMM_INTERRUPT; \ + type SLAVE_COMM_INTERRUPT; \ type DPHY_RX_FAST_TRAINING_CAPABLE; \ type DPHY_LOAD_BS_COUNT; \ type STATIC_SCREEN1_INT_TO_UC_EN; \ @@ -211,6 +217,11 @@ struct dce_dmcu_registers { uint32_t MASTER_COMM_DATA_REG3; uint32_t MASTER_COMM_CMD_REG; uint32_t MASTER_COMM_CNTL_REG; + uint32_t SLAVE_COMM_DATA_REG1; + uint32_t SLAVE_COMM_DATA_REG2; + uint32_t SLAVE_COMM_DATA_REG3; + uint32_t SLAVE_COMM_CMD_REG; + uint32_t SLAVE_COMM_CNTL_REG; uint32_t DMCU_IRAM_RD_CTRL; uint32_t DMCU_IRAM_RD_DATA; uint32_t DMCU_INTERRUPT_TO_UC_EN_MASK; @@ -317,6 +328,4 @@ struct dmcu *dcn21_dmcu_create( void dce_dmcu_destroy(struct dmcu **dmcu); -static const uint32_t abm_gain_stepsize = 0x0060; - #endif /* _DCE_ABM_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c index 7fbd92fbc63a..a524f471e0d7 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c @@ -435,7 +435,7 @@ struct dce_i2c_hw *acquire_i2c_hw_engine( return dce_i2c_hw; } -enum i2c_channel_operation_result dce_i2c_hw_engine_wait_on_operation_result( +static enum i2c_channel_operation_result dce_i2c_hw_engine_wait_on_operation_result( struct dce_i2c_hw *dce_i2c_hw, uint32_t timeout, enum i2c_channel_operation_result expected_result) @@ -502,7 +502,7 @@ static uint32_t get_transaction_timeout_hw( return period_timeout * num_of_clock_stretches; } -bool dce_i2c_hw_engine_submit_payload( +static bool dce_i2c_hw_engine_submit_payload( struct dce_i2c_hw *dce_i2c_hw, struct i2c_payload *payload, bool middle_of_transaction, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c index 87d8428df6c4..6846afd83701 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c @@ -339,7 +339,7 @@ static bool start_sync_sw( return false; } -void dce_i2c_sw_engine_set_speed( +static void dce_i2c_sw_engine_set_speed( struct dce_i2c_sw *engine, uint32_t speed) { @@ -353,7 +353,7 @@ void dce_i2c_sw_engine_set_speed( engine->clock_delay = 12; } -bool dce_i2c_sw_engine_acquire_engine( +static bool dce_i2c_sw_engine_acquire_engine( struct dce_i2c_sw *engine, struct ddc *ddc) { @@ -397,7 +397,7 @@ bool dce_i2c_engine_acquire_sw( -void dce_i2c_sw_engine_submit_channel_request( +static void dce_i2c_sw_engine_submit_channel_request( struct dce_i2c_sw *engine, struct i2c_request_transaction_data *req) { @@ -440,7 +440,8 @@ void dce_i2c_sw_engine_submit_channel_request( I2C_CHANNEL_OPERATION_SUCCEEDED : I2C_CHANNEL_OPERATION_FAILED; } -bool dce_i2c_sw_engine_submit_payload( + +static bool dce_i2c_sw_engine_submit_payload( struct dce_i2c_sw *engine, struct i2c_payload *payload, bool middle_of_transaction) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c index 210466b2d863..1e77ffee71b3 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c @@ -1197,7 +1197,7 @@ void dce110_link_encoder_enable_dp_mst_output( #if defined(CONFIG_DRM_AMD_DC_SI) /* enables DP PHY output */ -void dce60_link_encoder_enable_dp_output( +static void dce60_link_encoder_enable_dp_output( struct link_encoder *enc, const struct dc_link_settings *link_settings, enum clock_source_id clock_source) @@ -1236,7 +1236,7 @@ void dce60_link_encoder_enable_dp_output( } /* enables DP PHY output in MST mode */ -void dce60_link_encoder_enable_dp_mst_output( +static void dce60_link_encoder_enable_dp_mst_output( struct link_encoder *enc, const struct dc_link_settings *link_settings, enum clock_source_id clock_source) @@ -1426,7 +1426,7 @@ void dce110_link_encoder_dp_set_phy_pattern( #if defined(CONFIG_DRM_AMD_DC_SI) /* set DP PHY test and training patterns */ -void dce60_link_encoder_dp_set_phy_pattern( +static void dce60_link_encoder_dp_set_phy_pattern( struct link_encoder *enc, const struct encoder_set_dp_phy_pattern_param *param) { @@ -1503,7 +1503,6 @@ void dce110_link_encoder_update_mst_stream_allocation_table( const struct link_mst_stream_allocation_table *table) { struct dce110_link_encoder *enc110 = TO_DCE110_LINK_ENC(enc); - uint32_t value0 = 0; uint32_t value1 = 0; uint32_t value2 = 0; uint32_t slots = 0; @@ -1604,7 +1603,7 @@ void dce110_link_encoder_update_mst_stream_allocation_table( do { udelay(10); - value0 = REG_READ(DP_MSE_SAT_UPDATE); + REG_READ(DP_MSE_SAT_UPDATE); REG_GET(DP_MSE_SAT_UPDATE, DP_MSE_SAT_UPDATE, &value1); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c index e459ae65aaf7..4600231da6cb 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c @@ -97,7 +97,7 @@ enum { -/** +/* * set_truncation * 1) set truncation depth: 0 for 18 bpp or 1 for 24 bpp * 2) enable truncation @@ -142,7 +142,7 @@ static void set_truncation( } #if defined(CONFIG_DRM_AMD_DC_SI) -/** +/* * dce60_set_truncation * 1) set truncation depth: 0 for 18 bpp or 1 for 24 bpp * 2) enable truncation @@ -183,7 +183,7 @@ static void dce60_set_truncation( } #endif -/** +/* * set_spatial_dither * 1) set spatial dithering mode: pattern of seed * 2) set spatial dithering depth: 0 for 18bpp or 1 for 24bpp @@ -291,7 +291,7 @@ static void set_spatial_dither( FMT_SPATIAL_DITHER_EN, 1); } -/** +/* * SetTemporalDither (Frame Modulation) * 1) set temporal dither depth * 2) select pattern: from hard-coded pattern or programmable pattern @@ -355,7 +355,7 @@ static void set_temporal_dither( FMT_TEMPORAL_DITHER_EN, 1); } -/** +/* * Set Clamping * 1) Set clamping format based on bpc - 0 for 6bpc (No clamping) * 1 for 8 bpc @@ -415,7 +415,7 @@ void dce110_opp_set_clamping( } #if defined(CONFIG_DRM_AMD_DC_SI) -/** +/* * Set Clamping for DCE6 parts * 1) Set clamping format based on bpc - 0 for 6bpc (No clamping) * 1 for 8 bpc @@ -424,7 +424,7 @@ void dce110_opp_set_clamping( * 7 for programable * 2) Enable clamp if Limited range requested */ -void dce60_opp_set_clamping( +static void dce60_opp_set_clamping( struct dce110_opp *opp110, const struct clamping_and_pixel_encoding_params *params) { @@ -465,7 +465,7 @@ void dce60_opp_set_clamping( } #endif -/** +/* * set_pixel_encoding * * Set Pixel Encoding @@ -501,7 +501,7 @@ static void set_pixel_encoding( } #if defined(CONFIG_DRM_AMD_DC_SI) -/** +/* * dce60_set_pixel_encoding * DCE6 has no FMT_SUBSAMPLING_{MODE,ORDER} bits in FMT_CONTROL reg * Set Pixel Encoding @@ -545,7 +545,7 @@ void dce110_opp_program_bit_depth_reduction( } #if defined(CONFIG_DRM_AMD_DC_SI) -void dce60_opp_program_bit_depth_reduction( +static void dce60_opp_program_bit_depth_reduction( struct output_pixel_processor *opp, const struct bit_depth_reduction_params *params) { @@ -568,7 +568,7 @@ void dce110_opp_program_clamping_and_pixel_encoding( } #if defined(CONFIG_DRM_AMD_DC_SI) -void dce60_opp_program_clamping_and_pixel_encoding( +static void dce60_opp_program_clamping_and_pixel_encoding( struct output_pixel_processor *opp, const struct clamping_and_pixel_encoding_params *params) { @@ -678,7 +678,7 @@ void dce110_opp_program_fmt( } #if defined(CONFIG_DRM_AMD_DC_SI) -void dce60_opp_program_fmt( +static void dce60_opp_program_fmt( struct output_pixel_processor *opp, struct bit_depth_reduction_params *fmt_bit_depth, struct clamping_and_pixel_encoding_params *clamping) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h index 4d484ef60f35..bf1ffc3629c7 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h @@ -111,7 +111,6 @@ enum dce110_opp_reg_type { OPP_SF(FMT_DITHER_RAND_R_SEED, FMT_RAND_R_SEED, mask_sh),\ OPP_SF(FMT_DITHER_RAND_G_SEED, FMT_RAND_G_SEED, mask_sh),\ OPP_SF(FMT_DITHER_RAND_B_SEED, FMT_RAND_B_SEED, mask_sh),\ - OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_EN, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_RESET, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_OFFSET, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_DEPTH, mask_sh),\ @@ -219,7 +218,6 @@ enum dce110_opp_reg_type { OPP_SF(FMT_DITHER_RAND_R_SEED, FMT_RAND_R_SEED, mask_sh),\ OPP_SF(FMT_DITHER_RAND_G_SEED, FMT_RAND_G_SEED, mask_sh),\ OPP_SF(FMT_DITHER_RAND_B_SEED, FMT_RAND_B_SEED, mask_sh),\ - OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_EN, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_RESET, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_OFFSET, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_DEPTH, mask_sh),\ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c b/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c index 761fdfc1f5bd..e92339235863 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c @@ -50,16 +50,16 @@ static unsigned int dce_get_16_bit_backlight_from_pwm(struct panel_cntl *panel_c { uint64_t current_backlight; uint32_t round_result; - uint32_t pwm_period_cntl, bl_period, bl_int_count; - uint32_t bl_pwm_cntl, bl_pwm, fractional_duty_cycle_en; + uint32_t bl_period, bl_int_count; + uint32_t bl_pwm, fractional_duty_cycle_en; uint32_t bl_period_mask, bl_pwm_mask; struct dce_panel_cntl *dce_panel_cntl = TO_DCE_PANEL_CNTL(panel_cntl); - pwm_period_cntl = REG_READ(BL_PWM_PERIOD_CNTL); + REG_READ(BL_PWM_PERIOD_CNTL); REG_GET(BL_PWM_PERIOD_CNTL, BL_PWM_PERIOD, &bl_period); REG_GET(BL_PWM_PERIOD_CNTL, BL_PWM_PERIOD_BITCNT, &bl_int_count); - bl_pwm_cntl = REG_READ(BL_PWM_CNTL); + REG_READ(BL_PWM_CNTL); REG_GET(BL_PWM_CNTL, BL_ACTIVE_INT_FRAC_CNT, (uint32_t *)(&bl_pwm)); REG_GET(BL_PWM_CNTL, BL_PWM_FRACTIONAL_EN, &fractional_duty_cycle_en); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c index ada57f745fd7..8d4263da59f2 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c @@ -67,7 +67,6 @@ static void dce110_update_generic_info_packet( uint32_t packet_index, const struct dc_info_packet *info_packet) { - uint32_t regval; /* TODOFPGA Figure out a proper number for max_retries polling for lock * use 50 for now. */ @@ -99,7 +98,7 @@ static void dce110_update_generic_info_packet( } /* choose which generic packet to use */ { - regval = REG_READ(AFMT_VBI_PACKET_CONTROL); + REG_READ(AFMT_VBI_PACKET_CONTROL); REG_UPDATE(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_INDEX, packet_index); } @@ -564,6 +563,7 @@ static void dce110_stream_encoder_hdmi_set_stream_attribute( cntl.enable_dp_audio = enable_audio; cntl.pixel_clock = actual_pix_clk_khz; cntl.lanes_number = LANE_COUNT_FOUR; + cntl.color_depth = crtc_timing->display_color_depth; if (enc110->base.bp->funcs->encoder_control( enc110->base.bp, &cntl) != BP_RESULT_OK) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c index 130a0a0c8332..151dc7bf6d23 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c @@ -493,7 +493,6 @@ static void dce60_transform_set_scaler( { struct dce_transform *xfm_dce = TO_DCE_TRANSFORM(xfm); bool is_scaling_required; - bool filter_updated = false; const uint16_t *coeffs_v, *coeffs_h; /*Use whole line buffer memory always*/ @@ -558,7 +557,6 @@ static void dce60_transform_set_scaler( xfm_dce->filter_v = coeffs_v; xfm_dce->filter_h = coeffs_h; - filter_updated = true; } } @@ -601,12 +599,12 @@ static void set_clamp( clamp_max = 0x3FC0; break; case COLOR_DEPTH_101010: - /* 10bit MSB aligned on 14 bit bus '11 1111 1111 1100' */ - clamp_max = 0x3FFC; + /* 10bit MSB aligned on 14 bit bus '11 1111 1111 0000' */ + clamp_max = 0x3FF0; break; case COLOR_DEPTH_121212: - /* 12bit MSB aligned on 14 bit bus '11 1111 1111 1111' */ - clamp_max = 0x3FFF; + /* 12bit MSB aligned on 14 bit bus '11 1111 1111 1100' */ + clamp_max = 0x3FFC; break; default: clamp_max = 0x3FC0; @@ -1037,34 +1035,23 @@ static void dce60_transform_set_pixel_storage_depth( const struct bit_depth_reduction_params *bit_depth_params) { struct dce_transform *xfm_dce = TO_DCE_TRANSFORM(xfm); - int pixel_depth, expan_mode; enum dc_color_depth color_depth; switch (depth) { case LB_PIXEL_DEPTH_18BPP: color_depth = COLOR_DEPTH_666; - pixel_depth = 2; - expan_mode = 1; break; case LB_PIXEL_DEPTH_24BPP: color_depth = COLOR_DEPTH_888; - pixel_depth = 1; - expan_mode = 1; break; case LB_PIXEL_DEPTH_30BPP: color_depth = COLOR_DEPTH_101010; - pixel_depth = 0; - expan_mode = 1; break; case LB_PIXEL_DEPTH_36BPP: color_depth = COLOR_DEPTH_121212; - pixel_depth = 3; - expan_mode = 0; break; default: color_depth = COLOR_DEPTH_101010; - pixel_depth = 0; - expan_mode = 1; BREAK_TO_DEBUGGER(); break; } @@ -1113,7 +1100,7 @@ static void program_gamut_remap( } -/** +/* ***************************************************************************** * Function: dal_transform_wide_gamut_set_gamut_remap * diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c index 0cf130dc4e52..453aaa5757bd 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c @@ -57,6 +57,7 @@ static void dmub_abm_enable_fractional_pwm(struct dc_context *dc) union dmub_rb_cmd cmd; uint32_t fractional_pwm = (dc->dc->config.disable_fractional_pwm == false) ? 1 : 0; + memset(&cmd, 0, sizeof(cmd)); cmd.abm_set_pwm_frac.header.type = DMUB_CMD__ABM; cmd.abm_set_pwm_frac.header.sub_type = DMUB_CMD__ABM_SET_PWM_FRAC; cmd.abm_set_pwm_frac.abm_set_pwm_frac_data.fractional_pwm = fractional_pwm; @@ -135,6 +136,7 @@ static bool dmub_abm_set_level(struct abm *abm, uint32_t level) union dmub_rb_cmd cmd; struct dc_context *dc = abm->ctx; + memset(&cmd, 0, sizeof(cmd)); cmd.abm_set_level.header.type = DMUB_CMD__ABM; cmd.abm_set_level.header.sub_type = DMUB_CMD__ABM_SET_LEVEL; cmd.abm_set_level.abm_set_level_data.level = level; @@ -160,6 +162,7 @@ static bool dmub_abm_init_config(struct abm *abm, // Copy iramtable into cw7 memcpy(dc->dmub_srv->dmub->scratch_mem_fb.cpu_addr, (void *)src, bytes); + memset(&cmd, 0, sizeof(cmd)); // Fw will copy from cw7 to fw_state cmd.abm_init_config.header.type = DMUB_CMD__ABM; cmd.abm_init_config.header.sub_type = DMUB_CMD__ABM_INIT_CONFIG; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c index d399270fd17e..c97ee5abc0ef 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c @@ -33,8 +33,9 @@ void dmub_hw_lock_mgr_cmd(struct dc_dmub_srv *dmub_srv, union dmub_hw_lock_flags *hw_locks, struct dmub_hw_lock_inst_flags *inst_flags) { - union dmub_rb_cmd cmd = { 0 }; + union dmub_rb_cmd cmd; + memset(&cmd, 0, sizeof(cmd)); cmd.lock_hw.header.type = DMUB_CMD__HW_LOCK; cmd.lock_hw.header.sub_type = 0; cmd.lock_hw.header.payload_bytes = sizeof(struct dmub_cmd_lock_hw_data); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c index 17e84f34ceba..69e34bef274c 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c @@ -31,7 +31,7 @@ #define MAX_PIPES 6 -/** +/* * Convert dmcub psr state to dmcu psr state. */ static enum dc_psr_state convert_psr_state(uint32_t raw_state) @@ -74,7 +74,7 @@ static enum dc_psr_state convert_psr_state(uint32_t raw_state) return state; } -/** +/* * Get PSR state from firmware. */ static void dmub_psr_get_state(struct dmub_psr *dmub, enum dc_psr_state *state) @@ -90,7 +90,7 @@ static void dmub_psr_get_state(struct dmub_psr *dmub, enum dc_psr_state *state) *state = convert_psr_state(raw_state); } -/** +/* * Set PSR version. */ static bool dmub_psr_set_version(struct dmub_psr *dmub, struct dc_stream_state *stream) @@ -101,6 +101,7 @@ static bool dmub_psr_set_version(struct dmub_psr *dmub, struct dc_stream_state * if (stream->link->psr_settings.psr_version == DC_PSR_VERSION_UNSUPPORTED) return false; + memset(&cmd, 0, sizeof(cmd)); cmd.psr_set_version.header.type = DMUB_CMD__PSR; cmd.psr_set_version.header.sub_type = DMUB_CMD__PSR_SET_VERSION; switch (stream->link->psr_settings.psr_version) { @@ -121,7 +122,7 @@ static bool dmub_psr_set_version(struct dmub_psr *dmub, struct dc_stream_state * return true; } -/** +/* * Enable/Disable PSR. */ static void dmub_psr_enable(struct dmub_psr *dmub, bool enable, bool wait) @@ -131,7 +132,7 @@ static void dmub_psr_enable(struct dmub_psr *dmub, bool enable, bool wait) uint32_t retry_count; enum dc_psr_state state = PSR_STATE0; - + memset(&cmd, 0, sizeof(cmd)); cmd.psr_enable.header.type = DMUB_CMD__PSR; if (enable) @@ -170,7 +171,7 @@ static void dmub_psr_enable(struct dmub_psr *dmub, bool enable, bool wait) } } -/** +/* * Set PSR level. */ static void dmub_psr_set_level(struct dmub_psr *dmub, uint16_t psr_level) @@ -184,6 +185,7 @@ static void dmub_psr_set_level(struct dmub_psr *dmub, uint16_t psr_level) if (state == PSR_STATE0) return; + memset(&cmd, 0, sizeof(cmd)); cmd.psr_set_level.header.type = DMUB_CMD__PSR; cmd.psr_set_level.header.sub_type = DMUB_CMD__PSR_SET_LEVEL; cmd.psr_set_level.header.payload_bytes = sizeof(struct dmub_cmd_psr_set_level_data); @@ -194,7 +196,7 @@ static void dmub_psr_set_level(struct dmub_psr *dmub, uint16_t psr_level) dc_dmub_srv_wait_idle(dc->dmub_srv); } -/** +/* * Setup PSR by programming phy registers and sending psr hw context values to firmware. */ static bool dmub_psr_copy_settings(struct dmub_psr *dmub, @@ -233,6 +235,7 @@ static bool dmub_psr_copy_settings(struct dmub_psr *dmub, link->link_enc->funcs->psr_program_secondary_packet(link->link_enc, psr_context->sdpTransmitLineNumDeadline); + memset(&cmd, 0, sizeof(cmd)); cmd.psr_copy_settings.header.type = DMUB_CMD__PSR; cmd.psr_copy_settings.header.sub_type = DMUB_CMD__PSR_COPY_SETTINGS; cmd.psr_copy_settings.header.payload_bytes = sizeof(struct dmub_cmd_psr_copy_settings_data); @@ -277,7 +280,7 @@ static bool dmub_psr_copy_settings(struct dmub_psr *dmub, return true; } -/** +/* * Send command to PSR to force static ENTER and ignore all state changes until exit */ static void dmub_psr_force_static(struct dmub_psr *dmub) @@ -285,6 +288,7 @@ static void dmub_psr_force_static(struct dmub_psr *dmub) union dmub_rb_cmd cmd; struct dc_context *dc = dmub->ctx; + memset(&cmd, 0, sizeof(cmd)); cmd.psr_force_static.header.type = DMUB_CMD__PSR; cmd.psr_force_static.header.sub_type = DMUB_CMD__PSR_FORCE_STATIC; cmd.psr_enable.header.payload_bytes = 0; @@ -294,7 +298,7 @@ static void dmub_psr_force_static(struct dmub_psr *dmub) dc_dmub_srv_wait_idle(dc->dmub_srv); } -/** +/* * Get PSR residency from firmware. */ static void dmub_psr_get_residency(struct dmub_psr *dmub, uint32_t *residency) @@ -316,7 +320,7 @@ static const struct dmub_psr_funcs psr_funcs = { .psr_get_residency = dmub_psr_get_residency, }; -/** +/* * Construct PSR object. */ static void dmub_psr_construct(struct dmub_psr *psr, struct dc_context *ctx) @@ -325,7 +329,7 @@ static void dmub_psr_construct(struct dmub_psr *psr, struct dc_context *ctx) psr->funcs = &psr_funcs; } -/** +/* * Allocate and initialize PSR object. */ struct dmub_psr *dmub_psr_create(struct dc_context *ctx) @@ -342,7 +346,7 @@ struct dmub_psr *dmub_psr_create(struct dc_context *ctx) return psr; } -/** +/* * Deallocate PSR object. */ void dmub_psr_destroy(struct dmub_psr **dmub) diff --git a/drivers/gpu/drm/amd/display/dc/dce100/Makefile b/drivers/gpu/drm/amd/display/dc/dce100/Makefile index a822d4e2a169..ff20c47f559e 100644 --- a/drivers/gpu/drm/amd/display/dc/dce100/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce100/Makefile @@ -23,6 +23,8 @@ # Makefile for the 'controller' sub-component of DAL. # It provides the control and status of HW CRTC block. +CFLAGS_$(AMDDALPATH)/dc/dce100/dce100_resource.o = $(call cc-disable-warning, override-init) + DCE100 = dce100_resource.o dce100_hw_sequencer.o AMD_DAL_DCE100 = $(addprefix $(AMDDALPATH)/dc/dce100/,$(DCE100)) diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c index 8ab9d6c79808..635ef0e7c782 100644 --- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c @@ -58,6 +58,8 @@ #include "dce/dce_abm.h" #include "dce/dce_i2c.h" +#include "dce100_resource.h" + #ifndef mmMC_HUB_RDREQ_DMIF_LIMIT #include "gmc/gmc_8_2_d.h" #include "gmc/gmc_8_2_sh_mask.h" @@ -385,7 +387,7 @@ static const struct dc_plane_cap plane_cap = { .pixel_format_support = { .argb8888 = true, .nv12 = false, - .fp16 = false + .fp16 = true }, .max_upscale_factor = { @@ -611,7 +613,7 @@ static const struct encoder_feature_support link_enc_feature = { .flags.bits.IS_TPS3_CAPABLE = true }; -struct link_encoder *dce100_link_encoder_create( +static struct link_encoder *dce100_link_encoder_create( const struct encoder_init_data *enc_init_data) { struct dce110_link_encoder *enc110 = @@ -650,7 +652,7 @@ static struct panel_cntl *dce100_panel_cntl_create(const struct panel_cntl_init_ return &panel_cntl->base; } -struct output_pixel_processor *dce100_opp_create( +static struct output_pixel_processor *dce100_opp_create( struct dc_context *ctx, uint32_t inst) { @@ -665,7 +667,7 @@ struct output_pixel_processor *dce100_opp_create( return &opp->base; } -struct dce_aux *dce100_aux_engine_create( +static struct dce_aux *dce100_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -703,7 +705,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK) }; -struct dce_i2c_hw *dce100_i2c_hw_create( +static struct dce_i2c_hw *dce100_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { @@ -718,7 +720,7 @@ struct dce_i2c_hw *dce100_i2c_hw_create( return dce_i2c_hw; } -struct clock_source *dce100_clock_source_create( +static struct clock_source *dce100_clock_source_create( struct dc_context *ctx, struct dc_bios *bios, enum clock_source_id id, @@ -742,7 +744,7 @@ struct clock_source *dce100_clock_source_create( return NULL; } -void dce100_clock_source_destroy(struct clock_source **clk_src) +static void dce100_clock_source_destroy(struct clock_source **clk_src) { kfree(TO_DCE110_CLK_SRC(*clk_src)); *clk_src = NULL; @@ -831,7 +833,7 @@ static enum dc_status build_mapped_resource( return DC_OK; } -bool dce100_validate_bandwidth( +static bool dce100_validate_bandwidth( struct dc *dc, struct dc_state *context, bool fast_validate) @@ -876,7 +878,7 @@ static bool dce100_validate_surface_sets( return true; } -enum dc_status dce100_validate_global( +static enum dc_status dce100_validate_global( struct dc *dc, struct dc_state *context) { diff --git a/drivers/gpu/drm/amd/display/dc/dce110/Makefile b/drivers/gpu/drm/amd/display/dc/dce110/Makefile index d564c0eb8b04..84ab48df0c26 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce110/Makefile @@ -23,6 +23,8 @@ # Makefile for the 'controller' sub-component of DAL. # It provides the control and status of HW CRTC block. +CFLAGS_$(AMDDALPATH)/dc/dce110/dce110_resource.o = $(call cc-disable-warning, override-init) + DCE110 = dce110_timing_generator.o \ dce110_compressor.o dce110_hw_sequencer.o dce110_resource.o \ dce110_opp_regamma_v.o dce110_opp_csc_v.o dce110_timing_generator_v.o \ diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c index 72b580a4eb85..44564a4742b5 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c @@ -412,36 +412,6 @@ void dce110_compressor_destroy(struct compressor **compressor) *compressor = NULL; } -bool dce110_get_required_compressed_surfacesize(struct fbc_input_info fbc_input_info, - struct fbc_requested_compressed_size size) -{ - bool result = false; - - unsigned int max_x = FBC_MAX_X, max_y = FBC_MAX_Y; - - get_max_support_fbc_buffersize(&max_x, &max_y); - - if (fbc_input_info.dynamic_fbc_buffer_alloc == 0) { - /* - * For DCE11 here use Max HW supported size: HW Support up to 3840x2400 resolution - * or 18000 chunks. - */ - size.preferred_size = size.min_size = align_to_chunks_number_per_line(max_x) * max_y * 4; /* (For FBC when LPT not supported). */ - size.preferred_size_alignment = size.min_size_alignment = 0x100; /* For FBC when LPT not supported */ - size.bits.preferred_must_be_framebuffer_pool = 1; - size.bits.min_must_be_framebuffer_pool = 1; - - result = true; - } - /* - * Maybe to add registry key support with optional size here to override above - * for debugging purposes - */ - - return result; -} - - void get_max_support_fbc_buffersize(unsigned int *max_x, unsigned int *max_y) { *max_x = FBC_MAX_X; @@ -455,31 +425,6 @@ void get_max_support_fbc_buffersize(unsigned int *max_x, unsigned int *max_y) */ } - -unsigned int controller_id_to_index(enum controller_id controller_id) -{ - unsigned int index = 0; - - switch (controller_id) { - case CONTROLLER_ID_D0: - index = 0; - break; - case CONTROLLER_ID_D1: - index = 1; - break; - case CONTROLLER_ID_D2: - index = 2; - break; - case CONTROLLER_ID_D3: - index = 3; - break; - default: - break; - } - return index; -} - - static const struct compressor_funcs dce110_compressor_funcs = { .power_up_fbc = dce110_compressor_power_up_fbc, .enable_fbc = dce110_compressor_enable_fbc, diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c index 4c230f1de9a3..caee1c9f54bd 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c @@ -921,6 +921,37 @@ void dce110_edp_power_control( } } +void dce110_edp_wait_for_T12( + struct dc_link *link) +{ + struct dc_context *ctx = link->ctx; + + if (dal_graphics_object_id_get_connector_id(link->link_enc->connector) + != CONNECTOR_ID_EDP) { + BREAK_TO_DEBUGGER(); + return; + } + + if (!link->panel_cntl) + return; + + if (!link->panel_cntl->funcs->is_panel_powered_on(link->panel_cntl) && + link->link_trace.time_stamp.edp_poweroff != 0) { + unsigned int t12_duration = 500; // Default T12 as per spec + unsigned long long current_ts = dm_get_timestamp(ctx); + unsigned long long time_since_edp_poweroff_ms = + div64_u64(dm_get_elapse_time_in_ns( + ctx, + current_ts, + link->link_trace.time_stamp.edp_poweroff), 1000000); + + t12_duration += link->local_sink->edid_caps.panel_patch.extra_t12_ms; // Add extra T12 + + if (time_since_edp_poweroff_ms < t12_duration) + msleep(t12_duration - time_since_edp_poweroff_ms); + } +} + /*todo: cloned in stream enc, fix*/ /* * @brief @@ -1628,7 +1659,7 @@ static struct dc_link *get_edp_link_with_sink( return link; } -/** +/* * When ASIC goes from VBIOS/VGA mode to driver/accelerated mode we need: * 1. Power down all DC HW blocks * 2. Disable VGA engine on all controllers diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c index d54172d88f5f..8bbb499067f7 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c @@ -34,6 +34,7 @@ #include "inc/dce_calcs.h" #include "dce/dce_mem_input.h" +#include "dce110_mem_input_v.h" static void set_flip_control( struct dce_mem_input *mem_input110, @@ -468,7 +469,7 @@ static void program_pixel_format( } } -bool dce_mem_input_v_is_surface_pending(struct mem_input *mem_input) +static bool dce_mem_input_v_is_surface_pending(struct mem_input *mem_input) { struct dce_mem_input *mem_input110 = TO_DCE_MEM_INPUT(mem_input); uint32_t value; @@ -483,7 +484,7 @@ bool dce_mem_input_v_is_surface_pending(struct mem_input *mem_input) return false; } -bool dce_mem_input_v_program_surface_flip_and_addr( +static bool dce_mem_input_v_program_surface_flip_and_addr( struct mem_input *mem_input, const struct dc_plane_address *address, bool flip_immediate) @@ -560,7 +561,7 @@ static const unsigned int *get_dvmm_hw_setting( } } -void dce_mem_input_v_program_pte_vm( +static void dce_mem_input_v_program_pte_vm( struct mem_input *mem_input, enum surface_pixel_format format, union dc_tiling_info *tiling_info, @@ -633,7 +634,7 @@ void dce_mem_input_v_program_pte_vm( dm_write_reg(mem_input110->base.ctx, mmUNP_DVMM_PTE_ARB_CONTROL_C, value); } -void dce_mem_input_v_program_surface_config( +static void dce_mem_input_v_program_surface_config( struct mem_input *mem_input, enum surface_pixel_format format, union dc_tiling_info *tiling_info, @@ -919,7 +920,7 @@ static void program_nbp_watermark_c( marks); } -void dce_mem_input_v_program_display_marks( +static void dce_mem_input_v_program_display_marks( struct mem_input *mem_input, struct dce_watermarks nbp, struct dce_watermarks stutter, @@ -942,7 +943,7 @@ void dce_mem_input_v_program_display_marks( } -void dce_mem_input_program_chroma_display_marks( +static void dce_mem_input_program_chroma_display_marks( struct mem_input *mem_input, struct dce_watermarks nbp, struct dce_watermarks stutter, @@ -963,7 +964,7 @@ void dce_mem_input_program_chroma_display_marks( stutter); } -void dce110_allocate_mem_input_v( +static void dce110_allocate_mem_input_v( struct mem_input *mi, uint32_t h_total,/* for current stream */ uint32_t v_total,/* for current stream */ @@ -1005,7 +1006,7 @@ void dce110_allocate_mem_input_v( } -void dce110_free_mem_input_v( +static void dce110_free_mem_input_v( struct mem_input *mi, uint32_t total_stream_num) { diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c index 3f63822b8e28..d7fcc5cccdce 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c @@ -410,7 +410,7 @@ static const struct dc_plane_cap plane_cap = { .pixel_format_support = { .argb8888 = true, .nv12 = false, - .fp16 = false + .fp16 = true }, .max_upscale_factor = { @@ -715,7 +715,7 @@ static struct output_pixel_processor *dce110_opp_create( return &opp->base; } -struct dce_aux *dce110_aux_engine_create( +static struct dce_aux *dce110_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -753,7 +753,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCE110(_MASK) }; -struct dce_i2c_hw *dce110_i2c_hw_create( +static struct dce_i2c_hw *dce110_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { @@ -768,7 +768,7 @@ struct dce_i2c_hw *dce110_i2c_hw_create( return dce_i2c_hw; } -struct clock_source *dce110_clock_source_create( +static struct clock_source *dce110_clock_source_create( struct dc_context *ctx, struct dc_bios *bios, enum clock_source_id id, @@ -792,7 +792,7 @@ struct clock_source *dce110_clock_source_create( return NULL; } -void dce110_clock_source_destroy(struct clock_source **clk_src) +static void dce110_clock_source_destroy(struct clock_source **clk_src) { struct dce110_clk_src *dce110_clk_src; @@ -1034,8 +1034,8 @@ static bool dce110_validate_bandwidth( return result; } -enum dc_status dce110_validate_plane(const struct dc_plane_state *plane_state, - struct dc_caps *caps) +static enum dc_status dce110_validate_plane(const struct dc_plane_state *plane_state, + struct dc_caps *caps) { if (((plane_state->dst_rect.width * 2) < plane_state->src_rect.width) || ((plane_state->dst_rect.height * 2) < plane_state->src_rect.height)) @@ -1089,7 +1089,7 @@ static bool dce110_validate_surface_sets( return true; } -enum dc_status dce110_validate_global( +static enum dc_status dce110_validate_global( struct dc *dc, struct dc_state *context) { @@ -1272,7 +1272,6 @@ static bool underlay_create(struct dc_context *ctx, struct resource_pool *pool) /* update the public caps to indicate an underlay is available */ ctx->dc->caps.max_slave_planes = 1; - ctx->dc->caps.max_slave_planes = 1; return true; } @@ -1333,7 +1332,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc) 1000); } -const struct resource_caps *dce110_resource_cap( +static const struct resource_caps *dce110_resource_cap( struct hw_asic_id *asic_id) { if (ASIC_REV_IS_STONEY(asic_id->hw_internal_rev)) diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c index 1ea7db8eeb98..d88a74559edd 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c @@ -75,7 +75,7 @@ static void dce110_timing_generator_apply_front_porch_workaround( } } -/** +/* ***************************************************************************** * Function: is_in_vertical_blank * @@ -116,7 +116,7 @@ void dce110_timing_generator_set_early_control( dm_write_reg(tg->ctx, address, regval); } -/** +/* * Enable CRTC * Enable CRTC - call ASIC Control Object to enable Timing generator. */ @@ -175,7 +175,7 @@ void dce110_timing_generator_program_blank_color( dm_write_reg(tg->ctx, addr, value); } -/** +/* ***************************************************************************** * Function: disable_stereo * @@ -226,7 +226,7 @@ static void disable_stereo(struct timing_generator *tg) } #endif -/** +/* * disable_crtc - call ASIC Control Object to disable Timing generator. */ bool dce110_timing_generator_disable_crtc(struct timing_generator *tg) @@ -247,11 +247,10 @@ bool dce110_timing_generator_disable_crtc(struct timing_generator *tg) return result == BP_RESULT_OK; } -/** -* program_horz_count_by_2 -* Programs DxCRTC_HORZ_COUNT_BY2_EN - 1 for DVI 30bpp mode, 0 otherwise -* -*/ +/* + * program_horz_count_by_2 + * Programs DxCRTC_HORZ_COUNT_BY2_EN - 1 for DVI 30bpp mode, 0 otherwise + */ static void program_horz_count_by_2( struct timing_generator *tg, const struct dc_crtc_timing *timing) @@ -273,7 +272,7 @@ static void program_horz_count_by_2( CRTC_REG(mmCRTC_COUNT_CONTROL), regval); } -/** +/* * program_timing_generator * Program CRTC Timing Registers - DxCRTC_H_*, DxCRTC_V_*, Pixel repetition. * Call ASIC Control Object to program Timings. @@ -352,7 +351,7 @@ bool dce110_timing_generator_program_timing_generator( return result == BP_RESULT_OK; } -/** +/* ***************************************************************************** * Function: set_drr * @@ -521,7 +520,7 @@ uint32_t dce110_timing_generator_get_vblank_counter(struct timing_generator *tg) return field; } -/** +/* ***************************************************************************** * Function: dce110_timing_generator_get_position * @@ -557,7 +556,7 @@ void dce110_timing_generator_get_position(struct timing_generator *tg, CRTC_VERT_COUNT_NOM); } -/** +/* ***************************************************************************** * Function: get_crtc_scanoutpos * @@ -1106,11 +1105,11 @@ void dce110_timing_generator_set_test_pattern( } } -/** -* dce110_timing_generator_validate_timing -* The timing generators support a maximum display size of is 8192 x 8192 pixels, -* including both active display and blanking periods. Check H Total and V Total. -*/ +/* + * dce110_timing_generator_validate_timing + * The timing generators support a maximum display size of is 8192 x 8192 pixels, + * including both active display and blanking periods. Check H Total and V Total. + */ bool dce110_timing_generator_validate_timing( struct timing_generator *tg, const struct dc_crtc_timing *timing, @@ -1167,9 +1166,9 @@ bool dce110_timing_generator_validate_timing( return true; } -/** -* Wait till we are at the beginning of VBlank. -*/ +/* + * Wait till we are at the beginning of VBlank. + */ void dce110_timing_generator_wait_for_vblank(struct timing_generator *tg) { /* We want to catch beginning of VBlank here, so if the first try are @@ -1191,9 +1190,9 @@ void dce110_timing_generator_wait_for_vblank(struct timing_generator *tg) } } -/** -* Wait till we are in VActive (anywhere in VActive) -*/ +/* + * Wait till we are in VActive (anywhere in VActive) + */ void dce110_timing_generator_wait_for_vactive(struct timing_generator *tg) { while (dce110_timing_generator_is_in_vertical_blank(tg)) { @@ -1204,7 +1203,7 @@ void dce110_timing_generator_wait_for_vactive(struct timing_generator *tg) } } -/** +/* ***************************************************************************** * Function: dce110_timing_generator_setup_global_swap_lock * @@ -1215,7 +1214,6 @@ void dce110_timing_generator_wait_for_vactive(struct timing_generator *tg) * @param [in] gsl_params: setup data ***************************************************************************** */ - void dce110_timing_generator_setup_global_swap_lock( struct timing_generator *tg, const struct dcp_gsl_params *gsl_params) @@ -1351,10 +1349,7 @@ void dce110_timing_generator_tear_down_global_swap_lock( /* Restore DCP_GSL_PURPOSE_SURFACE_FLIP */ { - uint32_t value_crtc_vtotal; - - value_crtc_vtotal = dm_read_reg(tg->ctx, - CRTC_REG(mmCRTC_V_TOTAL)); + dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_V_TOTAL)); set_reg_field_value(value, 0, @@ -1385,7 +1380,7 @@ void dce110_timing_generator_tear_down_global_swap_lock( dm_write_reg(tg->ctx, address, value); } -/** +/* ***************************************************************************** * Function: is_counter_moving * @@ -1767,7 +1762,7 @@ void dce110_timing_generator_disable_reset_trigger( dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_TRIGB_CNTL), value); } -/** +/* ***************************************************************************** * @brief * Checks whether CRTC triggered reset occurred @@ -1794,7 +1789,7 @@ bool dce110_timing_generator_did_triggered_reset_occur( return (force || vert_sync); } -/** +/* * dce110_timing_generator_disable_vga * Turn OFF VGA Mode and Timing - DxVGA_CONTROL * VGA Mode and VGA Timing is used by VBIOS on CRT Monitors; @@ -1840,14 +1835,13 @@ void dce110_timing_generator_disable_vga( dm_write_reg(tg->ctx, addr, value); } -/** -* set_overscan_color_black -* -* @param :black_color is one of the color space -* :this routine will set overscan black color according to the color space. -* @return none -*/ - +/* + * set_overscan_color_black + * + * @param :black_color is one of the color space + * :this routine will set overscan black color according to the color space. + * @return none + */ void dce110_timing_generator_set_overscan_color_black( struct timing_generator *tg, const struct tg_color *color) diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c index a13a2f58944e..c509384fff54 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c @@ -46,17 +46,16 @@ * **********************************************************************************/ -/** -* Enable CRTCV -*/ +/* + * Enable CRTCV + */ static bool dce110_timing_generator_v_enable_crtc(struct timing_generator *tg) { /* -* Set MASTER_UPDATE_MODE to 0 -* This is needed for DRR, and also suggested to be default value by Syed. -*/ - + * Set MASTER_UPDATE_MODE to 0 + * This is needed for DRR, and also suggested to be default value by Syed. + */ uint32_t value; value = 0; @@ -209,9 +208,9 @@ static void dce110_timing_generator_v_wait_for_vblank(struct timing_generator *t } } -/** -* Wait till we are in VActive (anywhere in VActive) -*/ +/* + * Wait till we are in VActive (anywhere in VActive) + */ static void dce110_timing_generator_v_wait_for_vactive(struct timing_generator *tg) { while (dce110_timing_generator_v_is_in_vertical_blank(tg)) { diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c index b1aaab5590cc..29438c6050db 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c @@ -217,16 +217,15 @@ static bool setup_scaling_configuration( return is_scaling_needed; } -/** -* Function: -* void program_overscan -* -* Purpose: Programs overscan border -* Input: overscan -* -* Output: - void -*/ +/* + * Function: + * void program_overscan + * + * Purpose: Programs overscan border + * Input: overscan + * + * Output: void + */ static void program_overscan( struct dce_transform *xfm_dce, const struct scaler_data *data) diff --git a/drivers/gpu/drm/amd/display/dc/dce112/Makefile b/drivers/gpu/drm/amd/display/dc/dce112/Makefile index 8e090446d511..9de6501702d2 100644 --- a/drivers/gpu/drm/amd/display/dc/dce112/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce112/Makefile @@ -23,6 +23,8 @@ # Makefile for the 'controller' sub-component of DAL. # It provides the control and status of HW CRTC block. +CFLAGS_$(AMDDALPATH)/dc/dce112/dce112_resource.o = $(call cc-disable-warning, override-init) + DCE112 = dce112_compressor.o dce112_hw_sequencer.o \ dce112_resource.o diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c index f99b1c084590..ee55cda854bf 100644 --- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c @@ -59,7 +59,9 @@ #include "dce/dce_11_2_sh_mask.h" #include "dce100/dce100_resource.h" -#define DC_LOGGER \ +#include "dce112_resource.h" + +#define DC_LOGGER \ dc->ctx->logger #ifndef mmDP_DPHY_INTERNAL_CTRL @@ -617,7 +619,7 @@ static const struct encoder_feature_support link_enc_feature = { .flags.bits.IS_TPS4_CAPABLE = true }; -struct link_encoder *dce112_link_encoder_create( +static struct link_encoder *dce112_link_encoder_create( const struct encoder_init_data *enc_init_data) { struct dce110_link_encoder *enc110 = @@ -671,7 +673,7 @@ static struct input_pixel_processor *dce112_ipp_create( return &ipp->base; } -struct output_pixel_processor *dce112_opp_create( +static struct output_pixel_processor *dce112_opp_create( struct dc_context *ctx, uint32_t inst) { @@ -686,7 +688,7 @@ struct output_pixel_processor *dce112_opp_create( return &opp->base; } -struct dce_aux *dce112_aux_engine_create( +static struct dce_aux *dce112_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -724,7 +726,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCE110(_MASK) }; -struct dce_i2c_hw *dce112_i2c_hw_create( +static struct dce_i2c_hw *dce112_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { @@ -739,7 +741,7 @@ struct dce_i2c_hw *dce112_i2c_hw_create( return dce_i2c_hw; } -struct clock_source *dce112_clock_source_create( +static struct clock_source *dce112_clock_source_create( struct dc_context *ctx, struct dc_bios *bios, enum clock_source_id id, @@ -763,7 +765,7 @@ struct clock_source *dce112_clock_source_create( return NULL; } -void dce112_clock_source_destroy(struct clock_source **clk_src) +static void dce112_clock_source_destroy(struct clock_source **clk_src) { kfree(TO_DCE110_CLK_SRC(*clk_src)); *clk_src = NULL; @@ -1024,7 +1026,7 @@ enum dc_status dce112_add_stream_to_ctx( return result; } -enum dc_status dce112_validate_global( +static enum dc_status dce112_validate_global( struct dc *dc, struct dc_state *context) { @@ -1202,7 +1204,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc) dm_pp_notify_wm_clock_changes(dc->ctx, &clk_ranges); } -const struct resource_caps *dce112_resource_cap( +static const struct resource_caps *dce112_resource_cap( struct hw_asic_id *asic_id) { if (ASIC_REV_IS_POLARIS11_M(asic_id->hw_internal_rev) || diff --git a/drivers/gpu/drm/amd/display/dc/dce120/Makefile b/drivers/gpu/drm/amd/display/dc/dce120/Makefile index 37db1f8d45ea..a9cc4b73270b 100644 --- a/drivers/gpu/drm/amd/display/dc/dce120/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce120/Makefile @@ -24,6 +24,8 @@ # It provides the control and status of HW CRTC block. +CFLAGS_$(AMDDALPATH)/dc/dce120/dce120_resource.o = $(call cc-disable-warning, override-init) + DCE120 = dce120_resource.o dce120_timing_generator.o \ dce120_hw_sequencer.o diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c index 66a13aa39c95..d4afe6c824d2 100644 --- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c @@ -50,6 +50,7 @@ struct dce120_hw_seq_reg_offsets { uint32_t crtc; }; +#if 0 static const struct dce120_hw_seq_reg_offsets reg_offsets[] = { { .crtc = (mmCRTC0_CRTC_GSL_CONTROL - mmCRTC0_CRTC_GSL_CONTROL), @@ -79,7 +80,6 @@ static const struct dce120_hw_seq_reg_offsets reg_offsets[] = { /******************************************************************************* * Private definitions ******************************************************************************/ -#if 0 static void dce120_init_pte(struct dc_context *ctx, uint8_t controller_id) { uint32_t addr; diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c index f1e3d2888eac..c65e4d125c8e 100644 --- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c @@ -423,7 +423,7 @@ static const struct dce110_clk_src_mask cs_mask = { CS_COMMON_MASK_SH_LIST_DCE_112(_MASK) }; -struct output_pixel_processor *dce120_opp_create( +static struct output_pixel_processor *dce120_opp_create( struct dc_context *ctx, uint32_t inst) { @@ -437,7 +437,7 @@ struct output_pixel_processor *dce120_opp_create( ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask); return &opp->base; } -struct dce_aux *dce120_aux_engine_create( +static struct dce_aux *dce120_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -475,7 +475,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCE110(_MASK) }; -struct dce_i2c_hw *dce120_i2c_hw_create( +static struct dce_i2c_hw *dce120_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c index 915fbb8e8168..b57c466124e7 100644 --- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c +++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c @@ -69,7 +69,7 @@ #define CRTC_REG_SET_3(reg, field1, val1, field2, val2, field3, val3) \ CRTC_REG_SET_N(reg, 3, FD(reg##__##field1), val1, FD(reg##__##field2), val2, FD(reg##__##field3), val3) -/** +/* ***************************************************************************** * Function: is_in_vertical_blank * @@ -98,7 +98,7 @@ static bool dce120_timing_generator_is_in_vertical_blank( /* determine if given timing can be supported by TG */ -bool dce120_timing_generator_validate_timing( +static bool dce120_timing_generator_validate_timing( struct timing_generator *tg, const struct dc_crtc_timing *timing, enum signal_type signal) @@ -125,7 +125,7 @@ bool dce120_timing_generator_validate_timing( return true; } -bool dce120_tg_validate_timing(struct timing_generator *tg, +static bool dce120_tg_validate_timing(struct timing_generator *tg, const struct dc_crtc_timing *timing) { return dce120_timing_generator_validate_timing(tg, timing, SIGNAL_TYPE_NONE); @@ -133,7 +133,7 @@ bool dce120_tg_validate_timing(struct timing_generator *tg, /******** HW programming ************/ /* Disable/Enable Timing Generator */ -bool dce120_timing_generator_enable_crtc(struct timing_generator *tg) +static bool dce120_timing_generator_enable_crtc(struct timing_generator *tg) { enum bp_result result; struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -153,7 +153,7 @@ bool dce120_timing_generator_enable_crtc(struct timing_generator *tg) return result == BP_RESULT_OK; } -void dce120_timing_generator_set_early_control( +static void dce120_timing_generator_set_early_control( struct timing_generator *tg, uint32_t early_cntl) { @@ -166,7 +166,7 @@ void dce120_timing_generator_set_early_control( /**************** TG current status ******************/ /* return the current frame counter. Used by Linux kernel DRM */ -uint32_t dce120_timing_generator_get_vblank_counter( +static uint32_t dce120_timing_generator_get_vblank_counter( struct timing_generator *tg) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -181,7 +181,7 @@ uint32_t dce120_timing_generator_get_vblank_counter( } /* Get current H and V position */ -void dce120_timing_generator_get_crtc_position( +static void dce120_timing_generator_get_crtc_position( struct timing_generator *tg, struct crtc_position *position) { @@ -207,7 +207,7 @@ void dce120_timing_generator_get_crtc_position( } /* wait until TG is in beginning of vertical blank region */ -void dce120_timing_generator_wait_for_vblank(struct timing_generator *tg) +static void dce120_timing_generator_wait_for_vblank(struct timing_generator *tg) { /* We want to catch beginning of VBlank here, so if the first try are * in VBlank, we might be very close to Active, in this case wait for @@ -229,7 +229,7 @@ void dce120_timing_generator_wait_for_vblank(struct timing_generator *tg) } /* wait until TG is in beginning of active region */ -void dce120_timing_generator_wait_for_vactive(struct timing_generator *tg) +static void dce120_timing_generator_wait_for_vactive(struct timing_generator *tg) { while (dce120_timing_generator_is_in_vertical_blank(tg)) { if (!tg->funcs->is_counter_moving(tg)) { @@ -242,7 +242,7 @@ void dce120_timing_generator_wait_for_vactive(struct timing_generator *tg) /*********** Timing Generator Synchronization routines ****/ /* Setups Global Swap Lock group, TimingServer or TimingClient*/ -void dce120_timing_generator_setup_global_swap_lock( +static void dce120_timing_generator_setup_global_swap_lock( struct timing_generator *tg, const struct dcp_gsl_params *gsl_params) { @@ -279,7 +279,7 @@ void dce120_timing_generator_setup_global_swap_lock( } /* Clear all the register writes done by setup_global_swap_lock */ -void dce120_timing_generator_tear_down_global_swap_lock( +static void dce120_timing_generator_tear_down_global_swap_lock( struct timing_generator *tg) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -300,7 +300,7 @@ void dce120_timing_generator_tear_down_global_swap_lock( } /* Reset slave controllers on master VSync */ -void dce120_timing_generator_enable_reset_trigger( +static void dce120_timing_generator_enable_reset_trigger( struct timing_generator *tg, int source) { @@ -347,7 +347,7 @@ void dce120_timing_generator_enable_reset_trigger( } /* disabling trigger-reset */ -void dce120_timing_generator_disable_reset_trigger( +static void dce120_timing_generator_disable_reset_trigger( struct timing_generator *tg) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -367,7 +367,7 @@ void dce120_timing_generator_disable_reset_trigger( } /* Checks whether CRTC triggered reset occurred */ -bool dce120_timing_generator_did_triggered_reset_occur( +static bool dce120_timing_generator_did_triggered_reset_occur( struct timing_generator *tg) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -384,7 +384,7 @@ bool dce120_timing_generator_did_triggered_reset_occur( /******** Stuff to move to other virtual HW objects *****************/ /* Move to enable accelerated mode */ -void dce120_timing_generator_disable_vga(struct timing_generator *tg) +static void dce120_timing_generator_disable_vga(struct timing_generator *tg) { uint32_t offset = 0; uint32_t value = 0; @@ -425,7 +425,7 @@ void dce120_timing_generator_disable_vga(struct timing_generator *tg) } /* TODO: Should we move it to transform */ /* Fully program CRTC timing in timing generator */ -void dce120_timing_generator_program_blanking( +static void dce120_timing_generator_program_blanking( struct timing_generator *tg, const struct dc_crtc_timing *timing) { @@ -485,7 +485,7 @@ void dce120_timing_generator_program_blanking( /* TODO: Should we move it to opp? */ /* Combine with below and move YUV/RGB color conversion to SW layer */ -void dce120_timing_generator_program_blank_color( +static void dce120_timing_generator_program_blank_color( struct timing_generator *tg, const struct tg_color *black_color) { @@ -498,7 +498,7 @@ void dce120_timing_generator_program_blank_color( CRTC_BLACK_COLOR_R_CR, black_color->color_r_cr); } /* Combine with above and move YUV/RGB color conversion to SW layer */ -void dce120_timing_generator_set_overscan_color_black( +static void dce120_timing_generator_set_overscan_color_black( struct timing_generator *tg, const struct tg_color *color) { @@ -540,7 +540,7 @@ void dce120_timing_generator_set_overscan_color_black( */ } -void dce120_timing_generator_set_drr( +static void dce120_timing_generator_set_drr( struct timing_generator *tg, const struct drr_params *params) { @@ -589,50 +589,7 @@ void dce120_timing_generator_set_drr( } } -/** - ***************************************************************************** - * Function: dce120_timing_generator_get_position - * - * @brief - * Returns CRTC vertical/horizontal counters - * - * @param [out] position - ***************************************************************************** - */ -void dce120_timing_generator_get_position(struct timing_generator *tg, - struct crtc_position *position) -{ - uint32_t value; - struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); - - value = dm_read_reg_soc15( - tg->ctx, - mmCRTC0_CRTC_STATUS_POSITION, - tg110->offsets.crtc); - - position->horizontal_count = get_reg_field_value( - value, - CRTC0_CRTC_STATUS_POSITION, - CRTC_HORZ_COUNT); - - position->vertical_count = get_reg_field_value( - value, - CRTC0_CRTC_STATUS_POSITION, - CRTC_VERT_COUNT); - - value = dm_read_reg_soc15( - tg->ctx, - mmCRTC0_CRTC_NOM_VERT_POSITION, - tg110->offsets.crtc); - - position->nominal_vcount = get_reg_field_value( - value, - CRTC0_CRTC_NOM_VERT_POSITION, - CRTC_VERT_COUNT_NOM); -} - - -void dce120_timing_generator_get_crtc_scanoutpos( +static void dce120_timing_generator_get_crtc_scanoutpos( struct timing_generator *tg, uint32_t *v_blank_start, uint32_t *v_blank_end, @@ -661,7 +618,7 @@ void dce120_timing_generator_get_crtc_scanoutpos( *v_position = position.vertical_count; } -void dce120_timing_generator_enable_advanced_request( +static void dce120_timing_generator_enable_advanced_request( struct timing_generator *tg, bool enable, const struct dc_crtc_timing *timing) @@ -699,7 +656,7 @@ void dce120_timing_generator_enable_advanced_request( value); } -void dce120_tg_program_blank_color(struct timing_generator *tg, +static void dce120_tg_program_blank_color(struct timing_generator *tg, const struct tg_color *black_color) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -722,7 +679,7 @@ void dce120_tg_program_blank_color(struct timing_generator *tg, value); } -void dce120_tg_set_overscan_color(struct timing_generator *tg, +static void dce120_tg_set_overscan_color(struct timing_generator *tg, const struct tg_color *overscan_color) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -749,7 +706,7 @@ static void dce120_tg_program_timing(struct timing_generator *tg, dce120_timing_generator_program_blanking(tg, timing); } -bool dce120_tg_is_blanked(struct timing_generator *tg) +static bool dce120_tg_is_blanked(struct timing_generator *tg) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); uint32_t value = dm_read_reg_soc15( @@ -770,7 +727,7 @@ bool dce120_tg_is_blanked(struct timing_generator *tg) return false; } -void dce120_tg_set_blank(struct timing_generator *tg, +static void dce120_tg_set_blank(struct timing_generator *tg, bool enable_blanking) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -789,7 +746,7 @@ void dce120_tg_set_blank(struct timing_generator *tg, bool dce120_tg_validate_timing(struct timing_generator *tg, const struct dc_crtc_timing *timing); -void dce120_tg_wait_for_state(struct timing_generator *tg, +static void dce120_tg_wait_for_state(struct timing_generator *tg, enum crtc_state state) { switch (state) { @@ -806,7 +763,7 @@ void dce120_tg_wait_for_state(struct timing_generator *tg, } } -void dce120_tg_set_colors(struct timing_generator *tg, +static void dce120_tg_set_colors(struct timing_generator *tg, const struct tg_color *blank_color, const struct tg_color *overscan_color) { @@ -833,7 +790,7 @@ static void dce120_timing_generator_set_static_screen_control( CRTC_STATIC_SCREEN_FRAME_COUNT, num_frames); } -void dce120_timing_generator_set_test_pattern( +static void dce120_timing_generator_set_test_pattern( struct timing_generator *tg, /* TODO: replace 'controller_dp_test_pattern' by 'test_pattern_mode' * because this is not DP-specific (which is probably somewhere in DP diff --git a/drivers/gpu/drm/amd/display/dc/dce60/Makefile b/drivers/gpu/drm/amd/display/dc/dce60/Makefile index 7036c3bd0f87..dda596fa1cd7 100644 --- a/drivers/gpu/drm/amd/display/dc/dce60/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce60/Makefile @@ -23,6 +23,8 @@ # Makefile for the 'controller' sub-component of DAL. # It provides the control and status of HW CRTC block. +CFLAGS_AMDDALPATH)/dc/dce60/dce60_resource.o = $(call cc-disable-warning, override-init) + DCE60 = dce60_timing_generator.o dce60_hw_sequencer.o \ dce60_resource.o diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c index e9dd78c484d6..dcfa0a3efa00 100644 --- a/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c @@ -60,6 +60,8 @@ #include "dce/dce_i2c.h" /* TODO remove this include */ +#include "dce60_resource.h" + #ifndef mmMC_HUB_RDREQ_DMIF_LIMIT #include "gmc/gmc_6_0_d.h" #include "gmc/gmc_6_0_sh_mask.h" @@ -519,7 +521,7 @@ static struct output_pixel_processor *dce60_opp_create( return &opp->base; } -struct dce_aux *dce60_aux_engine_create( +static struct dce_aux *dce60_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -557,7 +559,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK) }; -struct dce_i2c_hw *dce60_i2c_hw_create( +static struct dce_i2c_hw *dce60_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { @@ -573,7 +575,7 @@ struct dce_i2c_hw *dce60_i2c_hw_create( return dce_i2c_hw; } -struct dce_i2c_sw *dce60_i2c_sw_create( +static struct dce_i2c_sw *dce60_i2c_sw_create( struct dc_context *ctx) { struct dce_i2c_sw *dce_i2c_sw = @@ -707,7 +709,7 @@ static const struct encoder_feature_support link_enc_feature = { .flags.bits.IS_TPS3_CAPABLE = true }; -struct link_encoder *dce60_link_encoder_create( +static struct link_encoder *dce60_link_encoder_create( const struct encoder_init_data *enc_init_data) { struct dce110_link_encoder *enc110 = @@ -746,7 +748,7 @@ static struct panel_cntl *dce60_panel_cntl_create(const struct panel_cntl_init_d return &panel_cntl->base; } -struct clock_source *dce60_clock_source_create( +static struct clock_source *dce60_clock_source_create( struct dc_context *ctx, struct dc_bios *bios, enum clock_source_id id, @@ -770,7 +772,7 @@ struct clock_source *dce60_clock_source_create( return NULL; } -void dce60_clock_source_destroy(struct clock_source **clk_src) +static void dce60_clock_source_destroy(struct clock_source **clk_src) { kfree(TO_DCE110_CLK_SRC(*clk_src)); *clk_src = NULL; @@ -860,7 +862,7 @@ static void dce60_resource_destruct(struct dce110_resource_pool *pool) } } -bool dce60_validate_bandwidth( +static bool dce60_validate_bandwidth( struct dc *dc, struct dc_state *context, bool fast_validate) @@ -905,7 +907,7 @@ static bool dce60_validate_surface_sets( return true; } -enum dc_status dce60_validate_global( +static enum dc_status dce60_validate_global( struct dc *dc, struct dc_state *context) { diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c index fc1af0ff0ca4..c1a85ee374d9 100644 --- a/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c @@ -189,8 +189,8 @@ static bool dce60_is_tg_enabled(struct timing_generator *tg) return field == 1; } -bool dce60_configure_crc(struct timing_generator *tg, - const struct crc_params *params) +static bool dce60_configure_crc(struct timing_generator *tg, + const struct crc_params *params) { /* Cannot configure crc on a CRTC that is disabled */ if (!dce60_is_tg_enabled(tg)) diff --git a/drivers/gpu/drm/amd/display/dc/dce80/Makefile b/drivers/gpu/drm/amd/display/dc/dce80/Makefile index 666fcb2bdbba..0a9d1a350d8b 100644 --- a/drivers/gpu/drm/amd/display/dc/dce80/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce80/Makefile @@ -23,6 +23,8 @@ # Makefile for the 'controller' sub-component of DAL. # It provides the control and status of HW CRTC block. +CFLAGS_$(AMDDALPATH)/dc/dce80/dce80_resource.o = $(call cc-disable-warning, override-init) + DCE80 = dce80_timing_generator.o dce80_hw_sequencer.o \ dce80_resource.o diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c index 390a0fa37239..612450f99278 100644 --- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c @@ -60,6 +60,8 @@ #include "dce/dce_i2c.h" /* TODO remove this include */ +#include "dce80_resource.h" + #ifndef mmMC_HUB_RDREQ_DMIF_LIMIT #include "gmc/gmc_7_1_d.h" #include "gmc/gmc_7_1_sh_mask.h" @@ -402,7 +404,7 @@ static const struct dc_plane_cap plane_cap = { .pixel_format_support = { .argb8888 = true, .nv12 = false, - .fp16 = false + .fp16 = true }, .max_upscale_factor = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c index 4d3f7d5e1473..904c2d278998 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c @@ -577,7 +577,7 @@ void dpp1_power_on_degamma_lut( struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base); REG_SET(CM_MEM_PWR_CTRL, 0, - SHARED_MEM_PWR_DIS, power_on == true ? 0:1); + SHARED_MEM_PWR_DIS, power_on ? 0:1); } diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c index 017b67b830e6..89912bb5014f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c @@ -1224,6 +1224,7 @@ void dcn10_init_pipes(struct dc *dc, struct dc_state *context) // signals when OTG blanked. This is to prevent pipe from // requesting data while in PSR. tg->funcs->tg_init(tg); + hubp->power_gated = true; continue; } @@ -2634,7 +2635,7 @@ static void dcn10_update_dchubp_dpp( hws->funcs.update_plane_addr(dc, pipe_ctx); if (is_pipe_tree_visible(pipe_ctx)) - hubp->funcs->set_blank(hubp, false); + dc->hwss.set_hubp_blank(dc, pipe_ctx, false); } void dcn10_blank_pixel_data( @@ -3134,7 +3135,7 @@ void dcn10_setup_stereo(struct pipe_ctx *pipe_ctx, struct dc *dc) pipe_ctx->stream_res.opp->funcs->opp_program_stereo( pipe_ctx->stream_res.opp, - flags.PROGRAM_STEREO == 1 ? true:false, + flags.PROGRAM_STEREO == 1, &stream->timing); pipe_ctx->stream_res.tg->funcs->program_stereo( @@ -3145,13 +3146,16 @@ void dcn10_setup_stereo(struct pipe_ctx *pipe_ctx, struct dc *dc) return; } -static struct hubp *get_hubp_by_inst(struct resource_pool *res_pool, int mpcc_inst) +static struct pipe_ctx *get_pipe_ctx_by_hubp_inst(struct dc_state *context, int mpcc_inst) { int i; - for (i = 0; i < res_pool->pipe_count; i++) { - if (res_pool->hubps[i]->inst == mpcc_inst) - return res_pool->hubps[i]; + for (i = 0; i < MAX_PIPES; i++) { + if (context->res_ctx.pipe_ctx[i].plane_res.hubp + && context->res_ctx.pipe_ctx[i].plane_res.hubp->inst == mpcc_inst) { + return &context->res_ctx.pipe_ctx[i]; + } + } ASSERT(false); return NULL; @@ -3174,11 +3178,23 @@ void dcn10_wait_for_mpcc_disconnect( for (mpcc_inst = 0; mpcc_inst < MAX_PIPES; mpcc_inst++) { if (pipe_ctx->stream_res.opp->mpcc_disconnect_pending[mpcc_inst]) { - struct hubp *hubp = get_hubp_by_inst(res_pool, mpcc_inst); + struct pipe_ctx *restore_bottom_pipe; + struct pipe_ctx *restore_top_pipe; + struct pipe_ctx *inst_pipe_ctx = get_pipe_ctx_by_hubp_inst(dc->current_state, mpcc_inst); + ASSERT(inst_pipe_ctx); res_pool->mpc->funcs->wait_for_idle(res_pool->mpc, mpcc_inst); pipe_ctx->stream_res.opp->mpcc_disconnect_pending[mpcc_inst] = false; - hubp->funcs->set_blank(hubp, true); + /* + * Set top and bottom pipes NULL, as we don't want + * to blank those pipes when disconnecting from MPCC + */ + restore_bottom_pipe = inst_pipe_ctx->bottom_pipe; + restore_top_pipe = inst_pipe_ctx->top_pipe; + inst_pipe_ctx->top_pipe = inst_pipe_ctx->bottom_pipe = NULL; + dc->hwss.set_hubp_blank(dc, inst_pipe_ctx, true); + inst_pipe_ctx->top_pipe = restore_top_pipe; + inst_pipe_ctx->bottom_pipe = restore_bottom_pipe; } } @@ -3731,3 +3747,10 @@ void dcn10_get_clock(struct dc *dc, dc->clk_mgr->funcs->get_clock(dc->clk_mgr, context, clock_type, clock_cfg); } + +void dcn10_set_hubp_blank(const struct dc *dc, + struct pipe_ctx *pipe_ctx, + bool blank_enable) +{ + pipe_ctx->plane_res.hubp->funcs->set_blank(pipe_ctx->plane_res.hubp, blank_enable); +} diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h index e5691e499023..89e6dfb63da0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h @@ -163,6 +163,8 @@ void dcn10_wait_for_mpcc_disconnect( void dce110_edp_backlight_control( struct dc_link *link, bool enable); +void dce110_edp_wait_for_T12( + struct dc_link *link); void dce110_edp_power_control( struct dc_link *link, bool power_up); @@ -202,5 +204,8 @@ void dcn10_wait_for_pending_cleared(struct dc *dc, struct dc_state *context); void dcn10_set_hdr_multiplier(struct pipe_ctx *pipe_ctx); void dcn10_verify_allow_pstate_change_high(struct dc *dc); +void dcn10_set_hubp_blank(const struct dc *dc, + struct pipe_ctx *pipe_ctx, + bool blank_enable); #endif /* __DC_HWSS_DCN10_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c index 7f4766e45dff..e8b6065fffad 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c @@ -47,7 +47,7 @@ unsigned int snprintf_count(char *pBuf, unsigned int bufSize, char *fmt, ...) { - unsigned int ret_vsnprintf; + int ret_vsnprintf; unsigned int chars_printed; va_list args; diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c index 254300b06b43..2f1b802e66a1 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c @@ -79,6 +79,7 @@ static const struct hw_sequencer_funcs dcn10_funcs = { .set_backlight_level = dce110_set_backlight_level, .set_abm_immediate_disable = dce110_set_abm_immediate_disable, .set_pipe = dce110_set_pipe, + .set_hubp_blank = dcn10_set_hubp_blank, }; static const struct hwseq_private_funcs dcn10_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c index 81db0179f7ea..e4701825b5a0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c @@ -480,7 +480,6 @@ unsigned int dcn10_get_dig_frontend(struct link_encoder *enc) break; default: // invalid source select DIG - ASSERT(false); result = ENGINE_ID_UNKNOWN; } @@ -956,6 +955,21 @@ void dcn10_link_encoder_enable_tmds_output( } } +void dcn10_link_encoder_enable_tmds_output_with_clk_pattern_wa( + struct link_encoder *enc, + enum clock_source_id clock_source, + enum dc_color_depth color_depth, + enum signal_type signal, + uint32_t pixel_clock) +{ + struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc); + + dcn10_link_encoder_enable_tmds_output( + enc, clock_source, color_depth, signal, pixel_clock); + + REG_UPDATE(DIG_CLOCK_PATTERN, DIG_CLOCK_PATTERN, 0x1F); +} + /* enables DP PHY output */ void dcn10_link_encoder_enable_dp_output( struct link_encoder *enc, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h index d4caad670855..3e1a582e4b88 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h @@ -42,6 +42,7 @@ #define LE_DCN_COMMON_REG_LIST(id) \ SRI(DIG_BE_CNTL, DIG, id), \ SRI(DIG_BE_EN_CNTL, DIG, id), \ + SRI(DIG_CLOCK_PATTERN, DIG, id), \ SRI(TMDS_CTL_BITS, DIG, id), \ SRI(DP_CONFIG, DP, id), \ SRI(DP_DPHY_CNTL, DP, id), \ @@ -83,6 +84,7 @@ struct dcn10_link_enc_hpd_registers { struct dcn10_link_enc_registers { uint32_t DIG_BE_CNTL; uint32_t DIG_BE_EN_CNTL; + uint32_t DIG_CLOCK_PATTERN; uint32_t DP_CONFIG; uint32_t DP_DPHY_CNTL; uint32_t DP_DPHY_INTERNAL_CTRL; @@ -168,6 +170,7 @@ struct dcn10_link_enc_registers { LE_SF(DIG0_DIG_BE_CNTL, DIG_HPD_SELECT, mask_sh),\ LE_SF(DIG0_DIG_BE_CNTL, DIG_MODE, mask_sh),\ LE_SF(DIG0_DIG_BE_CNTL, DIG_FE_SOURCE_SELECT, mask_sh),\ + LE_SF(DIG0_DIG_CLOCK_PATTERN, DIG_CLOCK_PATTERN, mask_sh),\ LE_SF(DIG0_TMDS_CTL_BITS, TMDS_CTL0, mask_sh), \ LE_SF(DP0_DP_DPHY_CNTL, DPHY_BYPASS, mask_sh),\ LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE0, mask_sh),\ @@ -218,6 +221,7 @@ struct dcn10_link_enc_registers { type DIG_HPD_SELECT;\ type DIG_MODE;\ type DIG_FE_SOURCE_SELECT;\ + type DIG_CLOCK_PATTERN;\ type DPHY_BYPASS;\ type DPHY_ATEST_SEL_LANE0;\ type DPHY_ATEST_SEL_LANE1;\ @@ -536,6 +540,13 @@ void dcn10_link_encoder_enable_tmds_output( enum signal_type signal, uint32_t pixel_clock); +void dcn10_link_encoder_enable_tmds_output_with_clk_pattern_wa( + struct link_encoder *enc, + enum clock_source_id clock_source, + enum dc_color_depth color_depth, + enum signal_type signal, + uint32_t pixel_clock); + /* enables DP PHY output */ void dcn10_link_encoder_enable_dp_output( struct link_encoder *enc, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c index f033397a84e9..6138f4887de7 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c @@ -659,6 +659,16 @@ void optc1_unlock(struct timing_generator *optc) OTG_MASTER_UPDATE_LOCK, 0); } +bool optc1_is_locked(struct timing_generator *optc) +{ + struct optc *optc1 = DCN10TG_FROM_TG(optc); + uint32_t locked; + + REG_GET(OTG_MASTER_UPDATE_LOCK, UPDATE_LOCK_STATUS, &locked); + + return (locked == 1); +} + void optc1_get_position(struct timing_generator *optc, struct crtc_position *position) { @@ -1513,6 +1523,7 @@ static const struct timing_generator_funcs dcn10_tg_funcs = { .enable_crtc_reset = optc1_enable_crtc_reset, .disable_reset_trigger = optc1_disable_reset_trigger, .lock = optc1_lock, + .is_locked = optc1_is_locked, .unlock = optc1_unlock, .enable_optc_clock = optc1_enable_optc_clock, .set_drr = optc1_set_drr, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h index b12bd9aae52f..b222c67973d4 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h @@ -638,6 +638,7 @@ void optc1_set_blank(struct timing_generator *optc, bool enable_blanking); bool optc1_is_blanked(struct timing_generator *optc); +bool optc1_is_locked(struct timing_generator *optc); void optc1_program_blank_color( struct timing_generator *optc, diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index 480d928cb1ca..0726fb435e2a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -1575,8 +1575,8 @@ static void dcn20_update_dchubp_dpp( - if (pipe_ctx->update_flags.bits.enable) - hubp->funcs->set_blank(hubp, false); + if (is_pipe_tree_visible(pipe_ctx)) + dc->hwss.set_hubp_blank(dc, pipe_ctx, false); } @@ -1770,6 +1770,14 @@ void dcn20_post_unlock_program_front_end( } } + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; + + if (pipe->vtp_locked) { + dc->hwss.set_hubp_blank(dc, pipe, true); + pipe->vtp_locked = false; + } + } /* WA to apply WM setting*/ if (hwseq->wa.DEGVIDCN21) dc->res_pool->hubbub->funcs->apply_DEDCN21_147_wa(dc->res_pool->hubbub); diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c index de9dcbeea150..51a4166e9750 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c @@ -94,6 +94,7 @@ static const struct hw_sequencer_funcs dcn20_funcs = { .optimize_timing_for_fsft = dcn20_optimize_timing_for_fsft, #endif .set_disp_pattern_generator = dcn20_set_disp_pattern_generator, + .set_hubp_blank = dcn10_set_hubp_blank, }; static const struct hwseq_private_funcs dcn20_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c index 15c2ff264ff6..fa013496e26b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c @@ -363,7 +363,7 @@ static const struct link_encoder_funcs dcn20_link_enc_funcs = { dcn10_link_encoder_validate_output_with_stream, .hw_init = enc2_hw_init, .setup = dcn10_link_encoder_setup, - .enable_tmds_output = dcn10_link_encoder_enable_tmds_output, + .enable_tmds_output = dcn10_link_encoder_enable_tmds_output_with_clk_pattern_wa, .enable_dp_output = dcn20_link_encoder_enable_dp_output, .enable_dp_mst_output = dcn10_link_encoder_enable_dp_mst_output, .disable_output = dcn10_link_encoder_disable_output, diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c index d6b488561871..2c2dbfcd8957 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c @@ -2097,6 +2097,7 @@ int dcn20_populate_dml_pipes_from_context( pipes[pipe_cnt].pipe.dest.pixel_rate_mhz *= 2; pipes[pipe_cnt].pipe.dest.otg_inst = res_ctx->pipe_ctx[i].stream_res.tg->inst; pipes[pipe_cnt].dout.dp_lanes = 4; + pipes[pipe_cnt].dout.is_virtual = 0; pipes[pipe_cnt].pipe.dest.vtotal_min = res_ctx->pipe_ctx[i].stream->adjust.v_total_min; pipes[pipe_cnt].pipe.dest.vtotal_max = res_ctx->pipe_ctx[i].stream->adjust.v_total_max; switch (get_num_odm_splits(&res_ctx->pipe_ctx[i])) { @@ -2150,6 +2151,7 @@ int dcn20_populate_dml_pipes_from_context( break; default: /* In case there is no signal, set dp with 4 lanes to allow max config */ + pipes[pipe_cnt].dout.is_virtual = 1; pipes[pipe_cnt].dout.output_type = dm_dp; pipes[pipe_cnt].dout.dp_lanes = 4; } @@ -3245,7 +3247,7 @@ restore_dml_state: bool dcn20_validate_bandwidth(struct dc *dc, struct dc_state *context, bool fast_validate) { - bool voltage_supported = false; + bool voltage_supported; DC_FP_START(); voltage_supported = dcn20_validate_bandwidth_fp(dc, context, fast_validate); DC_FP_END(); @@ -3506,7 +3508,8 @@ void dcn20_update_bounding_box(struct dc *dc, struct _vcs_dpi_soc_bounding_box_s calculated_states[i].dram_speed_mts = uclk_states[i] * 16 / 1000; // FCLK:UCLK ratio is 1.08 - min_fclk_required_by_uclk = mul_u64_u32_shr(BIT_ULL(32) * 1080 / 1000000, uclk_states[i], 32); + min_fclk_required_by_uclk = div_u64(((unsigned long long)uclk_states[i]) * 1080, + 1000000); calculated_states[i].fabricclk_mhz = (min_fclk_required_by_uclk < min_dcfclk) ? min_dcfclk : min_fclk_required_by_uclk; @@ -3606,7 +3609,6 @@ static enum dml_project get_dml_project_version(uint32_t hw_internal_rev) static bool init_soc_bounding_box(struct dc *dc, struct dcn20_resource_pool *pool) { - const struct gpu_info_soc_bounding_box_v1_0 *bb = dc->soc_bounding_box; struct _vcs_dpi_soc_bounding_box_st *loaded_bb = get_asic_rev_soc_bb(dc->ctx->asic_id.hw_internal_rev); struct _vcs_dpi_ip_params_st *loaded_ip = @@ -3614,116 +3616,6 @@ static bool init_soc_bounding_box(struct dc *dc, DC_LOGGER_INIT(dc->ctx->logger); - /* TODO: upstream NV12 bounding box when its launched */ - if (!bb && ASICREV_IS_NAVI12_P(dc->ctx->asic_id.hw_internal_rev)) { - DC_LOG_ERROR("%s: not valid soc bounding box/n", __func__); - return false; - } - - if (bb && ASICREV_IS_NAVI12_P(dc->ctx->asic_id.hw_internal_rev)) { - int i; - - dcn2_0_nv12_soc.sr_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_exit_time_us); - dcn2_0_nv12_soc.sr_enter_plus_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_enter_plus_exit_time_us); - dcn2_0_nv12_soc.urgent_latency_us = - fixed16_to_double_to_cpu(bb->urgent_latency_us); - dcn2_0_nv12_soc.urgent_latency_pixel_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_data_only_us); - dcn2_0_nv12_soc.urgent_latency_pixel_mixed_with_vm_data_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_mixed_with_vm_data_us); - dcn2_0_nv12_soc.urgent_latency_vm_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_vm_data_only_us); - dcn2_0_nv12_soc.urgent_out_of_order_return_per_channel_pixel_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_only_bytes); - dcn2_0_nv12_soc.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_and_vm_bytes); - dcn2_0_nv12_soc.urgent_out_of_order_return_per_channel_vm_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_vm_only_bytes); - dcn2_0_nv12_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_only); - dcn2_0_nv12_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm); - dcn2_0_nv12_soc.pct_ideal_dram_sdp_bw_after_urgent_vm_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_vm_only); - dcn2_0_nv12_soc.max_avg_sdp_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_sdp_bw_use_normal_percent); - dcn2_0_nv12_soc.max_avg_dram_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_dram_bw_use_normal_percent); - dcn2_0_nv12_soc.writeback_latency_us = - fixed16_to_double_to_cpu(bb->writeback_latency_us); - dcn2_0_nv12_soc.ideal_dram_bw_after_urgent_percent = - fixed16_to_double_to_cpu(bb->ideal_dram_bw_after_urgent_percent); - dcn2_0_nv12_soc.max_request_size_bytes = - le32_to_cpu(bb->max_request_size_bytes); - dcn2_0_nv12_soc.dram_channel_width_bytes = - le32_to_cpu(bb->dram_channel_width_bytes); - dcn2_0_nv12_soc.fabric_datapath_to_dcn_data_return_bytes = - le32_to_cpu(bb->fabric_datapath_to_dcn_data_return_bytes); - dcn2_0_nv12_soc.dcn_downspread_percent = - fixed16_to_double_to_cpu(bb->dcn_downspread_percent); - dcn2_0_nv12_soc.downspread_percent = - fixed16_to_double_to_cpu(bb->downspread_percent); - dcn2_0_nv12_soc.dram_page_open_time_ns = - fixed16_to_double_to_cpu(bb->dram_page_open_time_ns); - dcn2_0_nv12_soc.dram_rw_turnaround_time_ns = - fixed16_to_double_to_cpu(bb->dram_rw_turnaround_time_ns); - dcn2_0_nv12_soc.dram_return_buffer_per_channel_bytes = - le32_to_cpu(bb->dram_return_buffer_per_channel_bytes); - dcn2_0_nv12_soc.round_trip_ping_latency_dcfclk_cycles = - le32_to_cpu(bb->round_trip_ping_latency_dcfclk_cycles); - dcn2_0_nv12_soc.urgent_out_of_order_return_per_channel_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_bytes); - dcn2_0_nv12_soc.channel_interleave_bytes = - le32_to_cpu(bb->channel_interleave_bytes); - dcn2_0_nv12_soc.num_banks = - le32_to_cpu(bb->num_banks); - dcn2_0_nv12_soc.num_chans = - le32_to_cpu(bb->num_chans); - dcn2_0_nv12_soc.vmm_page_size_bytes = - le32_to_cpu(bb->vmm_page_size_bytes); - dcn2_0_nv12_soc.dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->dram_clock_change_latency_us); - // HACK!! Lower uclock latency switch time so we don't switch - dcn2_0_nv12_soc.dram_clock_change_latency_us = 10; - dcn2_0_nv12_soc.writeback_dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->writeback_dram_clock_change_latency_us); - dcn2_0_nv12_soc.return_bus_width_bytes = - le32_to_cpu(bb->return_bus_width_bytes); - dcn2_0_nv12_soc.dispclk_dppclk_vco_speed_mhz = - le32_to_cpu(bb->dispclk_dppclk_vco_speed_mhz); - dcn2_0_nv12_soc.xfc_bus_transport_time_us = - le32_to_cpu(bb->xfc_bus_transport_time_us); - dcn2_0_nv12_soc.xfc_xbuf_latency_tolerance_us = - le32_to_cpu(bb->xfc_xbuf_latency_tolerance_us); - dcn2_0_nv12_soc.use_urgent_burst_bw = - le32_to_cpu(bb->use_urgent_burst_bw); - dcn2_0_nv12_soc.num_states = - le32_to_cpu(bb->num_states); - - for (i = 0; i < dcn2_0_nv12_soc.num_states; i++) { - dcn2_0_nv12_soc.clock_limits[i].state = - le32_to_cpu(bb->clock_limits[i].state); - dcn2_0_nv12_soc.clock_limits[i].dcfclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dcfclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].fabricclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].fabricclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].dispclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dispclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].dppclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dppclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].phyclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].phyclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].socclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].socclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].dscclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dscclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].dram_speed_mts = - fixed16_to_double_to_cpu(bb->clock_limits[i].dram_speed_mts); - } - } - if (pool->base.pp_smu) { struct pp_smu_nv_clock_table max_clocks = {0}; unsigned int uclk_states[8] = {0}; diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c index 96ee0b82f458..d3b643089603 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c @@ -123,7 +123,7 @@ void dcn21_optimize_pwr_state( * PHY will hang on the next mode set attempt. * if enable PLL follow by disable PLL (without executing lane enable/disable), * RDPCS_PHY_DP_MPLLB_STATE remains 1, - * which indicate that PLL disable attempt actually didn’t go through. + * which indicate that PLL disable attempt actually didn't go through. * As a workaround, insert PHY lane enable/disable before PLL disable. */ void dcn21_PLAT_58856_wa(struct dc_state *context, struct pipe_ctx *pipe_ctx) @@ -143,6 +143,7 @@ static bool dmub_abm_set_pipe(struct abm *abm, uint32_t otg_inst, uint32_t optio struct dc_context *dc = abm->ctx; uint32_t ramping_boundary = 0xFFFF; + memset(&cmd, 0, sizeof(cmd)); cmd.abm_set_pipe.header.type = DMUB_CMD__ABM; cmd.abm_set_pipe.header.sub_type = DMUB_CMD__ABM_SET_PIPE; cmd.abm_set_pipe.abm_set_pipe_data.otg_inst = otg_inst; @@ -212,6 +213,7 @@ bool dcn21_set_backlight_level(struct pipe_ctx *pipe_ctx, if (abm && panel_cntl) dmub_abm_set_pipe(abm, otg_inst, SET_ABM_PIPE_NORMAL, panel_cntl->inst); + memset(&cmd, 0, sizeof(cmd)); cmd.abm_set_backlight.header.type = DMUB_CMD__ABM; cmd.abm_set_backlight.header.sub_type = DMUB_CMD__ABM_SET_BACKLIGHT; cmd.abm_set_backlight.abm_set_backlight_data.frame_ramp = frame_ramp; diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c index 074e2713257f..0597391b2171 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c @@ -99,6 +99,7 @@ static const struct hw_sequencer_funcs dcn21_funcs = { #endif .is_abm_supported = dcn21_is_abm_supported, .set_disp_pattern_generator = dcn20_set_disp_pattern_generator, + .set_hubp_blank = dcn10_set_hubp_blank, }; static const struct hwseq_private_funcs dcn21_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c index 674376428916..072f8c880924 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c @@ -1329,8 +1329,8 @@ validate_out: return out; } -bool dcn21_validate_bandwidth(struct dc *dc, struct dc_state *context, - bool fast_validate) +static noinline bool dcn21_validate_bandwidth_fp(struct dc *dc, + struct dc_state *context, bool fast_validate) { bool out = false; @@ -1383,6 +1383,22 @@ validate_out: return out; } + +/* + * Some of the functions further below use the FPU, so we need to wrap this + * with DC_FP_START()/DC_FP_END(). Use the same approach as for + * dcn20_validate_bandwidth in dcn20_resource.c. + */ +bool dcn21_validate_bandwidth(struct dc *dc, struct dc_state *context, + bool fast_validate) +{ + bool voltage_supported; + DC_FP_START(); + voltage_supported = dcn21_validate_bandwidth_fp(dc, context, fast_validate); + DC_FP_END(); + return voltage_supported; +} + static void dcn21_destroy_resource_pool(struct resource_pool **pool) { struct dcn21_resource_pool *dcn21_pool = TO_DCN21_RES_POOL(*pool); diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp_cm.c index 9da66e491116..33985401f25c 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp_cm.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp_cm.c @@ -133,7 +133,6 @@ static void dpp3_power_on_gamcor_lut( struct dpp *dpp_base, bool power_on) { - uint32_t power_status; struct dcn3_dpp *dpp = TO_DCN30_DPP(dpp_base); if (dpp_base->ctx->dc->debug.enable_mem_low_power.bits.cm) { @@ -143,12 +142,6 @@ static void dpp3_power_on_gamcor_lut( } else REG_SET(CM_MEM_PWR_CTRL, 0, GAMCOR_MEM_PWR_DIS, power_on == true ? 0:1); - - REG_GET(CM_MEM_PWR_STATUS, GAMCOR_MEM_PWR_STATE, &power_status); - if (power_status != 0) - BREAK_TO_DEBUGGER(); - - } void dpp3_program_cm_dealpha( diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.h index 5fa150f34c60..705fbfc37502 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.h +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.h @@ -62,6 +62,7 @@ HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_NO_OUTSTANDING_REQ, mask_sh),\ HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_VTG_SEL, mask_sh),\ HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_DISABLE, mask_sh),\ + HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_IN_BLANK, mask_sh),\ HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_PIPES, mask_sh),\ HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, PIPE_INTERLEAVE, mask_sh),\ HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, MAX_COMPRESSED_FRAGS, mask_sh),\ diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c index 3deb3fb1724d..06dc1e2e8383 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c @@ -539,6 +539,8 @@ void dcn30_init_hw(struct dc *dc) fe = dc->links[i]->link_enc->funcs->get_dig_frontend( dc->links[i]->link_enc); + if (fe == ENGINE_ID_UNKNOWN) + continue; for (j = 0; j < dc->res_pool->stream_enc_count; j++) { if (fe == dc->res_pool->stream_enc[j]->id) { @@ -710,8 +712,11 @@ void dcn30_program_dmdata_engine(struct pipe_ctx *pipe_ctx) bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable) { union dmub_rb_cmd cmd; - unsigned int surface_size, refresh_hz, denom; uint32_t tmr_delay = 0, tmr_scale = 0; + struct dc_cursor_attributes cursor_attr; + bool cursor_cache_enable = false; + struct dc_stream_state *stream = NULL; + struct dc_plane_state *plane = NULL; if (!dc->ctx->dmub_srv) return false; @@ -722,72 +727,150 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable) /* First, check no-memory-requests case */ for (i = 0; i < dc->current_state->stream_count; i++) { - if (dc->current_state->stream_status[i] - .plane_count) + if (dc->current_state->stream_status[i].plane_count) /* Fail eligibility on a visible stream */ break; } - if (dc->current_state->stream_count == 1 // single display only - && dc->current_state->stream_status[0].plane_count == 1 // single surface only - && dc->current_state->stream_status[0].plane_states[0]->address.page_table_base.quad_part == 0 // no VM - // Only 8 and 16 bit formats - && dc->current_state->stream_status[0].plane_states[0]->format <= SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F - && dc->current_state->stream_status[0].plane_states[0]->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB8888) { - surface_size = dc->current_state->stream_status[0].plane_states[0]->plane_size.surface_pitch * - dc->current_state->stream_status[0].plane_states[0]->plane_size.surface_size.height * - (dc->current_state->stream_status[0].plane_states[0]->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? - 8 : 4); - } else { - // TODO: remove hard code size - surface_size = 128 * 1024 * 1024; + if (i == dc->current_state->stream_count) { + /* Enable no-memory-requests case */ + memset(&cmd, 0, sizeof(cmd)); + cmd.mall.header.type = DMUB_CMD__MALL; + cmd.mall.header.sub_type = DMUB_CMD__MALL_ACTION_NO_DF_REQ; + cmd.mall.header.payload_bytes = sizeof(cmd.mall) - sizeof(cmd.mall.header); + + dc_dmub_srv_cmd_queue(dc->ctx->dmub_srv, &cmd); + dc_dmub_srv_cmd_execute(dc->ctx->dmub_srv); + + return true; } - // TODO: remove hard code size - if (surface_size < 128 * 1024 * 1024) { - refresh_hz = div_u64((unsigned long long) dc->current_state->streams[0]->timing.pix_clk_100hz * - 100LL, - (dc->current_state->streams[0]->timing.v_total * - dc->current_state->streams[0]->timing.h_total)); + stream = dc->current_state->streams[0]; + plane = (stream ? dc->current_state->stream_status[0].plane_states[0] : NULL); + + if (stream && plane) { + cursor_cache_enable = stream->cursor_position.enable && + plane->address.grph.cursor_cache_addr.quad_part; + cursor_attr = stream->cursor_attributes; + } + + /* + * Second, check MALL eligibility + * + * single display only, single surface only, 8 and 16 bit formats only, no VM, + * do not use MALL for displays that support PSR as they use D0i3.2 in DMCUB FW + * + * TODO: When we implement multi-display, PSR displays will be allowed if there is + * a non-PSR display present, since in that case we can't do D0i3.2 + */ + if (dc->current_state->stream_count == 1 && + stream->link->psr_settings.psr_version == DC_PSR_VERSION_UNSUPPORTED && + dc->current_state->stream_status[0].plane_count == 1 && + plane->format <= SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F && + plane->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB8888 && + plane->address.page_table_base.quad_part == 0 && + dc->hwss.does_plane_fit_in_mall && + dc->hwss.does_plane_fit_in_mall(dc, plane, + cursor_cache_enable ? &cursor_attr : NULL)) { + unsigned int v_total = stream->adjust.v_total_max ? + stream->adjust.v_total_max : stream->timing.v_total; + unsigned int refresh_hz = div_u64((unsigned long long) stream->timing.pix_clk_100hz * + 100LL, (v_total * stream->timing.h_total)); /* - * Delay_Us = 65.28 * (64 + MallFrameCacheTmrDly) * 2^MallFrameCacheTmrScale - * Delay_Us / 65.28 = (64 + MallFrameCacheTmrDly) * 2^MallFrameCacheTmrScale - * (Delay_Us / 65.28) / 2^MallFrameCacheTmrScale = 64 + MallFrameCacheTmrDly - * MallFrameCacheTmrDly = ((Delay_Us / 65.28) / 2^MallFrameCacheTmrScale) - 64 - * = (1000000 / refresh) / 65.28 / 2^MallFrameCacheTmrScale - 64 - * = 1000000 / (refresh * 65.28 * 2^MallFrameCacheTmrScale) - 64 - * = (1000000 * 100) / (refresh * 6528 * 2^MallFrameCacheTmrScale) - 64 + * one frame time in microsec: + * Delay_Us = 1000000 / refresh + * dynamic_delay_us = 1000000 / refresh + 2 * stutter_period + * + * one frame time modified by 'additional timer percent' (p): + * Delay_Us_modified = dynamic_delay_us + dynamic_delay_us * p / 100 + * = dynamic_delay_us * (1 + p / 100) + * = (1000000 / refresh + 2 * stutter_period) * (100 + p) / 100 + * = (1000000 + 2 * stutter_period * refresh) * (100 + p) / (100 * refresh) + * + * formula for timer duration based on parameters, from regspec: + * dynamic_delay_us = 65.28 * (64 + MallFrameCacheTmrDly) * 2^MallFrameCacheTmrScale + * + * dynamic_delay_us / 65.28 = (64 + MallFrameCacheTmrDly) * 2^MallFrameCacheTmrScale + * (dynamic_delay_us / 65.28) / 2^MallFrameCacheTmrScale = 64 + MallFrameCacheTmrDly + * MallFrameCacheTmrDly = ((dynamic_delay_us / 65.28) / 2^MallFrameCacheTmrScale) - 64 + * = (1000000 + 2 * stutter_period * refresh) * (100 + p) / (100 * refresh) / 65.28 / 2^MallFrameCacheTmrScale - 64 + * = (1000000 + 2 * stutter_period * refresh) * (100 + p) / (refresh * 6528 * 2^MallFrameCacheTmrScale) - 64 * * need to round up the result of the division before the subtraction */ - denom = refresh_hz * 6528; - tmr_delay = div_u64((100000000LL + denom - 1), denom) - 64LL; + unsigned int denom = refresh_hz * 6528; + unsigned int stutter_period = dc->current_state->perf_params.stutter_period_us; + + tmr_delay = div_u64(((1000000LL + 2 * stutter_period * refresh_hz) * + (100LL + dc->debug.mall_additional_timer_percent) + denom - 1), + denom) - 64LL; /* scale should be increased until it fits into 6 bits */ while (tmr_delay & ~0x3F) { tmr_scale++; if (tmr_scale > 3) { - /* The delay exceeds the range of the hystersis timer */ + /* Delay exceeds range of hysteresis timer */ ASSERT(false); return false; } denom *= 2; - tmr_delay = div_u64((100000000LL + denom - 1), denom) - 64LL; + tmr_delay = div_u64(((1000000LL + 2 * stutter_period * refresh_hz) * + (100LL + dc->debug.mall_additional_timer_percent) + denom - 1), + denom) - 64LL; + } + + /* Copy HW cursor */ + if (cursor_cache_enable) { + memset(&cmd, 0, sizeof(cmd)); + cmd.mall.header.type = DMUB_CMD__MALL; + cmd.mall.header.sub_type = DMUB_CMD__MALL_ACTION_COPY_CURSOR; + cmd.mall.header.payload_bytes = + sizeof(cmd.mall) - sizeof(cmd.mall.header); + + switch (cursor_attr.color_format) { + case CURSOR_MODE_MONO: + cmd.mall.cursor_bpp = 2; + break; + case CURSOR_MODE_COLOR_1BIT_AND: + case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA: + case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA: + cmd.mall.cursor_bpp = 32; + break; + + case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED: + case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED: + cmd.mall.cursor_bpp = 64; + break; + } + + cmd.mall.cursor_copy_src.quad_part = cursor_attr.address.quad_part; + cmd.mall.cursor_copy_dst.quad_part = + plane->address.grph.cursor_cache_addr.quad_part; + cmd.mall.cursor_width = cursor_attr.width; + cmd.mall.cursor_height = cursor_attr.height; + cmd.mall.cursor_pitch = cursor_attr.pitch; + + dc_dmub_srv_cmd_queue(dc->ctx->dmub_srv, &cmd); + dc_dmub_srv_cmd_execute(dc->ctx->dmub_srv); + dc_dmub_srv_wait_idle(dc->ctx->dmub_srv); + + /* Use copied cursor, and it's okay to not switch back */ + cursor_attr.address.quad_part = + plane->address.grph.cursor_cache_addr.quad_part; + dc_stream_set_cursor_attributes(stream, &cursor_attr); } /* Enable MALL */ memset(&cmd, 0, sizeof(cmd)); cmd.mall.header.type = DMUB_CMD__MALL; - cmd.mall.header.sub_type = - DMUB_CMD__MALL_ACTION_ALLOW; - cmd.mall.header.payload_bytes = - sizeof(cmd.mall) - - sizeof(cmd.mall.header); + cmd.mall.header.sub_type = DMUB_CMD__MALL_ACTION_ALLOW; + cmd.mall.header.payload_bytes = sizeof(cmd.mall) - sizeof(cmd.mall.header); cmd.mall.tmr_delay = tmr_delay; cmd.mall.tmr_scale = tmr_scale; + cmd.mall.debug_bits = dc->debug.mall_error_as_fatal; dc_dmub_srv_cmd_queue(dc->ctx->dmub_srv, &cmd); dc_dmub_srv_cmd_execute(dc->ctx->dmub_srv); @@ -814,6 +897,40 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable) return true; } +bool dcn30_does_plane_fit_in_mall(struct dc *dc, struct dc_plane_state *plane, struct dc_cursor_attributes *cursor_attr) +{ + // add meta size? + unsigned int surface_size = plane->plane_size.surface_pitch * plane->plane_size.surface_size.height * + (plane->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? 8 : 4); + unsigned int mall_size = dc->caps.mall_size_total; + unsigned int cursor_size = 0; + + if (dc->debug.mall_size_override) + mall_size = 1024 * 1024 * dc->debug.mall_size_override; + + if (cursor_attr) { + cursor_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size; + + switch (cursor_attr->color_format) { + case CURSOR_MODE_MONO: + cursor_size /= 2; + break; + case CURSOR_MODE_COLOR_1BIT_AND: + case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA: + case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA: + cursor_size *= 4; + break; + + case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED: + case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED: + cursor_size *= 8; + break; + } + } + + return (surface_size + cursor_size) < mall_size; +} + void dcn30_hardware_release(struct dc *dc) { /* if pstate unsupported, force it supported */ @@ -823,6 +940,53 @@ void dcn30_hardware_release(struct dc *dc) dc->res_pool->hubbub, true, true); } +void dcn30_set_hubp_blank(const struct dc *dc, + struct pipe_ctx *pipe_ctx, + bool blank_enable) +{ + struct pipe_ctx *mpcc_pipe; + struct pipe_ctx *odm_pipe; + + if (blank_enable) { + struct plane_resource *plane_res = &pipe_ctx->plane_res; + struct stream_resource *stream_res = &pipe_ctx->stream_res; + + /* Wait for enter vblank */ + stream_res->tg->funcs->wait_for_state(stream_res->tg, CRTC_STATE_VBLANK); + + /* Blank HUBP to allow p-state during blank on all timings */ + pipe_ctx->plane_res.hubp->funcs->set_blank(pipe_ctx->plane_res.hubp, true); + /* Confirm hubp in blank */ + ASSERT(plane_res->hubp->funcs->hubp_in_blank(plane_res->hubp)); + /* Toggle HUBP_DISABLE */ + plane_res->hubp->funcs->hubp_soft_reset(plane_res->hubp, true); + plane_res->hubp->funcs->hubp_soft_reset(plane_res->hubp, false); + for (mpcc_pipe = pipe_ctx->bottom_pipe; mpcc_pipe; mpcc_pipe = mpcc_pipe->bottom_pipe) { + mpcc_pipe->plane_res.hubp->funcs->set_blank(mpcc_pipe->plane_res.hubp, true); + /* Confirm hubp in blank */ + ASSERT(mpcc_pipe->plane_res.hubp->funcs->hubp_in_blank(mpcc_pipe->plane_res.hubp)); + /* Toggle HUBP_DISABLE */ + mpcc_pipe->plane_res.hubp->funcs->hubp_soft_reset(mpcc_pipe->plane_res.hubp, true); + mpcc_pipe->plane_res.hubp->funcs->hubp_soft_reset(mpcc_pipe->plane_res.hubp, false); + + } + for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe) { + odm_pipe->plane_res.hubp->funcs->set_blank(odm_pipe->plane_res.hubp, true); + /* Confirm hubp in blank */ + ASSERT(odm_pipe->plane_res.hubp->funcs->hubp_in_blank(odm_pipe->plane_res.hubp)); + /* Toggle HUBP_DISABLE */ + odm_pipe->plane_res.hubp->funcs->hubp_soft_reset(odm_pipe->plane_res.hubp, true); + odm_pipe->plane_res.hubp->funcs->hubp_soft_reset(odm_pipe->plane_res.hubp, false); + } + } else { + pipe_ctx->plane_res.hubp->funcs->set_blank(pipe_ctx->plane_res.hubp, false); + for (mpcc_pipe = pipe_ctx->bottom_pipe; mpcc_pipe; mpcc_pipe = mpcc_pipe->bottom_pipe) + mpcc_pipe->plane_res.hubp->funcs->set_blank(mpcc_pipe->plane_res.hubp, false); + for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe) + odm_pipe->plane_res.hubp->funcs->set_blank(odm_pipe->plane_res.hubp, false); + } +} + void dcn30_set_disp_pattern_generator(const struct dc *dc, struct pipe_ctx *pipe_ctx, enum controller_dp_test_pattern test_pattern, @@ -831,6 +995,25 @@ void dcn30_set_disp_pattern_generator(const struct dc *dc, const struct tg_color *solid_color, int width, int height, int offset) { - pipe_ctx->stream_res.opp->funcs->opp_set_disp_pattern_generator(pipe_ctx->stream_res.opp, test_pattern, - color_space, color_depth, solid_color, width, height, offset); + struct stream_resource *stream_res = &pipe_ctx->stream_res; + + if (test_pattern != CONTROLLER_DP_TEST_PATTERN_VIDEOMODE) { + pipe_ctx->vtp_locked = false; + /* turning on DPG */ + stream_res->opp->funcs->opp_set_disp_pattern_generator(stream_res->opp, test_pattern, color_space, + color_depth, solid_color, width, height, offset); + + /* Defer hubp blank if tg is locked */ + if (stream_res->tg->funcs->is_tg_enabled(stream_res->tg)) { + if (stream_res->tg->funcs->is_locked(stream_res->tg)) + pipe_ctx->vtp_locked = true; + else + dc->hwss.set_hubp_blank(dc, pipe_ctx, true); + } + } else { + dc->hwss.set_hubp_blank(dc, pipe_ctx, false); + /* turning off DPG */ + stream_res->opp->funcs->opp_set_disp_pattern_generator(stream_res->opp, test_pattern, color_space, + color_depth, solid_color, width, height, offset); + } } diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.h index 7d32c43aafe0..3b7d4812e311 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.h +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.h @@ -65,6 +65,9 @@ void dcn30_set_avmute(struct pipe_ctx *pipe_ctx, bool enable); void dcn30_update_info_frame(struct pipe_ctx *pipe_ctx); void dcn30_program_dmdata_engine(struct pipe_ctx *pipe_ctx); +bool dcn30_does_plane_fit_in_mall(struct dc *dc, struct dc_plane_state *plane, + struct dc_cursor_attributes *cursor_attr); + bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable); void dcn30_hardware_release(struct dc *dc); @@ -77,4 +80,8 @@ void dcn30_set_disp_pattern_generator(const struct dc *dc, const struct tg_color *solid_color, int width, int height, int offset); +void dcn30_set_hubp_blank(const struct dc *dc, + struct pipe_ctx *pipe_ctx, + bool blank_enable); + #endif /* __DC_HWSS_DCN30_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c index 6125fe440ad0..204444fead97 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c @@ -71,6 +71,7 @@ static const struct hw_sequencer_funcs dcn30_funcs = { .edp_backlight_control = dce110_edp_backlight_control, .edp_power_control = dce110_edp_power_control, .edp_wait_for_hpd_ready = dce110_edp_wait_for_hpd_ready, + .edp_wait_for_T12 = dce110_edp_wait_for_T12, .set_cursor_position = dcn10_set_cursor_position, .set_cursor_attribute = dcn10_set_cursor_attribute, .set_cursor_sdr_white_level = dcn10_set_cursor_sdr_white_level, @@ -91,11 +92,13 @@ static const struct hw_sequencer_funcs dcn30_funcs = { .get_vupdate_offset_from_vsync = dcn10_get_vupdate_offset_from_vsync, .calc_vupdate_position = dcn10_calc_vupdate_position, .apply_idle_power_optimizations = dcn30_apply_idle_power_optimizations, + .does_plane_fit_in_mall = dcn30_does_plane_fit_in_mall, .set_backlight_level = dcn21_set_backlight_level, .set_abm_immediate_disable = dcn21_set_abm_immediate_disable, .hardware_release = dcn30_hardware_release, .set_pipe = dcn21_set_pipe, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, + .set_hubp_blank = dcn30_set_hubp_blank, }; static const struct hwseq_private_funcs dcn30_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c index 3ba3991ee612..8980c90b2277 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c @@ -309,6 +309,7 @@ static struct timing_generator_funcs dcn30_tg_funcs = { .enable_crtc_reset = optc1_enable_crtc_reset, .disable_reset_trigger = optc1_disable_reset_trigger, .lock = optc3_lock, + .is_locked = optc1_is_locked, .unlock = optc1_unlock, .lock_doublebuffer_enable = optc3_lock_doublebuffer_enable, .lock_doublebuffer_disable = optc3_lock_doublebuffer_disable, diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c index 5e126fdf6ec1..8d0f663489ac 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c @@ -1716,125 +1716,22 @@ static bool is_soc_bounding_box_valid(struct dc *dc) static bool init_soc_bounding_box(struct dc *dc, struct dcn30_resource_pool *pool) { - const struct gpu_info_soc_bounding_box_v1_0 *bb = dc->soc_bounding_box; struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_0_soc; struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_0_ip; DC_LOGGER_INIT(dc->ctx->logger); - if (!bb && !is_soc_bounding_box_valid(dc)) { + if (!is_soc_bounding_box_valid(dc)) { DC_LOG_ERROR("%s: not valid soc bounding box/n", __func__); return false; } - if (bb && !is_soc_bounding_box_valid(dc)) { - int i; - - dcn3_0_soc.sr_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_exit_time_us); - dcn3_0_soc.sr_enter_plus_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_enter_plus_exit_time_us); - dcn3_0_soc.urgent_latency_us = - fixed16_to_double_to_cpu(bb->urgent_latency_us); - dcn3_0_soc.urgent_latency_pixel_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_data_only_us); - dcn3_0_soc.urgent_latency_pixel_mixed_with_vm_data_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_mixed_with_vm_data_us); - dcn3_0_soc.urgent_latency_vm_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_vm_data_only_us); - dcn3_0_soc.urgent_out_of_order_return_per_channel_pixel_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_only_bytes); - dcn3_0_soc.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_and_vm_bytes); - dcn3_0_soc.urgent_out_of_order_return_per_channel_vm_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_vm_only_bytes); - dcn3_0_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_only); - dcn3_0_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm); - dcn3_0_soc.pct_ideal_dram_sdp_bw_after_urgent_vm_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_vm_only); - dcn3_0_soc.max_avg_sdp_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_sdp_bw_use_normal_percent); - dcn3_0_soc.max_avg_dram_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_dram_bw_use_normal_percent); - dcn3_0_soc.writeback_latency_us = - fixed16_to_double_to_cpu(bb->writeback_latency_us); - dcn3_0_soc.ideal_dram_bw_after_urgent_percent = - fixed16_to_double_to_cpu(bb->ideal_dram_bw_after_urgent_percent); - dcn3_0_soc.max_request_size_bytes = - le32_to_cpu(bb->max_request_size_bytes); - dcn3_0_soc.dram_channel_width_bytes = - le32_to_cpu(bb->dram_channel_width_bytes); - dcn3_0_soc.fabric_datapath_to_dcn_data_return_bytes = - le32_to_cpu(bb->fabric_datapath_to_dcn_data_return_bytes); - dcn3_0_soc.dcn_downspread_percent = - fixed16_to_double_to_cpu(bb->dcn_downspread_percent); - dcn3_0_soc.downspread_percent = - fixed16_to_double_to_cpu(bb->downspread_percent); - dcn3_0_soc.dram_page_open_time_ns = - fixed16_to_double_to_cpu(bb->dram_page_open_time_ns); - dcn3_0_soc.dram_rw_turnaround_time_ns = - fixed16_to_double_to_cpu(bb->dram_rw_turnaround_time_ns); - dcn3_0_soc.dram_return_buffer_per_channel_bytes = - le32_to_cpu(bb->dram_return_buffer_per_channel_bytes); - dcn3_0_soc.round_trip_ping_latency_dcfclk_cycles = - le32_to_cpu(bb->round_trip_ping_latency_dcfclk_cycles); - dcn3_0_soc.urgent_out_of_order_return_per_channel_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_bytes); - dcn3_0_soc.channel_interleave_bytes = - le32_to_cpu(bb->channel_interleave_bytes); - dcn3_0_soc.num_banks = - le32_to_cpu(bb->num_banks); - dcn3_0_soc.num_chans = - le32_to_cpu(bb->num_chans); - dcn3_0_soc.gpuvm_min_page_size_bytes = - le32_to_cpu(bb->vmm_page_size_bytes); - dcn3_0_soc.dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->dram_clock_change_latency_us); - dcn3_0_soc.writeback_dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->writeback_dram_clock_change_latency_us); - dcn3_0_soc.return_bus_width_bytes = - le32_to_cpu(bb->return_bus_width_bytes); - dcn3_0_soc.dispclk_dppclk_vco_speed_mhz = - le32_to_cpu(bb->dispclk_dppclk_vco_speed_mhz); - dcn3_0_soc.xfc_bus_transport_time_us = - le32_to_cpu(bb->xfc_bus_transport_time_us); - dcn3_0_soc.xfc_xbuf_latency_tolerance_us = - le32_to_cpu(bb->xfc_xbuf_latency_tolerance_us); - dcn3_0_soc.use_urgent_burst_bw = - le32_to_cpu(bb->use_urgent_burst_bw); - dcn3_0_soc.num_states = - le32_to_cpu(bb->num_states); - - for (i = 0; i < dcn3_0_soc.num_states; i++) { - dcn3_0_soc.clock_limits[i].state = - le32_to_cpu(bb->clock_limits[i].state); - dcn3_0_soc.clock_limits[i].dcfclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dcfclk_mhz); - dcn3_0_soc.clock_limits[i].fabricclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].fabricclk_mhz); - dcn3_0_soc.clock_limits[i].dispclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dispclk_mhz); - dcn3_0_soc.clock_limits[i].dppclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dppclk_mhz); - dcn3_0_soc.clock_limits[i].phyclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].phyclk_mhz); - dcn3_0_soc.clock_limits[i].socclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].socclk_mhz); - dcn3_0_soc.clock_limits[i].dscclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dscclk_mhz); - dcn3_0_soc.clock_limits[i].dram_speed_mts = - fixed16_to_double_to_cpu(bb->clock_limits[i].dram_speed_mts); - } - } - loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator; loaded_ip->max_num_dpp = pool->base.pipe_count; loaded_ip->clamp_min_dcfclk = dc->config.clamp_min_dcfclk; dcn20_patch_bounding_box(dc, loaded_bb); - if (!bb && dc->ctx->dc_bios->funcs->get_soc_bb_info) { + if (dc->ctx->dc_bios->funcs->get_soc_bb_info) { struct bp_soc_bb_info bb_info = {0}; if (dc->ctx->dc_bios->funcs->get_soc_bb_info(dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) { @@ -2292,17 +2189,15 @@ static noinline void dcn30_calculate_wm_and_dlg_fp( unsigned int min_dram_speed_mts = context->bw_ctx.dml.vba.DRAMSpeed; unsigned int min_dram_speed_mts_margin = 160; - context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->dummy_pstate_table[0].dummy_pstate_latency_us; - if (context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb] == dm_dram_clock_change_unsupported) min_dram_speed_mts = dc->clk_mgr->bw_params->clk_table.entries[dc->clk_mgr->bw_params->clk_table.num_entries - 1].memclk_mhz * 16; - for (i = 3; i > 0; i--) { - if ((min_dram_speed_mts + min_dram_speed_mts_margin > dc->clk_mgr->bw_params->dummy_pstate_table[i].dram_speed_mts) && - (min_dram_speed_mts - min_dram_speed_mts_margin < dc->clk_mgr->bw_params->dummy_pstate_table[i].dram_speed_mts)) - context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->dummy_pstate_table[i].dummy_pstate_latency_us; - } + /* find largest table entry that is lower than dram speed, but lower than DPM0 still uses DPM0 */ + for (i = 3; i > 0; i--) + if (min_dram_speed_mts + min_dram_speed_mts_margin > dc->clk_mgr->bw_params->dummy_pstate_table[i].dram_speed_mts) + break; + context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->dummy_pstate_table[i].dummy_pstate_latency_us; context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_enter_plus_exit_time_us; context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_exit_time_us; } @@ -2437,16 +2332,28 @@ validate_out: return out; } -static noinline void get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts, - unsigned int *optimal_dcfclk, - unsigned int *optimal_fclk) +/* + * This must be noinline to ensure anything that deals with FP registers + * is contained within this call; previously our compiling with hard-float + * would result in fp instructions being emitted outside of the boundaries + * of the DC_FP_START/END macros, which makes sense as the compiler has no + * idea about what is wrapped and what is not + * + * This is largely just a workaround to avoid breakage introduced with 5.6, + * ideally all fp-using code should be moved into its own file, only that + * should be compiled with hard-float, and all code exported from there + * should be strictly wrapped with DC_FP_START/END + */ +static noinline void dcn30_get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts, + unsigned int *optimal_dcfclk, + unsigned int *optimal_fclk) { double bw_from_dram, bw_from_dram1, bw_from_dram2; bw_from_dram1 = uclk_mts * dcn3_0_soc.num_chans * - dcn3_0_soc.dram_channel_width_bytes * (dcn3_0_soc.max_avg_dram_bw_use_normal_percent / 100); + dcn3_0_soc.dram_channel_width_bytes * (dcn3_0_soc.max_avg_dram_bw_use_normal_percent / 100); bw_from_dram2 = uclk_mts * dcn3_0_soc.num_chans * - dcn3_0_soc.dram_channel_width_bytes * (dcn3_0_soc.max_avg_sdp_bw_use_normal_percent / 100); + dcn3_0_soc.dram_channel_width_bytes * (dcn3_0_soc.max_avg_sdp_bw_use_normal_percent / 100); bw_from_dram = (bw_from_dram1 < bw_from_dram2) ? bw_from_dram1 : bw_from_dram2; @@ -2505,7 +2412,7 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params // Calculate optimal dcfclk for each uclk for (i = 0; i < num_uclk_states; i++) { DC_FP_START(); - get_optimal_dcfclk_fclk_for_uclk(bw_params->clk_table.entries[i].memclk_mhz * 16, + dcn30_get_optimal_dcfclk_fclk_for_uclk(bw_params->clk_table.entries[i].memclk_mhz * 16, &optimal_dcfclk_for_uclk[i], NULL); DC_FP_END(); if (optimal_dcfclk_for_uclk[i] < bw_params->clk_table.entries[0].dcfclk_mhz) { @@ -2631,6 +2538,10 @@ static bool dcn30_resource_construct( dc->caps.max_cursor_size = 256; dc->caps.min_horizontal_blanking_period = 80; dc->caps.dmdata_alloc_size = 2048; + dc->caps.mall_size_per_mem_channel = 8; + /* total size = mall per channel * num channels * 1024 * 1024 */ + dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel * dc->ctx->dc_bios->vram_info.num_chans * 1048576; + dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8; dc->caps.max_slave_planes = 1; dc->caps.post_blend_color_processing = true; diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c index bdad72140cbc..b8bf6d61005b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c @@ -98,6 +98,7 @@ static const struct hw_sequencer_funcs dcn301_funcs = { .set_abm_immediate_disable = dcn21_set_abm_immediate_disable, .set_pipe = dcn21_set_pipe, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, + .set_hubp_blank = dcn30_set_hubp_blank, }; static const struct hwseq_private_funcs dcn301_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c index 35f5bf08ae96..5d4b2c60192e 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c @@ -1489,124 +1489,21 @@ static bool is_soc_bounding_box_valid(struct dc *dc) static bool init_soc_bounding_box(struct dc *dc, struct dcn301_resource_pool *pool) { - const struct gpu_info_soc_bounding_box_v1_0 *bb = dc->soc_bounding_box; struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_01_soc; struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_01_ip; DC_LOGGER_INIT(dc->ctx->logger); - if (!bb && !is_soc_bounding_box_valid(dc)) { + if (!is_soc_bounding_box_valid(dc)) { DC_LOG_ERROR("%s: not valid soc bounding box/n", __func__); return false; } - if (bb && !is_soc_bounding_box_valid(dc)) { - int i; - - dcn3_01_soc.sr_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_exit_time_us); - dcn3_01_soc.sr_enter_plus_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_enter_plus_exit_time_us); - dcn3_01_soc.urgent_latency_us = - fixed16_to_double_to_cpu(bb->urgent_latency_us); - dcn3_01_soc.urgent_latency_pixel_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_data_only_us); - dcn3_01_soc.urgent_latency_pixel_mixed_with_vm_data_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_mixed_with_vm_data_us); - dcn3_01_soc.urgent_latency_vm_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_vm_data_only_us); - dcn3_01_soc.urgent_out_of_order_return_per_channel_pixel_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_only_bytes); - dcn3_01_soc.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_and_vm_bytes); - dcn3_01_soc.urgent_out_of_order_return_per_channel_vm_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_vm_only_bytes); - dcn3_01_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_only); - dcn3_01_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm); - dcn3_01_soc.pct_ideal_dram_sdp_bw_after_urgent_vm_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_vm_only); - dcn3_01_soc.max_avg_sdp_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_sdp_bw_use_normal_percent); - dcn3_01_soc.max_avg_dram_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_dram_bw_use_normal_percent); - dcn3_01_soc.writeback_latency_us = - fixed16_to_double_to_cpu(bb->writeback_latency_us); - dcn3_01_soc.ideal_dram_bw_after_urgent_percent = - fixed16_to_double_to_cpu(bb->ideal_dram_bw_after_urgent_percent); - dcn3_01_soc.max_request_size_bytes = - le32_to_cpu(bb->max_request_size_bytes); - dcn3_01_soc.dram_channel_width_bytes = - le32_to_cpu(bb->dram_channel_width_bytes); - dcn3_01_soc.fabric_datapath_to_dcn_data_return_bytes = - le32_to_cpu(bb->fabric_datapath_to_dcn_data_return_bytes); - dcn3_01_soc.dcn_downspread_percent = - fixed16_to_double_to_cpu(bb->dcn_downspread_percent); - dcn3_01_soc.downspread_percent = - fixed16_to_double_to_cpu(bb->downspread_percent); - dcn3_01_soc.dram_page_open_time_ns = - fixed16_to_double_to_cpu(bb->dram_page_open_time_ns); - dcn3_01_soc.dram_rw_turnaround_time_ns = - fixed16_to_double_to_cpu(bb->dram_rw_turnaround_time_ns); - dcn3_01_soc.dram_return_buffer_per_channel_bytes = - le32_to_cpu(bb->dram_return_buffer_per_channel_bytes); - dcn3_01_soc.round_trip_ping_latency_dcfclk_cycles = - le32_to_cpu(bb->round_trip_ping_latency_dcfclk_cycles); - dcn3_01_soc.urgent_out_of_order_return_per_channel_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_bytes); - dcn3_01_soc.channel_interleave_bytes = - le32_to_cpu(bb->channel_interleave_bytes); - dcn3_01_soc.num_banks = - le32_to_cpu(bb->num_banks); - dcn3_01_soc.num_chans = - le32_to_cpu(bb->num_chans); - dcn3_01_soc.gpuvm_min_page_size_bytes = - le32_to_cpu(bb->vmm_page_size_bytes); - dcn3_01_soc.dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->dram_clock_change_latency_us); - dcn3_01_soc.writeback_dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->writeback_dram_clock_change_latency_us); - dcn3_01_soc.return_bus_width_bytes = - le32_to_cpu(bb->return_bus_width_bytes); - dcn3_01_soc.dispclk_dppclk_vco_speed_mhz = - le32_to_cpu(bb->dispclk_dppclk_vco_speed_mhz); - dcn3_01_soc.xfc_bus_transport_time_us = - le32_to_cpu(bb->xfc_bus_transport_time_us); - dcn3_01_soc.xfc_xbuf_latency_tolerance_us = - le32_to_cpu(bb->xfc_xbuf_latency_tolerance_us); - dcn3_01_soc.use_urgent_burst_bw = - le32_to_cpu(bb->use_urgent_burst_bw); - dcn3_01_soc.num_states = - le32_to_cpu(bb->num_states); - - for (i = 0; i < dcn3_01_soc.num_states; i++) { - dcn3_01_soc.clock_limits[i].state = - le32_to_cpu(bb->clock_limits[i].state); - dcn3_01_soc.clock_limits[i].dcfclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dcfclk_mhz); - dcn3_01_soc.clock_limits[i].fabricclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].fabricclk_mhz); - dcn3_01_soc.clock_limits[i].dispclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dispclk_mhz); - dcn3_01_soc.clock_limits[i].dppclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dppclk_mhz); - dcn3_01_soc.clock_limits[i].phyclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].phyclk_mhz); - dcn3_01_soc.clock_limits[i].socclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].socclk_mhz); - dcn3_01_soc.clock_limits[i].dscclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dscclk_mhz); - dcn3_01_soc.clock_limits[i].dram_speed_mts = - fixed16_to_double_to_cpu(bb->clock_limits[i].dram_speed_mts); - } - } - loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator; loaded_ip->max_num_dpp = pool->base.pipe_count; dcn20_patch_bounding_box(dc, loaded_bb); - if (!bb && dc->ctx->dc_bios->funcs->get_soc_bb_info) { + if (dc->ctx->dc_bios->funcs->get_soc_bb_info) { struct bp_soc_bb_info bb_info = {0}; if (dc->ctx->dc_bios->funcs->get_soc_bb_info(dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) { diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c index 808c4dcdb3ac..4b659b63f75b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c @@ -53,6 +53,8 @@ #include "dce/dce_i2c_hw.h" #include "dce/dce_panel_cntl.h" #include "dce/dmub_abm.h" +#include "dce/dmub_psr.h" +#include "clk_mgr.h" #include "hw_sequencer_private.h" #include "reg_helper.h" @@ -162,8 +164,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_02_soc = { .min_dcfclk = 500.0, /* TODO: set this to actual min DCFCLK */ .num_states = 1, - .sr_exit_time_us = 5.20, - .sr_enter_plus_exit_time_us = 9.60, + .sr_exit_time_us = 12, + .sr_enter_plus_exit_time_us = 20, .urgent_latency_us = 4.0, .urgent_latency_pixel_data_only_us = 4.0, .urgent_latency_pixel_mixed_with_vm_data_us = 4.0, @@ -190,7 +192,7 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_02_soc = { .num_banks = 8, .gpuvm_min_page_size_bytes = 4096, .hostvm_min_page_size_bytes = 4096, - .dram_clock_change_latency_us = 350, + .dram_clock_change_latency_us = 404, .dummy_pstate_latency_us = 5, .writeback_dram_clock_change_latency_us = 23.0, .return_bus_width_bytes = 64, @@ -238,6 +240,7 @@ static const struct dc_debug_options debug_defaults_diags = { .dwb_fi_phase = -1, // -1 = disable .dmub_command_table = true, .enable_tri_buf = true, + .disable_psr = true, }; enum dcn302_clk_src_array_id { @@ -1213,6 +1216,9 @@ static void dcn302_resource_destruct(struct resource_pool *pool) dce_abm_destroy(&pool->multiple_abms[i]); } + if (pool->psr != NULL) + dmub_psr_destroy(&pool->psr); + if (pool->dccg != NULL) dcn_dccg_destroy(&pool->dccg); } @@ -1224,6 +1230,165 @@ static void dcn302_destroy_resource_pool(struct resource_pool **pool) *pool = NULL; } +static void dcn302_get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts, + unsigned int *optimal_dcfclk, + unsigned int *optimal_fclk) +{ + double bw_from_dram, bw_from_dram1, bw_from_dram2; + + bw_from_dram1 = uclk_mts * dcn3_02_soc.num_chans * + dcn3_02_soc.dram_channel_width_bytes * (dcn3_02_soc.max_avg_dram_bw_use_normal_percent / 100); + bw_from_dram2 = uclk_mts * dcn3_02_soc.num_chans * + dcn3_02_soc.dram_channel_width_bytes * (dcn3_02_soc.max_avg_sdp_bw_use_normal_percent / 100); + + bw_from_dram = (bw_from_dram1 < bw_from_dram2) ? bw_from_dram1 : bw_from_dram2; + + if (optimal_fclk) + *optimal_fclk = bw_from_dram / + (dcn3_02_soc.fabric_datapath_to_dcn_data_return_bytes * (dcn3_02_soc.max_avg_sdp_bw_use_normal_percent / 100)); + + if (optimal_dcfclk) + *optimal_dcfclk = bw_from_dram / + (dcn3_02_soc.return_bus_width_bytes * (dcn3_02_soc.max_avg_sdp_bw_use_normal_percent / 100)); +} + +void dcn302_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params) +{ + unsigned int i, j; + unsigned int num_states = 0; + + unsigned int dcfclk_mhz[DC__VOLTAGE_STATES] = {0}; + unsigned int dram_speed_mts[DC__VOLTAGE_STATES] = {0}; + unsigned int optimal_uclk_for_dcfclk_sta_targets[DC__VOLTAGE_STATES] = {0}; + unsigned int optimal_dcfclk_for_uclk[DC__VOLTAGE_STATES] = {0}; + + unsigned int dcfclk_sta_targets[DC__VOLTAGE_STATES] = {694, 875, 1000, 1200}; + unsigned int num_dcfclk_sta_targets = 4; + unsigned int num_uclk_states; + + + if (dc->ctx->dc_bios->vram_info.num_chans) + dcn3_02_soc.num_chans = dc->ctx->dc_bios->vram_info.num_chans; + + if (dc->ctx->dc_bios->vram_info.dram_channel_width_bytes) + dcn3_02_soc.dram_channel_width_bytes = dc->ctx->dc_bios->vram_info.dram_channel_width_bytes; + + dcn3_02_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; + dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; + + if (bw_params->clk_table.entries[0].memclk_mhz) { + int max_dcfclk_mhz = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0, max_phyclk_mhz = 0; + + for (i = 0; i < MAX_NUM_DPM_LVL; i++) { + if (bw_params->clk_table.entries[i].dcfclk_mhz > max_dcfclk_mhz) + max_dcfclk_mhz = bw_params->clk_table.entries[i].dcfclk_mhz; + if (bw_params->clk_table.entries[i].dispclk_mhz > max_dispclk_mhz) + max_dispclk_mhz = bw_params->clk_table.entries[i].dispclk_mhz; + if (bw_params->clk_table.entries[i].dppclk_mhz > max_dppclk_mhz) + max_dppclk_mhz = bw_params->clk_table.entries[i].dppclk_mhz; + if (bw_params->clk_table.entries[i].phyclk_mhz > max_phyclk_mhz) + max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz; + } + if (!max_dcfclk_mhz) + max_dcfclk_mhz = dcn3_02_soc.clock_limits[0].dcfclk_mhz; + if (!max_dispclk_mhz) + max_dispclk_mhz = dcn3_02_soc.clock_limits[0].dispclk_mhz; + if (!max_dppclk_mhz) + max_dppclk_mhz = dcn3_02_soc.clock_limits[0].dppclk_mhz; + if (!max_phyclk_mhz) + max_phyclk_mhz = dcn3_02_soc.clock_limits[0].phyclk_mhz; + + if (max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) { + /* If max DCFCLK is greater than the max DCFCLK STA target, insert into the DCFCLK STA target array */ + dcfclk_sta_targets[num_dcfclk_sta_targets] = max_dcfclk_mhz; + num_dcfclk_sta_targets++; + } else if (max_dcfclk_mhz < dcfclk_sta_targets[num_dcfclk_sta_targets-1]) { + /* If max DCFCLK is less than the max DCFCLK STA target, cap values and remove duplicates */ + for (i = 0; i < num_dcfclk_sta_targets; i++) { + if (dcfclk_sta_targets[i] > max_dcfclk_mhz) { + dcfclk_sta_targets[i] = max_dcfclk_mhz; + break; + } + } + /* Update size of array since we "removed" duplicates */ + num_dcfclk_sta_targets = i + 1; + } + + num_uclk_states = bw_params->clk_table.num_entries; + + /* Calculate optimal dcfclk for each uclk */ + for (i = 0; i < num_uclk_states; i++) { + dcn302_get_optimal_dcfclk_fclk_for_uclk(bw_params->clk_table.entries[i].memclk_mhz * 16, + &optimal_dcfclk_for_uclk[i], NULL); + if (optimal_dcfclk_for_uclk[i] < bw_params->clk_table.entries[0].dcfclk_mhz) { + optimal_dcfclk_for_uclk[i] = bw_params->clk_table.entries[0].dcfclk_mhz; + } + } + + /* Calculate optimal uclk for each dcfclk sta target */ + for (i = 0; i < num_dcfclk_sta_targets; i++) { + for (j = 0; j < num_uclk_states; j++) { + if (dcfclk_sta_targets[i] < optimal_dcfclk_for_uclk[j]) { + optimal_uclk_for_dcfclk_sta_targets[i] = + bw_params->clk_table.entries[j].memclk_mhz * 16; + break; + } + } + } + + i = 0; + j = 0; + /* create the final dcfclk and uclk table */ + while (i < num_dcfclk_sta_targets && j < num_uclk_states && num_states < DC__VOLTAGE_STATES) { + if (dcfclk_sta_targets[i] < optimal_dcfclk_for_uclk[j] && i < num_dcfclk_sta_targets) { + dcfclk_mhz[num_states] = dcfclk_sta_targets[i]; + dram_speed_mts[num_states++] = optimal_uclk_for_dcfclk_sta_targets[i++]; + } else { + if (j < num_uclk_states && optimal_dcfclk_for_uclk[j] <= max_dcfclk_mhz) { + dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j]; + dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16; + } else { + j = num_uclk_states; + } + } + } + + while (i < num_dcfclk_sta_targets && num_states < DC__VOLTAGE_STATES) { + dcfclk_mhz[num_states] = dcfclk_sta_targets[i]; + dram_speed_mts[num_states++] = optimal_uclk_for_dcfclk_sta_targets[i++]; + } + + while (j < num_uclk_states && num_states < DC__VOLTAGE_STATES && + optimal_dcfclk_for_uclk[j] <= max_dcfclk_mhz) { + dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j]; + dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16; + } + + dcn3_02_soc.num_states = num_states; + for (i = 0; i < dcn3_02_soc.num_states; i++) { + dcn3_02_soc.clock_limits[i].state = i; + dcn3_02_soc.clock_limits[i].dcfclk_mhz = dcfclk_mhz[i]; + dcn3_02_soc.clock_limits[i].fabricclk_mhz = dcfclk_mhz[i]; + dcn3_02_soc.clock_limits[i].dram_speed_mts = dram_speed_mts[i]; + + /* Fill all states with max values of all other clocks */ + dcn3_02_soc.clock_limits[i].dispclk_mhz = max_dispclk_mhz; + dcn3_02_soc.clock_limits[i].dppclk_mhz = max_dppclk_mhz; + dcn3_02_soc.clock_limits[i].phyclk_mhz = max_phyclk_mhz; + dcn3_02_soc.clock_limits[i].dtbclk_mhz = dcn3_02_soc.clock_limits[0].dtbclk_mhz; + /* These clocks cannot come from bw_params, always fill from dcn3_02_soc[1] */ + /* FCLK, PHYCLK_D18, SOCCLK, DSCCLK */ + dcn3_02_soc.clock_limits[i].phyclk_d18_mhz = dcn3_02_soc.clock_limits[0].phyclk_d18_mhz; + dcn3_02_soc.clock_limits[i].socclk_mhz = dcn3_02_soc.clock_limits[0].socclk_mhz; + dcn3_02_soc.clock_limits[i].dscclk_mhz = dcn3_02_soc.clock_limits[0].dscclk_mhz; + } + /* re-init DML with updated bb */ + dml_init_instance(&dc->dml, &dcn3_02_soc, &dcn3_02_ip, DML_PROJECT_DCN30); + if (dc->current_state) + dml_init_instance(&dc->current_state->bw_ctx.dml, &dcn3_02_soc, &dcn3_02_ip, DML_PROJECT_DCN30); + } +} + static struct resource_funcs dcn302_res_pool_funcs = { .destroy = dcn302_destroy_resource_pool, .link_enc_create = dcn302_link_encoder_create, @@ -1240,7 +1405,7 @@ static struct resource_funcs dcn302_res_pool_funcs = { .find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link, .acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut, .release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut, - .update_bw_bounding_box = dcn30_update_bw_bounding_box, + .update_bw_bounding_box = dcn302_update_bw_bounding_box, .patch_unknown_plane_state = dcn20_patch_unknown_plane_state, }; @@ -1311,7 +1476,10 @@ static bool dcn302_resource_construct( dc->caps.max_cursor_size = 256; dc->caps.min_horizontal_blanking_period = 80; dc->caps.dmdata_alloc_size = 2048; - + dc->caps.mall_size_per_mem_channel = 4; + /* total size = mall per channel * num channels * 1024 * 1024 */ + dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel * dc->ctx->dc_bios->vram_info.num_chans * 1048576; + dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8; dc->caps.max_slave_planes = 1; dc->caps.post_blend_color_processing = true; dc->caps.force_dp_tps4_for_cp2520 = true; @@ -1354,8 +1522,6 @@ static bool dcn302_resource_construct( if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV) dc->debug = debug_defaults_drv; - else if (dc->ctx->dce_environment == DCE_ENV_FPGA_MAXIMUS) - dc->debug = debug_defaults_diags; else dc->debug = debug_defaults_diags; @@ -1469,6 +1635,14 @@ static bool dcn302_resource_construct( } pool->timing_generator_count = i; + /* PSR */ + pool->psr = dmub_psr_create(ctx); + if (pool->psr == NULL) { + dm_error("DC: failed to create psr!\n"); + BREAK_TO_DEBUGGER(); + goto create_fail; + } + /* ABMs */ for (i = 0; i < pool->res_cap->num_timing_generator; i++) { pool->multiple_abms[i] = dmub_abm_create(ctx, &abm_regs[i], &abm_shift, &abm_mask); diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h index 71f7deed18e3..42d2c73e30bc 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h +++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h @@ -30,4 +30,6 @@ struct resource_pool *dcn302_create_resource_pool(const struct dc_init_data *init_data, struct dc *dc); +void dcn302_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params); + #endif /* _DCN302_RESOURCE_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dm_cp_psp.h b/drivers/gpu/drm/amd/display/dc/dm_cp_psp.h index 5da7677627a1..cac0b2c0d31b 100644 --- a/drivers/gpu/drm/amd/display/dc/dm_cp_psp.h +++ b/drivers/gpu/drm/amd/display/dc/dm_cp_psp.h @@ -30,9 +30,10 @@ struct dc_link; struct cp_psp_stream_config { uint8_t otg_inst; - uint8_t link_enc_inst; - uint8_t stream_enc_inst; - uint8_t mst_supported; + uint8_t dig_be; + uint8_t dig_fe; + uint8_t assr_enabled; + uint8_t mst_enabled; void *dm_stream_ctx; bool dpms_off; }; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c index 45f028986a8d..0f3f510fd83b 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c @@ -3138,7 +3138,7 @@ static void CalculateFlipSchedule( 4.0 * (TimeForFetchingMetaPTEImmediateFlip / LineTime + 0.125), 1) / 4.0; - if ((GPUVMEnable == true || DCCEnable == true)) { + if ((GPUVMEnable || DCCEnable)) { mode_lib->vba.ImmediateFlipBW[0] = BandwidthAvailableForImmediateFlip * ImmediateFlipBytes / TotImmediateFlipBytes; TimeForFetchingRowInVBlankImmediateFlip = dml_max( @@ -4168,10 +4168,11 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l for (i = 0; i <= mode_lib->vba.soc.num_states; i++) { locals->DIOSupport[i] = true; for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) { - if (locals->OutputBppPerState[i][k] == BPP_INVALID - || (mode_lib->vba.OutputFormat[k] == dm_420 + if (!mode_lib->vba.skip_dio_check[k] + && (locals->OutputBppPerState[i][k] == BPP_INVALID + || (mode_lib->vba.OutputFormat[k] == dm_420 && mode_lib->vba.Interlace[k] == true - && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true)) { + && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true))) { locals->DIOSupport[i] = false; } } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c index 80170f9721ce..210c96cd5b03 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c @@ -3263,6 +3263,7 @@ static void CalculateFlipSchedule( static unsigned int TruncToValidBPP( double DecimalBPP, + double DesiredBPP, bool DSCEnabled, enum output_encoder_class Output, enum output_format_class Format, @@ -3270,31 +3271,31 @@ static unsigned int TruncToValidBPP( { if (Output == dm_hdmi) { if (Format == dm_420) { - if (DecimalBPP >= 18) + if (DecimalBPP >= 18 && (DesiredBPP == 0 || DesiredBPP == 18)) return 18; - else if (DecimalBPP >= 15) + else if (DecimalBPP >= 15 && (DesiredBPP == 0 || DesiredBPP == 15)) return 15; - else if (DecimalBPP >= 12) + else if (DecimalBPP >= 12 && (DesiredBPP == 0 || DesiredBPP == 12)) return 12; else return BPP_INVALID; } else if (Format == dm_444) { - if (DecimalBPP >= 36) + if (DecimalBPP >= 36 && (DesiredBPP == 0 || DesiredBPP == 36)) return 36; - else if (DecimalBPP >= 30) + else if (DecimalBPP >= 30 && (DesiredBPP == 0 || DesiredBPP == 30)) return 30; - else if (DecimalBPP >= 24) + else if (DecimalBPP >= 24 && (DesiredBPP == 0 || DesiredBPP == 24)) return 24; - else if (DecimalBPP >= 18) + else if (DecimalBPP >= 18 && (DesiredBPP == 0 || DesiredBPP == 18)) return 18; else return BPP_INVALID; } else { - if (DecimalBPP / 1.5 >= 24) + if (DecimalBPP / 1.5 >= 24 && (DesiredBPP == 0 || DesiredBPP == 24)) return 24; - else if (DecimalBPP / 1.5 >= 20) + else if (DecimalBPP / 1.5 >= 20 && (DesiredBPP == 0 || DesiredBPP == 20)) return 20; - else if (DecimalBPP / 1.5 >= 16) + else if (DecimalBPP / 1.5 >= 16 && (DesiredBPP == 0 || DesiredBPP == 16)) return 16; else return BPP_INVALID; @@ -3302,53 +3303,86 @@ static unsigned int TruncToValidBPP( } else { if (DSCEnabled) { if (Format == dm_420) { - if (DecimalBPP < 6) - return BPP_INVALID; - else if (DecimalBPP >= 1.5 * DSCInputBitPerComponent - 1 / 16) - return 1.5 * DSCInputBitPerComponent - 1 / 16; - else - return dml_floor(16 * DecimalBPP, 1) / 16; + if (DesiredBPP == 0) { + if (DecimalBPP < 6) + return BPP_INVALID; + else if (DecimalBPP >= 1.5 * DSCInputBitPerComponent - 1.0 / 16.0) + return 1.5 * DSCInputBitPerComponent - 1.0 / 16.0; + else + return dml_floor(16 * DecimalBPP, 1) / 16.0; + } else { + if (DecimalBPP < 6 + || DesiredBPP < 6 + || DesiredBPP > 1.5 * DSCInputBitPerComponent - 1.0 / 16.0 + || DecimalBPP < DesiredBPP) { + return BPP_INVALID; + } else { + return DesiredBPP; + } + } } else if (Format == dm_n422) { - if (DecimalBPP < 7) - return BPP_INVALID; - else if (DecimalBPP >= 2 * DSCInputBitPerComponent - 1 / 16) - return 2 * DSCInputBitPerComponent - 1 / 16; - else - return dml_floor(16 * DecimalBPP, 1) / 16; + if (DesiredBPP == 0) { + if (DecimalBPP < 7) + return BPP_INVALID; + else if (DecimalBPP >= 2 * DSCInputBitPerComponent - 1.0 / 16.0) + return 2 * DSCInputBitPerComponent - 1.0 / 16.0; + else + return dml_floor(16 * DecimalBPP, 1) / 16.0; + } else { + if (DecimalBPP < 7 + || DesiredBPP < 7 + || DesiredBPP > 2 * DSCInputBitPerComponent - 1.0 / 16.0 + || DecimalBPP < DesiredBPP) { + return BPP_INVALID; + } else { + return DesiredBPP; + } + } } else { - if (DecimalBPP < 8) - return BPP_INVALID; - else if (DecimalBPP >= 3 * DSCInputBitPerComponent - 1 / 16) - return 3 * DSCInputBitPerComponent - 1 / 16; - else - return dml_floor(16 * DecimalBPP, 1) / 16; + if (DesiredBPP == 0) { + if (DecimalBPP < 8) + return BPP_INVALID; + else if (DecimalBPP >= 3 * DSCInputBitPerComponent - 1.0 / 16.0) + return 3 * DSCInputBitPerComponent - 1.0 / 16.0; + else + return dml_floor(16 * DecimalBPP, 1) / 16.0; + } else { + if (DecimalBPP < 8 + || DesiredBPP < 8 + || DesiredBPP > 3 * DSCInputBitPerComponent - 1.0 / 16.0 + || DecimalBPP < DesiredBPP) { + return BPP_INVALID; + } else { + return DesiredBPP; + } + } } } else if (Format == dm_420) { - if (DecimalBPP >= 18) + if (DecimalBPP >= 18 && (DesiredBPP == 0 || DesiredBPP == 18)) return 18; - else if (DecimalBPP >= 15) + else if (DecimalBPP >= 15 && (DesiredBPP == 0 || DesiredBPP == 15)) return 15; - else if (DecimalBPP >= 12) + else if (DecimalBPP >= 12 && (DesiredBPP == 0 || DesiredBPP == 12)) return 12; else return BPP_INVALID; } else if (Format == dm_s422 || Format == dm_n422) { - if (DecimalBPP >= 24) + if (DecimalBPP >= 24 && (DesiredBPP == 0 || DesiredBPP == 24)) return 24; - else if (DecimalBPP >= 20) + else if (DecimalBPP >= 20 && (DesiredBPP == 0 || DesiredBPP == 20)) return 20; - else if (DecimalBPP >= 16) + else if (DecimalBPP >= 16 && (DesiredBPP == 0 || DesiredBPP == 16)) return 16; else return BPP_INVALID; } else { - if (DecimalBPP >= 36) + if (DecimalBPP >= 36 && (DesiredBPP == 0 || DesiredBPP == 36)) return 36; - else if (DecimalBPP >= 30) + else if (DecimalBPP >= 30 && (DesiredBPP == 0 || DesiredBPP == 30)) return 30; - else if (DecimalBPP >= 24) + else if (DecimalBPP >= 24 && (DesiredBPP == 0 || DesiredBPP == 24)) return 24; - else if (DecimalBPP >= 18) + else if (DecimalBPP >= 18 && (DesiredBPP == 0 || DesiredBPP == 18)) return 18; else return BPP_INVALID; @@ -4137,6 +4171,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode locals->RequiresFEC[i][k] = 0; locals->OutputBppPerState[i][k] = TruncToValidBPP( dml_min(600.0, mode_lib->vba.PHYCLKPerState[i]) / mode_lib->vba.PixelClockBackEnd[k] * 24, + mode_lib->vba.ForcedOutputLinkBPP[k], false, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4153,6 +4188,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.Outbpp = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * 270.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], false, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4160,6 +4196,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.OutbppDSC = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 270.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], true, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4182,6 +4219,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.Outbpp = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * 540.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], false, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4189,6 +4227,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.OutbppDSC = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 540.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], true, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4213,6 +4252,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.Outbpp = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * 810.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], false, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4220,6 +4260,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.OutbppDSC = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 810.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], true, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4248,10 +4289,11 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode for (i = 0; i <= mode_lib->vba.soc.num_states; i++) { locals->DIOSupport[i] = true; for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) { - if (locals->OutputBppPerState[i][k] == BPP_INVALID - || (mode_lib->vba.OutputFormat[k] == dm_420 + if (!mode_lib->vba.skip_dio_check[k] + && (locals->OutputBppPerState[i][k] == BPP_INVALID + || (mode_lib->vba.OutputFormat[k] == dm_420 && mode_lib->vba.Interlace[k] == true - && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true)) { + && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true))) { locals->DIOSupport[i] = false; } } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c index 86ff24dffc3e..398210d1af34 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c @@ -4257,10 +4257,11 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l for (i = 0; i <= mode_lib->vba.soc.num_states; i++) { locals->DIOSupport[i] = true; for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) { - if (locals->OutputBppPerState[i][k] == BPP_INVALID - || (mode_lib->vba.OutputFormat[k] == dm_420 + if (!mode_lib->vba.skip_dio_check[k] + && (locals->OutputBppPerState[i][k] == BPP_INVALID + || (mode_lib->vba.OutputFormat[k] == dm_420 && mode_lib->vba.Interlace[k] == true - && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true)) { + && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true))) { locals->DIOSupport[i] = false; } } @@ -5121,48 +5122,48 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l for (j = 0; j < 2; j++) { enum dm_validation_status status = DML_VALIDATION_OK; - if (mode_lib->vba.ScaleRatioAndTapsSupport != true) { + if (!mode_lib->vba.ScaleRatioAndTapsSupport) { status = DML_FAIL_SCALE_RATIO_TAP; - } else if (mode_lib->vba.SourceFormatPixelAndScanSupport != true) { + } else if (!mode_lib->vba.SourceFormatPixelAndScanSupport) { status = DML_FAIL_SOURCE_PIXEL_FORMAT; - } else if (locals->ViewportSizeSupport[i][0] != true) { + } else if (!locals->ViewportSizeSupport[i][0]) { status = DML_FAIL_VIEWPORT_SIZE; - } else if (locals->DIOSupport[i] != true) { + } else if (!locals->DIOSupport[i]) { status = DML_FAIL_DIO_SUPPORT; - } else if (locals->NotEnoughDSCUnits[i] != false) { + } else if (locals->NotEnoughDSCUnits[i]) { status = DML_FAIL_NOT_ENOUGH_DSC; - } else if (locals->DSCCLKRequiredMoreThanSupported[i] != false) { + } else if (locals->DSCCLKRequiredMoreThanSupported[i]) { status = DML_FAIL_DSC_CLK_REQUIRED; - } else if (locals->ROBSupport[i][0] != true) { + } else if (!locals->ROBSupport[i][0]) { status = DML_FAIL_REORDERING_BUFFER; - } else if (locals->DISPCLK_DPPCLK_Support[i][j] != true) { + } else if (!locals->DISPCLK_DPPCLK_Support[i][j]) { status = DML_FAIL_DISPCLK_DPPCLK; - } else if (locals->TotalAvailablePipesSupport[i][j] != true) { + } else if (!locals->TotalAvailablePipesSupport[i][j]) { status = DML_FAIL_TOTAL_AVAILABLE_PIPES; - } else if (mode_lib->vba.NumberOfOTGSupport != true) { + } else if (!mode_lib->vba.NumberOfOTGSupport) { status = DML_FAIL_NUM_OTG; - } else if (mode_lib->vba.WritebackModeSupport != true) { + } else if (!mode_lib->vba.WritebackModeSupport) { status = DML_FAIL_WRITEBACK_MODE; - } else if (mode_lib->vba.WritebackLatencySupport != true) { + } else if (!mode_lib->vba.WritebackLatencySupport) { status = DML_FAIL_WRITEBACK_LATENCY; - } else if (mode_lib->vba.WritebackScaleRatioAndTapsSupport != true) { + } else if (!mode_lib->vba.WritebackScaleRatioAndTapsSupport) { status = DML_FAIL_WRITEBACK_SCALE_RATIO_TAP; - } else if (mode_lib->vba.CursorSupport != true) { + } else if (!mode_lib->vba.CursorSupport) { status = DML_FAIL_CURSOR_SUPPORT; - } else if (mode_lib->vba.PitchSupport != true) { + } else if (!mode_lib->vba.PitchSupport) { status = DML_FAIL_PITCH_SUPPORT; - } else if (locals->TotalVerticalActiveBandwidthSupport[i][0] != true) { + } else if (!locals->TotalVerticalActiveBandwidthSupport[i][0]) { status = DML_FAIL_TOTAL_V_ACTIVE_BW; - } else if (locals->PTEBufferSizeNotExceeded[i][j] != true) { + } else if (!locals->PTEBufferSizeNotExceeded[i][j]) { status = DML_FAIL_PTE_BUFFER_SIZE; - } else if (mode_lib->vba.NonsupportedDSCInputBPC != false) { + } else if (mode_lib->vba.NonsupportedDSCInputBPC) { status = DML_FAIL_DSC_INPUT_BPC; - } else if ((mode_lib->vba.HostVMEnable != false - && locals->ImmediateFlipSupportedForState[i][j] != true)) { + } else if ((mode_lib->vba.HostVMEnable + && !locals->ImmediateFlipSupportedForState[i][j])) { status = DML_FAIL_HOST_VM_IMMEDIATE_FLIP; - } else if (locals->PrefetchSupported[i][j] != true) { + } else if (!locals->PrefetchSupported[i][j]) { status = DML_FAIL_PREFETCH_SUPPORT; - } else if (locals->VRatioInPrefetchSupported[i][j] != true) { + } else if (!locals->VRatioInPrefetchSupported[i][j]) { status = DML_FAIL_V_RATIO_PREFETCH; } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c index 319dec59bcd1..bc07082c1357 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c @@ -1219,13 +1219,13 @@ static bool CalculatePrefetchSchedule( dml_print("DML: prefetch_bw_equ: %f\n", prefetch_bw_equ); if (prefetch_bw_equ > 0) { - if (GPUVMEnable == true) { + if (GPUVMEnable) { Tvm_equ = dml_max3(*Tno_bw + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / prefetch_bw_equ, Tvm_trips, LineTime / 4); } else { Tvm_equ = LineTime / 4; } - if ((GPUVMEnable == true || myPipe->DCCEnable == true)) { + if ((GPUVMEnable || myPipe->DCCEnable)) { Tr0_equ = dml_max4( (MetaRowByte + PixelPTEBytesPerRow * HostVMInefficiencyFactor) / prefetch_bw_equ, Tr0_trips, @@ -4263,7 +4263,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l for (i = 0; i < v->soc.num_states; i++) { v->DIOSupport[i] = true; for (k = 0; k <= v->NumberOfActivePlanes - 1; k++) { - if (v->BlendingAndTiming[k] == k && (v->Output[k] == dm_dp || v->Output[k] == dm_edp || v->Output[k] == dm_hdmi) + if (!v->skip_dio_check[k] && v->BlendingAndTiming[k] == k && (v->Output[k] == dm_dp || v->Output[k] == dm_edp || v->Output[k] == dm_hdmi) && (v->OutputBppPerState[i][k] == 0 || (v->OutputFormat[k] == dm_420 && v->Interlace[k] == true && v->ProgressiveToInterlaceUnitInOPP == true))) { v->DIOSupport[i] = false; @@ -5558,7 +5558,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( } } - if (mode_lib->vba.MinActiveDRAMClockChangeMargin > 0 && PrefetchMode == 0) { + if (mode_lib->vba.MinActiveDRAMClockChangeMargin > 0) { *DRAMClockChangeSupport = dm_dram_clock_change_vactive; } else if (((mode_lib->vba.SynchronizedVBlank == true || mode_lib->vba.TotalNumberOfActiveOTG == 1 || SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank > 0) && PrefetchMode == 0)) { *DRAMClockChangeSupport = dm_dram_clock_change_vblank; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c index 5b5916b5bc71..0f14f205ebe5 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c @@ -165,8 +165,8 @@ static void handle_det_buf_split(struct display_mode_lib *mode_lib, unsigned int swath_bytes_c = 0; unsigned int full_swath_bytes_packed_l = 0; unsigned int full_swath_bytes_packed_c = 0; - bool req128_l = 0; - bool req128_c = 0; + bool req128_l = false; + bool req128_c = false; bool surf_linear = (pipe_src_param.sw_mode == dm_sw_linear); bool surf_vert = (pipe_src_param.source_scan == dm_vert); unsigned int log2_swath_height_l = 0; @@ -191,37 +191,37 @@ static void handle_det_buf_split(struct display_mode_lib *mode_lib, total_swath_bytes = 2 * full_swath_bytes_packed_l; if (total_swath_bytes <= detile_buf_size_in_bytes) { //full 256b request - req128_l = 0; - req128_c = 0; + req128_l = false; + req128_c = false; swath_bytes_l = full_swath_bytes_packed_l; swath_bytes_c = full_swath_bytes_packed_c; } else if (!rq_param->yuv420) { - req128_l = 1; - req128_c = 0; + req128_l = true; + req128_c = false; swath_bytes_c = full_swath_bytes_packed_c; swath_bytes_l = full_swath_bytes_packed_l / 2; } else if ((double)full_swath_bytes_packed_l / (double)full_swath_bytes_packed_c < 1.5) { - req128_l = 0; - req128_c = 1; + req128_l = false; + req128_c = true; swath_bytes_l = full_swath_bytes_packed_l; swath_bytes_c = full_swath_bytes_packed_c / 2; total_swath_bytes = 2 * swath_bytes_l + 2 * swath_bytes_c; if (total_swath_bytes > detile_buf_size_in_bytes) { - req128_l = 1; + req128_l = true; swath_bytes_l = full_swath_bytes_packed_l / 2; } } else { - req128_l = 1; - req128_c = 0; + req128_l = true; + req128_c = false; swath_bytes_l = full_swath_bytes_packed_l/2; swath_bytes_c = full_swath_bytes_packed_c; total_swath_bytes = 2 * swath_bytes_l + 2 * swath_bytes_c; if (total_swath_bytes > detile_buf_size_in_bytes) { - req128_c = 1; + req128_c = true; swath_bytes_c = full_swath_bytes_packed_c/2; } } @@ -1006,8 +1006,8 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib, double min_dst_y_ttu_vblank = 0; unsigned int dlg_vblank_start = 0; - bool dual_plane = 0; - bool mode_422 = 0; + bool dual_plane = false; + bool mode_422 = false; unsigned int access_dir = 0; unsigned int vp_height_l = 0; unsigned int vp_width_l = 0; @@ -1021,7 +1021,7 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib, double hratio_c = 0; double vratio_l = 0; double vratio_c = 0; - bool scl_enable = 0; + bool scl_enable = false; double line_time_in_us = 0; // double vinit_l; @@ -1156,7 +1156,7 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib, // Source // dcc_en = src.dcc; dual_plane = is_dual_plane((enum source_format_class)(src->source_format)); - mode_422 = 0; // TODO + mode_422 = false; // TODO access_dir = (src->source_scan == dm_vert); // vp access direction: horizontal or vertical accessed vp_height_l = src->viewport_height; vp_width_l = src->viewport_width; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h index dd0c3b1780d7..0c5128187e08 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h @@ -297,6 +297,7 @@ struct _vcs_dpi_display_output_params_st { int num_active_wb; int output_bpc; int output_type; + int is_virtual; int output_format; int dsc_slices; int max_audio_sample_rate; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c index c9fbb33f05a3..bc0485a59018 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c @@ -451,6 +451,8 @@ static void fetch_pipe_params(struct display_mode_lib *mode_lib) dout->output_bpp; mode_lib->vba.Output[mode_lib->vba.NumberOfActivePlanes] = (enum output_encoder_class) (dout->output_type); + mode_lib->vba.skip_dio_check[mode_lib->vba.NumberOfActivePlanes] = + dout->is_virtual; if (!dout->dsc_enable) mode_lib->vba.ForcedOutputLinkBPP[mode_lib->vba.NumberOfActivePlanes] = dout->output_bpp; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h index 3529fedc4c52..025aa5bd8ea0 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h @@ -340,6 +340,7 @@ struct vba_vars_st { unsigned int DSCInputBitPerComponent[DC__NUM_DPP__MAX]; enum output_format_class OutputFormat[DC__NUM_DPP__MAX]; enum output_encoder_class Output[DC__NUM_DPP__MAX]; + bool skip_dio_check[DC__NUM_DPP__MAX]; unsigned int BlendingAndTiming[DC__NUM_DPP__MAX]; bool SynchronizedVBlank; unsigned int NumberOfCursors[DC__NUM_DPP__MAX]; diff --git a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.c b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.c index df68430aeb0c..c6e28f6bf1a2 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.c @@ -28,6 +28,7 @@ */ #include "dm_services.h" +#include "hw_factory_diag.h" #include "include/gpio_types.h" #include "../hw_factory.h" diff --git a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.h b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.h index 8a74f6adb8ee..bf68eb1d9a1d 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.h +++ b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.h @@ -26,6 +26,8 @@ #ifndef __DAL_HW_FACTORY_DIAG_FPGA_H__ #define __DAL_HW_FACTORY_DIAG_FPGA_H__ +struct hw_factory; + /* Initialize HW factory function pointers and pin info */ void dal_hw_factory_diag_fpga_init(struct hw_factory *factory); diff --git a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_translate_diag.c b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_translate_diag.c index bf9068846927..e5138a5a8eb5 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_translate_diag.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_translate_diag.c @@ -24,6 +24,7 @@ */ #include "dm_services.h" +#include "hw_translate_diag.h" #include "include/gpio_types.h" #include "../hw_translate.h" diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c index 1ae153eab31d..7a8cec2d7a90 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c @@ -107,13 +107,12 @@ static enum gpio_result set_config( msleep(3); } } else { - uint32_t reg2; uint32_t sda_pd_dis = 0; uint32_t scl_pd_dis = 0; - reg2 = REG_GET_2(gpio.MASK_reg, - DC_GPIO_SDA_PD_DIS, &sda_pd_dis, - DC_GPIO_SCL_PD_DIS, &scl_pd_dis); + REG_GET_2(gpio.MASK_reg, + DC_GPIO_SDA_PD_DIS, &sda_pd_dis, + DC_GPIO_SCL_PD_DIS, &scl_pd_dis); if (sda_pd_dis) { REG_SET(gpio.MASK_reg, regval, diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c index da73bfb3cacd..92c65d2fa7d7 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c @@ -119,17 +119,3 @@ bool dal_hw_factory_init( return false; } } - -void dal_hw_factory_destroy( - struct dc_context *ctx, - struct hw_factory **factory) -{ - if (!factory || !*factory) { - BREAK_TO_DEBUGGER(); - return; - } - - kfree(*factory); - - *factory = NULL; -} diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h index 2d77eac66cb0..8efa1b80546d 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h +++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h @@ -333,6 +333,7 @@ struct pipe_ctx { union pipe_update_flags update_flags; struct dwbc *dwbc; struct mcif_wb *mcif_wb; + bool vtp_locked; }; struct resource_context { diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h index ffd37696b6b9..316301fc1e30 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h @@ -309,9 +309,9 @@ static inline bool should_set_clock(bool safe_to_lower, int calc_clk, int cur_cl static inline bool should_update_pstate_support(bool safe_to_lower, bool calc_support, bool cur_support) { if (cur_support != calc_support) { - if (calc_support == true && safe_to_lower) + if (calc_support && safe_to_lower) return true; - else if (calc_support == false && !safe_to_lower) + else if (!calc_support && !safe_to_lower) return true; } diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h index 69d9fbfb4bec..cd1c0dc32bf8 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h @@ -74,6 +74,16 @@ struct dmcu_funcs { bool (*is_dmcu_initialized)(struct dmcu *dmcu); bool (*lock_phy)(struct dmcu *dmcu); bool (*unlock_phy)(struct dmcu *dmcu); + bool (*send_edid_cea)(struct dmcu *dmcu, + int offset, + int total_length, + uint8_t *data, + int length); + bool (*recv_amd_vsdb)(struct dmcu *dmcu, + int *version, + int *min_frame_rate, + int *max_frame_rate); + bool (*recv_edid_cea_ack)(struct dmcu *dmcu, int *offset); }; #endif diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h index f7632fe25976..754832d216fd 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h @@ -190,6 +190,7 @@ struct timing_generator_funcs { void (*set_blank)(struct timing_generator *tg, bool enable_blanking); bool (*is_blanked)(struct timing_generator *tg); + bool (*is_locked)(struct timing_generator *tg); void (*set_overscan_blank_color) (struct timing_generator *tg, const struct tg_color *color); void (*set_blank_color)(struct timing_generator *tg, const struct tg_color *color); void (*set_colors)(struct timing_generator *tg, diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h index 62804dc7b698..0586ab2ffd6a 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h @@ -54,6 +54,7 @@ struct hw_sequencer_funcs { /* Embedded Display Related */ void (*edp_power_control)(struct dc_link *link, bool enable); void (*edp_wait_for_hpd_ready)(struct dc_link *link, bool power_up); + void (*edp_wait_for_T12)(struct dc_link *link); /* Pipe Programming Related */ void (*init_hw)(struct dc *dc); @@ -217,6 +218,9 @@ struct hw_sequencer_funcs { /* Idle Optimization Related */ bool (*apply_idle_power_optimizations)(struct dc *dc, bool enable); + bool (*does_plane_fit_in_mall)(struct dc *dc, struct dc_plane_state *plane, + struct dc_cursor_attributes *cursor_attr); + bool (*is_abm_supported)(struct dc *dc, struct dc_state *context, struct dc_stream_state *stream); @@ -227,6 +231,10 @@ struct hw_sequencer_funcs { enum dc_color_depth color_depth, const struct tg_color *solid_color, int width, int height, int offset); + + void (*set_hubp_blank)(const struct dc *dc, + struct pipe_ctx *pipe_ctx, + bool blank_enable); }; void color_space_to_black_color( diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c b/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c index f956b3bde680..34f43cb650f8 100644 --- a/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c +++ b/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c @@ -58,6 +58,18 @@ enum dc_irq_source to_dal_irq_source_dcn10( return DC_IRQ_SOURCE_VBLANK5; case DCN_1_0__SRCID__DC_D6_OTG_VSTARTUP: return DC_IRQ_SOURCE_VBLANK6; + case DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC1_VLINE0; + case DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC2_VLINE0; + case DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC3_VLINE0; + case DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC4_VLINE0; + case DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC5_VLINE0; + case DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC6_VLINE0; case DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT: return DC_IRQ_SOURCE_VUPDATE1; case DCN_1_0__SRCID__OTG1_IHC_V_UPDATE_NO_LOCK_INTERRUPT: @@ -167,6 +179,11 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = { .ack = NULL }; +static const struct irq_source_info_funcs vline0_irq_info_funcs = { + .set = NULL, + .ack = NULL +}; + static const struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = { .set = NULL, .ack = NULL @@ -241,6 +258,14 @@ static const struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = { .funcs = &vblank_irq_info_funcs\ } +#define vline0_int_entry(reg_num)\ + [DC_IRQ_SOURCE_DC1_VLINE0 + reg_num] = {\ + IRQ_REG_ENTRY(OTG, reg_num,\ + OTG_VERTICAL_INTERRUPT0_CONTROL, OTG_VERTICAL_INTERRUPT0_INT_ENABLE,\ + OTG_VERTICAL_INTERRUPT0_CONTROL, OTG_VERTICAL_INTERRUPT0_CLEAR),\ + .funcs = &vline0_irq_info_funcs\ + } + #define dummy_irq_entry() \ {\ .funcs = &dummy_irq_info_funcs\ @@ -349,6 +374,12 @@ irq_source_info_dcn10[DAL_IRQ_SOURCES_NUMBER] = { vblank_int_entry(3), vblank_int_entry(4), vblank_int_entry(5), + vline0_int_entry(0), + vline0_int_entry(1), + vline0_int_entry(2), + vline0_int_entry(3), + vline0_int_entry(4), + vline0_int_entry(5), }; static const struct irq_service_funcs irq_service_funcs_dcn10 = { diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c b/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c index 1b971265418b..0e0f494fbb5e 100644 --- a/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c +++ b/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c @@ -168,6 +168,11 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = { .ack = NULL }; +static const struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = { + .set = NULL, + .ack = NULL +}; + #undef BASE_INNER #define BASE_INNER(seg) DMU_BASE__INST0_SEG ## seg @@ -230,6 +235,17 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = { .funcs = &vblank_irq_info_funcs\ } +/* vupdate_no_lock_int_entry maps to DC_IRQ_SOURCE_VUPDATEx, to match semantic + * of DCE's DC_IRQ_SOURCE_VUPDATEx. + */ +#define vupdate_no_lock_int_entry(reg_num)\ + [DC_IRQ_SOURCE_VUPDATE1 + reg_num] = {\ + IRQ_REG_ENTRY(OTG, reg_num,\ + OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_INT_EN,\ + OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_EVENT_CLEAR),\ + .funcs = &vupdate_no_lock_irq_info_funcs\ + } + #define vblank_int_entry(reg_num)\ [DC_IRQ_SOURCE_VBLANK1 + reg_num] = {\ IRQ_REG_ENTRY(OTG, reg_num,\ @@ -338,6 +354,12 @@ irq_source_info_dcn21[DAL_IRQ_SOURCES_NUMBER] = { vupdate_int_entry(3), vupdate_int_entry(4), vupdate_int_entry(5), + vupdate_no_lock_int_entry(0), + vupdate_no_lock_int_entry(1), + vupdate_no_lock_int_entry(2), + vupdate_no_lock_int_entry(3), + vupdate_no_lock_int_entry(4), + vupdate_no_lock_int_entry(5), vblank_int_entry(0), vblank_int_entry(1), vblank_int_entry(2), diff --git a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c index 6bf27bde8724..5f245bde54ff 100644 --- a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c +++ b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c @@ -79,7 +79,7 @@ void dal_irq_service_destroy(struct irq_service **irq_service) *irq_service = NULL; } -const struct irq_source_info *find_irq_source_info( +static const struct irq_source_info *find_irq_source_info( struct irq_service *irq_service, enum dc_irq_source source) { diff --git a/drivers/gpu/drm/amd/display/dc/irq_types.h b/drivers/gpu/drm/amd/display/dc/irq_types.h index d0ccd81ad5b4..87812d81fed3 100644 --- a/drivers/gpu/drm/amd/display/dc/irq_types.h +++ b/drivers/gpu/drm/amd/display/dc/irq_types.h @@ -160,6 +160,7 @@ enum irq_type IRQ_TYPE_PFLIP = DC_IRQ_SOURCE_PFLIP1, IRQ_TYPE_VUPDATE = DC_IRQ_SOURCE_VUPDATE1, IRQ_TYPE_VBLANK = DC_IRQ_SOURCE_VBLANK1, + IRQ_TYPE_VLINE0 = DC_IRQ_SOURCE_DC1_VLINE0, }; #define DAL_VALID_IRQ_SRC_NUM(src) \ diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h index 249a076d6f69..072b4e7e624b 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h @@ -47,10 +47,10 @@ /* Firmware versioning. */ #ifdef DMUB_EXPOSE_VERSION -#define DMUB_FW_VERSION_GIT_HASH 0xf51b86a +#define DMUB_FW_VERSION_GIT_HASH 0x6444c02e7 #define DMUB_FW_VERSION_MAJOR 0 #define DMUB_FW_VERSION_MINOR 0 -#define DMUB_FW_VERSION_REVISION 47 +#define DMUB_FW_VERSION_REVISION 51 #define DMUB_FW_VERSION_TEST 0 #define DMUB_FW_VERSION_VBIOS 0 #define DMUB_FW_VERSION_HOTFIX 0 @@ -458,6 +458,10 @@ struct dmub_rb_cmd_mall { uint16_t cursor_pitch; uint16_t cursor_height; uint8_t cursor_bpp; + uint8_t debug_bits; + + uint8_t reserved1; + uint8_t reserved2; }; struct dmub_cmd_digx_encoder_control_data { @@ -487,13 +491,34 @@ struct dmub_rb_cmd_enable_disp_power_gating { struct dmub_cmd_enable_disp_power_gating_data power_gating; }; -struct dmub_cmd_dig1_transmitter_control_data { +struct dmub_dig_transmitter_control_data_v1_7 { + uint8_t phyid; /**< 0=UNIPHYA, 1=UNIPHYB, 2=UNIPHYC, 3=UNIPHYD, 4=UNIPHYE, 5=UNIPHYF */ + uint8_t action; /**< Defined as ATOM_TRANSMITER_ACTION_xxx */ + union { + uint8_t digmode; /**< enum atom_encode_mode_def */ + uint8_t dplaneset; /**< DP voltage swing and pre-emphasis value, "DP_LANE_SET__xDB_y_zV" */ + } mode_laneset; + uint8_t lanenum; /**< Number of lanes */ + union { + uint32_t symclk_10khz; /**< Symbol Clock in 10Khz */ + } symclk_units; + uint8_t hpdsel; /**< =1: HPD1, =2: HPD2, ..., =6: HPD6, =0: HPD is not assigned */ + uint8_t digfe_sel; /**< DIG front-end selection, bit0 means DIG0 FE is enabled */ + uint8_t connobj_id; /**< Connector Object Id defined in ObjectId.h */ + uint8_t reserved0; /**< For future use */ + uint8_t reserved1; /**< For future use */ + uint8_t reserved2[3]; /**< For future use */ + uint32_t reserved3[11]; /**< For future use */ +}; + +union dmub_cmd_dig1_transmitter_control_data { struct dig_transmitter_control_parameters_v1_6 dig; + struct dmub_dig_transmitter_control_data_v1_7 dig_v1_7; }; struct dmub_rb_cmd_dig1_transmitter_control { struct dmub_cmd_header header; - struct dmub_cmd_dig1_transmitter_control_data transmitter_control; + union dmub_cmd_dig1_transmitter_control_data transmitter_control; }; struct dmub_rb_cmd_dpphy_init { @@ -624,6 +649,7 @@ enum dmub_cmd_mall_type { DMUB_CMD__MALL_ACTION_ALLOW = 0, DMUB_CMD__MALL_ACTION_DISALLOW = 1, DMUB_CMD__MALL_ACTION_COPY_CURSOR = 2, + DMUB_CMD__MALL_ACTION_NO_DF_REQ = 3, }; struct dmub_cmd_psr_copy_settings_data { @@ -648,6 +674,7 @@ struct dmub_cmd_psr_copy_settings_data { uint8_t multi_disp_optimizations_en; uint16_t init_sdp_deadline; uint16_t pad2; + uint32_t line_time_in_us; }; struct dmub_rb_cmd_psr_copy_settings { diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c index cafba1d23c6a..8e8e65fa83c0 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c @@ -81,6 +81,13 @@ static inline void dmub_dcn20_translate_addr(const union dmub_addr *addr_in, addr_out->quad_part = addr_in->quad_part - fb_base + fb_offset; } +bool dmub_dcn20_use_cached_inbox(struct dmub_srv *dmub) +{ + /* Cached inbox is not supported in this fw version range */ + return !(dmub->fw_version >= DMUB_FW_VERSION(1, 0, 0) && + dmub->fw_version <= DMUB_FW_VERSION(1, 10, 0)); +} + void dmub_dcn20_reset(struct dmub_srv *dmub) { union dmub_gpint_data_register cmd; @@ -216,7 +223,7 @@ void dmub_dcn20_setup_windows(struct dmub_srv *dmub, dmub_dcn20_translate_addr(&cw4->offset, fb_base, fb_offset, &offset); /* New firmware can support CW4. */ - if (dmub->fw_version > DMUB_FW_VERSION(1, 0, 10)) { + if (dmub_dcn20_use_cached_inbox(dmub)) { REG_WRITE(DMCUB_REGION3_CW4_OFFSET, offset.u.low_part); REG_WRITE(DMCUB_REGION3_CW4_OFFSET_HIGH, offset.u.high_part); REG_WRITE(DMCUB_REGION3_CW4_BASE_ADDRESS, cw4->region.base); @@ -255,7 +262,7 @@ void dmub_dcn20_setup_mailbox(struct dmub_srv *dmub, const struct dmub_region *inbox1) { /* New firmware can support CW4 for the inbox. */ - if (dmub->fw_version > DMUB_FW_VERSION(1, 0, 10)) + if (dmub_dcn20_use_cached_inbox(dmub)) REG_WRITE(DMCUB_INBOX1_BASE_ADDRESS, inbox1->base); else REG_WRITE(DMCUB_INBOX1_BASE_ADDRESS, 0x80000000); diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h index d438f365cbb0..a62be9c0652e 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h @@ -198,4 +198,6 @@ void dmub_dcn20_skip_dmub_panel_power_sequence(struct dmub_srv *dmub, bool skip) union dmub_fw_boot_status dmub_dcn20_get_fw_boot_status(struct dmub_srv *dmub); +bool dmub_dcn20_use_cached_inbox(struct dmub_srv *dmub); + #endif /* _DMUB_DCN20_H_ */ diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn30.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn30.c index f00df02ded81..b4bc0df2f14a 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn30.c +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn30.c @@ -26,6 +26,7 @@ #include "../dmub_srv.h" #include "dmub_reg.h" #include "dmub_dcn20.h" +#include "dmub_dcn30.h" #include "sienna_cichlid_ip_offset.h" #include "dcn/dcn_3_0_0_offset.h" @@ -154,7 +155,7 @@ void dmub_dcn30_setup_windows(struct dmub_srv *dmub, offset = cw4->offset; /* New firmware can support CW4. */ - if (dmub->fw_version > DMUB_FW_VERSION(1, 0, 10)) { + if (dmub_dcn20_use_cached_inbox(dmub)) { REG_WRITE(DMCUB_REGION3_CW4_OFFSET, offset.u.low_part); REG_WRITE(DMCUB_REGION3_CW4_OFFSET_HIGH, offset.u.high_part); REG_WRITE(DMCUB_REGION3_CW4_BASE_ADDRESS, cw4->region.base); diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c index f388d36af0b6..61f64a295f06 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c @@ -406,6 +406,9 @@ enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub, dmub->fb_offset = params->fb_offset; dmub->psp_version = params->psp_version; + if (dmub->hw_funcs.reset) + dmub->hw_funcs.reset(dmub); + if (inst_fb && data_fb) { cw0.offset.quad_part = inst_fb->gpu_addr; cw0.region.base = DMUB_CW0_BASE; @@ -427,9 +430,6 @@ enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub, } - if (dmub->hw_funcs.reset) - dmub->hw_funcs.reset(dmub); - if (inst_fb && data_fb && bios_fb && mail_fb && tracebuff_fb && fw_state_fb && scratch_mem_fb) { cw2.offset.quad_part = data_fb->gpu_addr; @@ -489,9 +489,6 @@ enum dmub_status dmub_srv_hw_reset(struct dmub_srv *dmub) if (!dmub->sw_init) return DMUB_STATUS_INVALID; - if (dmub->hw_init == false) - return DMUB_STATUS_OK; - if (dmub->hw_funcs.reset) dmub->hw_funcs.reset(dmub); diff --git a/drivers/gpu/drm/amd/display/modules/color/color_table.c b/drivers/gpu/drm/amd/display/modules/color/color_table.c index 692e536e7d05..410f2a82b9a2 100644 --- a/drivers/gpu/drm/amd/display/modules/color/color_table.c +++ b/drivers/gpu/drm/amd/display/modules/color/color_table.c @@ -1,10 +1,26 @@ /* - * Copyright (c) 2019 Advanced Micro Devices, Inc. (unpublished) + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD * - * All rights reserved. This notice is intended as a precaution against - * inadvertent publication and does not imply publication or any waiver - * of confidentiality. The year included in the foregoing notice is the - * year of creation of the work. */ #include "color_table.h" diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h index 6c678cfb82e3..5c22cf7e6118 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h @@ -397,7 +397,7 @@ static inline uint8_t is_dp_hdcp(struct mod_hdcp *hdcp) static inline uint8_t is_dp_mst_hdcp(struct mod_hdcp *hdcp) { return (hdcp->connection.link.mode == MOD_HDCP_MODE_DP && - hdcp->connection.link.dp.mst_supported); + hdcp->connection.link.dp.mst_enabled); } static inline uint8_t is_hdmi_dvi_sl_hdcp(struct mod_hdcp *hdcp) diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c index 3a367a5968ae..904ce9b88088 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c @@ -106,7 +106,7 @@ enum mod_hdcp_status mod_hdcp_add_display_to_topology(struct mod_hdcp *hdcp, dtm_cmd->dtm_in_message.topology_update_v2.dig_be = link->dig_be; dtm_cmd->dtm_in_message.topology_update_v2.dig_fe = display->dig_fe; if (is_dp_hdcp(hdcp)) - dtm_cmd->dtm_in_message.topology_update_v2.is_assr = link->dp.assr_supported; + dtm_cmd->dtm_in_message.topology_update_v2.is_assr = link->dp.assr_enabled; dtm_cmd->dtm_in_message.topology_update_v2.dp_mst_vcid = display->vc_id; dtm_cmd->dtm_in_message.topology_update_v2.max_hdcp_supported_version = @@ -548,6 +548,8 @@ enum mod_hdcp_status mod_hdcp_hdcp2_validate_ake_cert(struct mod_hdcp *hdcp) TA_HDCP2_MSG_AUTHENTICATION_STATUS__RECEIVERID_REVOKED) { hdcp->connection.is_hdcp2_revoked = 1; status = MOD_HDCP_STATUS_HDCP2_AKE_CERT_REVOKED; + } else { + status = MOD_HDCP_STATUS_HDCP2_VALIDATE_AKE_CERT_FAILURE; } } mutex_unlock(&psp->hdcp_context.mutex); diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h b/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h index eed560eecbab..d223ed3be5d3 100644 --- a/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h @@ -101,8 +101,8 @@ enum mod_hdcp_status { struct mod_hdcp_displayport { uint8_t rev; - uint8_t assr_supported; - uint8_t mst_supported; + uint8_t assr_enabled; + uint8_t mst_enabled; }; struct mod_hdcp_hdmi { diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c index 0fdf7a3e96de..57f198de5e2c 100644 --- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c +++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c @@ -409,16 +409,11 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream, } /** - ***************************************************************************** - * Function: mod_build_hf_vsif_infopacket + * mod_build_hf_vsif_infopacket - Prepare HDMI Vendor Specific info frame. + * Follows HDMI Spec to build up Vendor Specific info frame * - * @brief - * Prepare HDMI Vendor Specific info frame. - * Follows HDMI Spec to build up Vendor Specific info frame - * - * @param [in] stream: contains data we may need to construct VSIF (i.e. timing_3d_format, etc.) - * @param [out] info_packet: output structure where to store VSIF - ***************************************************************************** + * @stream: contains data we may need to construct VSIF (i.e. timing_3d_format, etc.) + * @info_packet: output structure where to store VSIF */ void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream, struct dc_info_packet *info_packet) diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c index 4fd8bce95d84..6270ecbd2438 100644 --- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c +++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c @@ -266,7 +266,7 @@ static void fill_backlight_transform_table_v_2_2(struct dmcu_iram_parameters par * format U4.10. */ for (i = 1; i+1 < num_entries; i++) { - lut_index = (params.backlight_lut_array_size - 1) * i / (num_entries - 1); + lut_index = DIV_ROUNDUP((i * params.backlight_lut_array_size), num_entries); ASSERT(lut_index < params.backlight_lut_array_size); table->backlight_thresholds[i] = (big_endian) ? @@ -278,7 +278,7 @@ static void fill_backlight_transform_table_v_2_2(struct dmcu_iram_parameters par } } -void fill_iram_v_2(struct iram_table_v_2 *ram_table, struct dmcu_iram_parameters params) +static void fill_iram_v_2(struct iram_table_v_2 *ram_table, struct dmcu_iram_parameters params) { unsigned int set = params.set; @@ -452,7 +452,7 @@ void fill_iram_v_2(struct iram_table_v_2 *ram_table, struct dmcu_iram_parameters params, ram_table); } -void fill_iram_v_2_2(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params) +static void fill_iram_v_2_2(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params) { unsigned int set = params.set; @@ -598,7 +598,7 @@ void fill_iram_v_2_2(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parame params, ram_table, true); } -void fill_iram_v_2_3(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params, bool big_endian) +static void fill_iram_v_2_3(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params, bool big_endian) { unsigned int i, j; unsigned int set = params.set; diff --git a/drivers/gpu/drm/amd/include/amd_pcie.h b/drivers/gpu/drm/amd/include/amd_pcie.h index 9cb9ceb4d74d..a1ece3eecdf5 100644 --- a/drivers/gpu/drm/amd/include/amd_pcie.h +++ b/drivers/gpu/drm/amd/include/amd_pcie.h @@ -28,6 +28,7 @@ #define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 0x00020000 #define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 0x00040000 #define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 0x00080000 +#define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5 0x00100000 #define CAIL_PCIE_LINK_SPEED_SUPPORT_MASK 0xFFFF0000 #define CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT 16 @@ -36,6 +37,7 @@ #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 0x00000002 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 0x00000004 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 0x00000008 +#define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5 0x00000010 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_MASK 0x0000FFFF #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_SHIFT 0 diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h index 9676016a37ce..43ed6291b2b8 100644 --- a/drivers/gpu/drm/amd/include/amd_shared.h +++ b/drivers/gpu/drm/amd/include/amd_shared.h @@ -213,6 +213,7 @@ enum PP_FEATURE_MASK { PP_ACG_MASK = 0x10000, PP_STUTTER_MODE = 0x20000, PP_AVFS_MASK = 0x40000, + PP_GFX_DCS_MASK = 0x80000, }; enum DC_FEATURE_MASK { diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_offset.h new file mode 100644 index 000000000000..bd129266ebfd --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_offset.h @@ -0,0 +1,345 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef _osssys_4_2_0_OFFSET_HEADER +#define _osssys_4_2_0_OFFSET_HEADER + + + +// addressBlock: osssys_osssysdec +// base address: 0x4280 +#define mmIH_VMID_0_LUT 0x0000 +#define mmIH_VMID_0_LUT_BASE_IDX 0 +#define mmIH_VMID_1_LUT 0x0001 +#define mmIH_VMID_1_LUT_BASE_IDX 0 +#define mmIH_VMID_2_LUT 0x0002 +#define mmIH_VMID_2_LUT_BASE_IDX 0 +#define mmIH_VMID_3_LUT 0x0003 +#define mmIH_VMID_3_LUT_BASE_IDX 0 +#define mmIH_VMID_4_LUT 0x0004 +#define mmIH_VMID_4_LUT_BASE_IDX 0 +#define mmIH_VMID_5_LUT 0x0005 +#define mmIH_VMID_5_LUT_BASE_IDX 0 +#define mmIH_VMID_6_LUT 0x0006 +#define mmIH_VMID_6_LUT_BASE_IDX 0 +#define mmIH_VMID_7_LUT 0x0007 +#define mmIH_VMID_7_LUT_BASE_IDX 0 +#define mmIH_VMID_8_LUT 0x0008 +#define mmIH_VMID_8_LUT_BASE_IDX 0 +#define mmIH_VMID_9_LUT 0x0009 +#define mmIH_VMID_9_LUT_BASE_IDX 0 +#define mmIH_VMID_10_LUT 0x000a +#define mmIH_VMID_10_LUT_BASE_IDX 0 +#define mmIH_VMID_11_LUT 0x000b +#define mmIH_VMID_11_LUT_BASE_IDX 0 +#define mmIH_VMID_12_LUT 0x000c +#define mmIH_VMID_12_LUT_BASE_IDX 0 +#define mmIH_VMID_13_LUT 0x000d +#define mmIH_VMID_13_LUT_BASE_IDX 0 +#define mmIH_VMID_14_LUT 0x000e +#define mmIH_VMID_14_LUT_BASE_IDX 0 +#define mmIH_VMID_15_LUT 0x000f +#define mmIH_VMID_15_LUT_BASE_IDX 0 +#define mmIH_VMID_0_LUT_MM 0x0010 +#define mmIH_VMID_0_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_1_LUT_MM 0x0011 +#define mmIH_VMID_1_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_2_LUT_MM 0x0012 +#define mmIH_VMID_2_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_3_LUT_MM 0x0013 +#define mmIH_VMID_3_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_4_LUT_MM 0x0014 +#define mmIH_VMID_4_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_5_LUT_MM 0x0015 +#define mmIH_VMID_5_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_6_LUT_MM 0x0016 +#define mmIH_VMID_6_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_7_LUT_MM 0x0017 +#define mmIH_VMID_7_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_8_LUT_MM 0x0018 +#define mmIH_VMID_8_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_9_LUT_MM 0x0019 +#define mmIH_VMID_9_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_10_LUT_MM 0x001a +#define mmIH_VMID_10_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_11_LUT_MM 0x001b +#define mmIH_VMID_11_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_12_LUT_MM 0x001c +#define mmIH_VMID_12_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_13_LUT_MM 0x001d +#define mmIH_VMID_13_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_14_LUT_MM 0x001e +#define mmIH_VMID_14_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_15_LUT_MM 0x001f +#define mmIH_VMID_15_LUT_MM_BASE_IDX 0 +#define mmIH_COOKIE_0 0x0020 +#define mmIH_COOKIE_0_BASE_IDX 0 +#define mmIH_COOKIE_1 0x0021 +#define mmIH_COOKIE_1_BASE_IDX 0 +#define mmIH_COOKIE_2 0x0022 +#define mmIH_COOKIE_2_BASE_IDX 0 +#define mmIH_COOKIE_3 0x0023 +#define mmIH_COOKIE_3_BASE_IDX 0 +#define mmIH_COOKIE_4 0x0024 +#define mmIH_COOKIE_4_BASE_IDX 0 +#define mmIH_COOKIE_5 0x0025 +#define mmIH_COOKIE_5_BASE_IDX 0 +#define mmIH_COOKIE_6 0x0026 +#define mmIH_COOKIE_6_BASE_IDX 0 +#define mmIH_COOKIE_7 0x0027 +#define mmIH_COOKIE_7_BASE_IDX 0 +#define mmIH_REGISTER_LAST_PART0 0x003f +#define mmIH_REGISTER_LAST_PART0_BASE_IDX 0 +#define mmSEM_REQ_INPUT_0 0x0040 +#define mmSEM_REQ_INPUT_0_BASE_IDX 0 +#define mmSEM_REQ_INPUT_1 0x0041 +#define mmSEM_REQ_INPUT_1_BASE_IDX 0 +#define mmSEM_REQ_INPUT_2 0x0042 +#define mmSEM_REQ_INPUT_2_BASE_IDX 0 +#define mmSEM_REQ_INPUT_3 0x0043 +#define mmSEM_REQ_INPUT_3_BASE_IDX 0 +#define mmSEM_REGISTER_LAST_PART0 0x007f +#define mmSEM_REGISTER_LAST_PART0_BASE_IDX 0 +#define mmIH_RB_CNTL 0x0080 +#define mmIH_RB_CNTL_BASE_IDX 0 +#define mmIH_RB_BASE 0x0081 +#define mmIH_RB_BASE_BASE_IDX 0 +#define mmIH_RB_BASE_HI 0x0082 +#define mmIH_RB_BASE_HI_BASE_IDX 0 +#define mmIH_RB_RPTR 0x0083 +#define mmIH_RB_RPTR_BASE_IDX 0 +#define mmIH_RB_WPTR 0x0084 +#define mmIH_RB_WPTR_BASE_IDX 0 +#define mmIH_RB_WPTR_ADDR_HI 0x0085 +#define mmIH_RB_WPTR_ADDR_HI_BASE_IDX 0 +#define mmIH_RB_WPTR_ADDR_LO 0x0086 +#define mmIH_RB_WPTR_ADDR_LO_BASE_IDX 0 +#define mmIH_DOORBELL_RPTR 0x0087 +#define mmIH_DOORBELL_RPTR_BASE_IDX 0 +#define mmIH_RB_CNTL_RING1 0x008c +#define mmIH_RB_CNTL_RING1_BASE_IDX 0 +#define mmIH_RB_BASE_RING1 0x008d +#define mmIH_RB_BASE_RING1_BASE_IDX 0 +#define mmIH_RB_BASE_HI_RING1 0x008e +#define mmIH_RB_BASE_HI_RING1_BASE_IDX 0 +#define mmIH_RB_RPTR_RING1 0x008f +#define mmIH_RB_RPTR_RING1_BASE_IDX 0 +#define mmIH_RB_WPTR_RING1 0x0090 +#define mmIH_RB_WPTR_RING1_BASE_IDX 0 +#define mmIH_DOORBELL_RPTR_RING1 0x0093 +#define mmIH_DOORBELL_RPTR_RING1_BASE_IDX 0 +#define mmIH_RB_CNTL_RING2 0x0098 +#define mmIH_RB_CNTL_RING2_BASE_IDX 0 +#define mmIH_RB_BASE_RING2 0x0099 +#define mmIH_RB_BASE_RING2_BASE_IDX 0 +#define mmIH_RB_BASE_HI_RING2 0x009a +#define mmIH_RB_BASE_HI_RING2_BASE_IDX 0 +#define mmIH_RB_RPTR_RING2 0x009b +#define mmIH_RB_RPTR_RING2_BASE_IDX 0 +#define mmIH_RB_WPTR_RING2 0x009c +#define mmIH_RB_WPTR_RING2_BASE_IDX 0 +#define mmIH_DOORBELL_RPTR_RING2 0x009f +#define mmIH_DOORBELL_RPTR_RING2_BASE_IDX 0 +#define mmIH_VERSION 0x00a5 +#define mmIH_VERSION_BASE_IDX 0 +#define mmIH_CNTL 0x00c0 +#define mmIH_CNTL_BASE_IDX 0 +#define mmIH_CNTL2 0x00c1 +#define mmIH_CNTL2_BASE_IDX 0 +#define mmIH_STATUS 0x00c2 +#define mmIH_STATUS_BASE_IDX 0 +#define mmIH_PERFMON_CNTL 0x00c3 +#define mmIH_PERFMON_CNTL_BASE_IDX 0 +#define mmIH_PERFCOUNTER0_RESULT 0x00c4 +#define mmIH_PERFCOUNTER0_RESULT_BASE_IDX 0 +#define mmIH_PERFCOUNTER1_RESULT 0x00c5 +#define mmIH_PERFCOUNTER1_RESULT_BASE_IDX 0 +#define mmIH_DSM_MATCH_VALUE_BIT_31_0 0x00c7 +#define mmIH_DSM_MATCH_VALUE_BIT_31_0_BASE_IDX 0 +#define mmIH_DSM_MATCH_VALUE_BIT_63_32 0x00c8 +#define mmIH_DSM_MATCH_VALUE_BIT_63_32_BASE_IDX 0 +#define mmIH_DSM_MATCH_VALUE_BIT_95_64 0x00c9 +#define mmIH_DSM_MATCH_VALUE_BIT_95_64_BASE_IDX 0 +#define mmIH_DSM_MATCH_FIELD_CONTROL 0x00ca +#define mmIH_DSM_MATCH_FIELD_CONTROL_BASE_IDX 0 +#define mmIH_DSM_MATCH_DATA_CONTROL 0x00cb +#define mmIH_DSM_MATCH_DATA_CONTROL_BASE_IDX 0 +#define mmIH_DSM_MATCH_FCN_ID 0x00cc +#define mmIH_DSM_MATCH_FCN_ID_BASE_IDX 0 +#define mmIH_LIMIT_INT_RATE_CNTL 0x00cd +#define mmIH_LIMIT_INT_RATE_CNTL_BASE_IDX 0 +#define mmIH_VF_RB_STATUS 0x00ce +#define mmIH_VF_RB_STATUS_BASE_IDX 0 +#define mmIH_VF_RB_STATUS2 0x00cf +#define mmIH_VF_RB_STATUS2_BASE_IDX 0 +#define mmIH_VF_RB1_STATUS 0x00d0 +#define mmIH_VF_RB1_STATUS_BASE_IDX 0 +#define mmIH_VF_RB1_STATUS2 0x00d1 +#define mmIH_VF_RB1_STATUS2_BASE_IDX 0 +#define mmIH_VF_RB2_STATUS 0x00d2 +#define mmIH_VF_RB2_STATUS_BASE_IDX 0 +#define mmIH_VF_RB2_STATUS2 0x00d3 +#define mmIH_VF_RB2_STATUS2_BASE_IDX 0 +#define mmIH_INT_FLOOD_CNTL 0x00d5 +#define mmIH_INT_FLOOD_CNTL_BASE_IDX 0 +#define mmIH_RB0_INT_FLOOD_STATUS 0x00d6 +#define mmIH_RB0_INT_FLOOD_STATUS_BASE_IDX 0 +#define mmIH_RB1_INT_FLOOD_STATUS 0x00d7 +#define mmIH_RB1_INT_FLOOD_STATUS_BASE_IDX 0 +#define mmIH_RB2_INT_FLOOD_STATUS 0x00d8 +#define mmIH_RB2_INT_FLOOD_STATUS_BASE_IDX 0 +#define mmIH_INT_FLOOD_STATUS 0x00d9 +#define mmIH_INT_FLOOD_STATUS_BASE_IDX 0 +#define mmIH_STORM_CLIENT_LIST_CNTL 0x00da +#define mmIH_STORM_CLIENT_LIST_CNTL_BASE_IDX 0 +#define mmIH_CLK_CTRL 0x00db +#define mmIH_CLK_CTRL_BASE_IDX 0 +#define mmIH_INT_FLAGS 0x00dc +#define mmIH_INT_FLAGS_BASE_IDX 0 +#define mmIH_LAST_INT_INFO0 0x00dd +#define mmIH_LAST_INT_INFO0_BASE_IDX 0 +#define mmIH_LAST_INT_INFO1 0x00de +#define mmIH_LAST_INT_INFO1_BASE_IDX 0 +#define mmIH_LAST_INT_INFO2 0x00df +#define mmIH_LAST_INT_INFO2_BASE_IDX 0 +#define mmIH_SCRATCH 0x00e0 +#define mmIH_SCRATCH_BASE_IDX 0 +#define mmIH_CLIENT_CREDIT_ERROR 0x00e1 +#define mmIH_CLIENT_CREDIT_ERROR_BASE_IDX 0 +#define mmIH_GPU_IOV_VIOLATION_LOG 0x00e2 +#define mmIH_GPU_IOV_VIOLATION_LOG_BASE_IDX 0 +#define mmIH_COOKIE_REC_VIOLATION_LOG 0x00e3 +#define mmIH_COOKIE_REC_VIOLATION_LOG_BASE_IDX 0 +#define mmIH_CREDIT_STATUS 0x00e4 +#define mmIH_CREDIT_STATUS_BASE_IDX 0 +#define mmIH_MMHUB_ERROR 0x00e5 +#define mmIH_MMHUB_ERROR_BASE_IDX 0 +#define mmIH_MEM_POWER_CTRL 0x00e8 +#define mmIH_MEM_POWER_CTRL_BASE_IDX 0 +#define mmIH_REGISTER_LAST_PART2 0x00ff +#define mmIH_REGISTER_LAST_PART2_BASE_IDX 0 +#define mmSEM_CLK_CTRL 0x0100 +#define mmSEM_CLK_CTRL_BASE_IDX 0 +#define mmSEM_UTC_CREDIT 0x0101 +#define mmSEM_UTC_CREDIT_BASE_IDX 0 +#define mmSEM_UTC_CONFIG 0x0102 +#define mmSEM_UTC_CONFIG_BASE_IDX 0 +#define mmSEM_UTCL2_TRAN_EN_LUT 0x0103 +#define mmSEM_UTCL2_TRAN_EN_LUT_BASE_IDX 0 +#define mmSEM_MCIF_CONFIG 0x0104 +#define mmSEM_MCIF_CONFIG_BASE_IDX 0 +#define mmSEM_PERFMON_CNTL 0x0105 +#define mmSEM_PERFMON_CNTL_BASE_IDX 0 +#define mmSEM_PERFCOUNTER0_RESULT 0x0106 +#define mmSEM_PERFCOUNTER0_RESULT_BASE_IDX 0 +#define mmSEM_PERFCOUNTER1_RESULT 0x0107 +#define mmSEM_PERFCOUNTER1_RESULT_BASE_IDX 0 +#define mmSEM_STATUS 0x0108 +#define mmSEM_STATUS_BASE_IDX 0 +#define mmSEM_MAILBOX_CLIENTCONFIG 0x0109 +#define mmSEM_MAILBOX_CLIENTCONFIG_BASE_IDX 0 +#define mmSEM_MAILBOX 0x010a +#define mmSEM_MAILBOX_BASE_IDX 0 +#define mmSEM_MAILBOX_CONTROL 0x010b +#define mmSEM_MAILBOX_CONTROL_BASE_IDX 0 +#define mmSEM_CHICKEN_BITS 0x010c +#define mmSEM_CHICKEN_BITS_BASE_IDX 0 +#define mmSEM_MAILBOX_CLIENTCONFIG_EXTRA 0x010d +#define mmSEM_MAILBOX_CLIENTCONFIG_EXTRA_BASE_IDX 0 +#define mmSEM_GPU_IOV_VIOLATION_LOG 0x010e +#define mmSEM_GPU_IOV_VIOLATION_LOG_BASE_IDX 0 +#define mmSEM_OUTSTANDING_THRESHOLD 0x010f +#define mmSEM_OUTSTANDING_THRESHOLD_BASE_IDX 0 +#define mmSEM_MEM_POWER_CTRL 0x0110 +#define mmSEM_MEM_POWER_CTRL_BASE_IDX 0 +#define mmSEM_REGISTER_LAST_PART2 0x017f +#define mmSEM_REGISTER_LAST_PART2_BASE_IDX 0 +#define mmIH_ACTIVE_FCN_ID 0x0180 +#define mmIH_ACTIVE_FCN_ID_BASE_IDX 0 +#define mmIH_VIRT_RESET_REQ 0x0181 +#define mmIH_VIRT_RESET_REQ_BASE_IDX 0 +#define mmIH_CLIENT_CFG 0x0184 +#define mmIH_CLIENT_CFG_BASE_IDX 0 +#define mmIH_CLIENT_CFG_INDEX 0x0188 +#define mmIH_CLIENT_CFG_INDEX_BASE_IDX 0 +#define mmIH_CLIENT_CFG_DATA 0x0189 +#define mmIH_CLIENT_CFG_DATA_BASE_IDX 0 +#define mmIH_CID_REMAP_INDEX 0x018a +#define mmIH_CID_REMAP_INDEX_BASE_IDX 0 +#define mmIH_CID_REMAP_DATA 0x018b +#define mmIH_CID_REMAP_DATA_BASE_IDX 0 +#define mmIH_CHICKEN 0x018c +#define mmIH_CHICKEN_BASE_IDX 0 +#define mmIH_MMHUB_CNTL 0x018d +#define mmIH_MMHUB_CNTL_BASE_IDX 0 +#define mmIH_INT_DROP_CNTL 0x018e +#define mmIH_INT_DROP_CNTL_BASE_IDX 0 +#define mmIH_INT_DROP_MATCH_VALUE0 0x018f +#define mmIH_INT_DROP_MATCH_VALUE0_BASE_IDX 0 +#define mmIH_INT_DROP_MATCH_VALUE1 0x0190 +#define mmIH_INT_DROP_MATCH_VALUE1_BASE_IDX 0 +#define mmIH_INT_DROP_MATCH_MASK0 0x0191 +#define mmIH_INT_DROP_MATCH_MASK0_BASE_IDX 0 +#define mmIH_INT_DROP_MATCH_MASK1 0x0192 +#define mmIH_INT_DROP_MATCH_MASK1_BASE_IDX 0 +#define mmIH_REGISTER_LAST_PART1 0x019f +#define mmIH_REGISTER_LAST_PART1_BASE_IDX 0 +#define mmSEM_ACTIVE_FCN_ID 0x01a0 +#define mmSEM_ACTIVE_FCN_ID_BASE_IDX 0 +#define mmSEM_VIRT_RESET_REQ 0x01a1 +#define mmSEM_VIRT_RESET_REQ_BASE_IDX 0 +#define mmSEM_RESP_SDMA0 0x01a4 +#define mmSEM_RESP_SDMA0_BASE_IDX 0 +#define mmSEM_RESP_SDMA1 0x01a5 +#define mmSEM_RESP_SDMA1_BASE_IDX 0 +#define mmSEM_RESP_UVD 0x01a6 +#define mmSEM_RESP_UVD_BASE_IDX 0 +#define mmSEM_RESP_VCE_0 0x01a7 +#define mmSEM_RESP_VCE_0_BASE_IDX 0 +#define mmSEM_RESP_ACP 0x01a8 +#define mmSEM_RESP_ACP_BASE_IDX 0 +#define mmSEM_RESP_ISP 0x01a9 +#define mmSEM_RESP_ISP_BASE_IDX 0 +#define mmSEM_RESP_VCE_1 0x01aa +#define mmSEM_RESP_VCE_1_BASE_IDX 0 +#define mmSEM_RESP_VP8 0x01ab +#define mmSEM_RESP_VP8_BASE_IDX 0 +#define mmSEM_RESP_GC 0x01ac +#define mmSEM_RESP_GC_BASE_IDX 0 +#define mmSEM_RESP_UVD_1 0x01ad +#define mmSEM_RESP_UVD_1_BASE_IDX 0 +#define mmSEM_CID_REMAP_INDEX 0x01b0 +#define mmSEM_CID_REMAP_INDEX_BASE_IDX 0 +#define mmSEM_CID_REMAP_DATA 0x01b1 +#define mmSEM_CID_REMAP_DATA_BASE_IDX 0 +#define mmSEM_ATOMIC_OP_LUT 0x01b2 +#define mmSEM_ATOMIC_OP_LUT_BASE_IDX 0 +#define mmSEM_EDC_CONFIG 0x01b3 +#define mmSEM_EDC_CONFIG_BASE_IDX 0 +#define mmSEM_CHICKEN_BITS2 0x01b4 +#define mmSEM_CHICKEN_BITS2_BASE_IDX 0 +#define mmSEM_MMHUB_CNTL 0x01b5 +#define mmSEM_MMHUB_CNTL_BASE_IDX 0 +#define mmSEM_REGISTER_LAST_PART1 0x01bf +#define mmSEM_REGISTER_LAST_PART1_BASE_IDX 0 + +#endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_sh_mask.h new file mode 100644 index 000000000000..3ea83ea9ce3a --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_sh_mask.h @@ -0,0 +1,1300 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef _osssys_4_2_0_SH_MASK_HEADER +#define _osssys_4_2_0_SH_MASK_HEADER + + +// addressBlock: osssys_osssysdec +//IH_VMID_0_LUT +#define IH_VMID_0_LUT__PASID__SHIFT 0x0 +#define IH_VMID_0_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_1_LUT +#define IH_VMID_1_LUT__PASID__SHIFT 0x0 +#define IH_VMID_1_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_2_LUT +#define IH_VMID_2_LUT__PASID__SHIFT 0x0 +#define IH_VMID_2_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_3_LUT +#define IH_VMID_3_LUT__PASID__SHIFT 0x0 +#define IH_VMID_3_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_4_LUT +#define IH_VMID_4_LUT__PASID__SHIFT 0x0 +#define IH_VMID_4_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_5_LUT +#define IH_VMID_5_LUT__PASID__SHIFT 0x0 +#define IH_VMID_5_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_6_LUT +#define IH_VMID_6_LUT__PASID__SHIFT 0x0 +#define IH_VMID_6_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_7_LUT +#define IH_VMID_7_LUT__PASID__SHIFT 0x0 +#define IH_VMID_7_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_8_LUT +#define IH_VMID_8_LUT__PASID__SHIFT 0x0 +#define IH_VMID_8_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_9_LUT +#define IH_VMID_9_LUT__PASID__SHIFT 0x0 +#define IH_VMID_9_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_10_LUT +#define IH_VMID_10_LUT__PASID__SHIFT 0x0 +#define IH_VMID_10_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_11_LUT +#define IH_VMID_11_LUT__PASID__SHIFT 0x0 +#define IH_VMID_11_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_12_LUT +#define IH_VMID_12_LUT__PASID__SHIFT 0x0 +#define IH_VMID_12_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_13_LUT +#define IH_VMID_13_LUT__PASID__SHIFT 0x0 +#define IH_VMID_13_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_14_LUT +#define IH_VMID_14_LUT__PASID__SHIFT 0x0 +#define IH_VMID_14_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_15_LUT +#define IH_VMID_15_LUT__PASID__SHIFT 0x0 +#define IH_VMID_15_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_0_LUT_MM +#define IH_VMID_0_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_0_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_1_LUT_MM +#define IH_VMID_1_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_1_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_2_LUT_MM +#define IH_VMID_2_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_2_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_3_LUT_MM +#define IH_VMID_3_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_3_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_4_LUT_MM +#define IH_VMID_4_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_4_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_5_LUT_MM +#define IH_VMID_5_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_5_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_6_LUT_MM +#define IH_VMID_6_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_6_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_7_LUT_MM +#define IH_VMID_7_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_7_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_8_LUT_MM +#define IH_VMID_8_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_8_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_9_LUT_MM +#define IH_VMID_9_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_9_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_10_LUT_MM +#define IH_VMID_10_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_10_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_11_LUT_MM +#define IH_VMID_11_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_11_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_12_LUT_MM +#define IH_VMID_12_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_12_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_13_LUT_MM +#define IH_VMID_13_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_13_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_14_LUT_MM +#define IH_VMID_14_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_14_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_15_LUT_MM +#define IH_VMID_15_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_15_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_COOKIE_0 +#define IH_COOKIE_0__CLIENT_ID__SHIFT 0x0 +#define IH_COOKIE_0__SOURCE_ID__SHIFT 0x8 +#define IH_COOKIE_0__RING_ID__SHIFT 0x10 +#define IH_COOKIE_0__VM_ID__SHIFT 0x18 +#define IH_COOKIE_0__RESERVED__SHIFT 0x1c +#define IH_COOKIE_0__VMID_TYPE__SHIFT 0x1f +#define IH_COOKIE_0__CLIENT_ID_MASK 0x000000FFL +#define IH_COOKIE_0__SOURCE_ID_MASK 0x0000FF00L +#define IH_COOKIE_0__RING_ID_MASK 0x00FF0000L +#define IH_COOKIE_0__VM_ID_MASK 0x0F000000L +#define IH_COOKIE_0__RESERVED_MASK 0x70000000L +#define IH_COOKIE_0__VMID_TYPE_MASK 0x80000000L +//IH_COOKIE_1 +#define IH_COOKIE_1__TIMESTAMP_31_0__SHIFT 0x0 +#define IH_COOKIE_1__TIMESTAMP_31_0_MASK 0xFFFFFFFFL +//IH_COOKIE_2 +#define IH_COOKIE_2__TIMESTAMP_47_32__SHIFT 0x0 +#define IH_COOKIE_2__RESERVED__SHIFT 0x10 +#define IH_COOKIE_2__TIMESTAMP_SRC__SHIFT 0x1f +#define IH_COOKIE_2__TIMESTAMP_47_32_MASK 0x0000FFFFL +#define IH_COOKIE_2__RESERVED_MASK 0x7FFF0000L +#define IH_COOKIE_2__TIMESTAMP_SRC_MASK 0x80000000L +//IH_COOKIE_3 +#define IH_COOKIE_3__PAS_ID__SHIFT 0x0 +#define IH_COOKIE_3__RESERVED__SHIFT 0x10 +#define IH_COOKIE_3__PASID_SRC__SHIFT 0x1f +#define IH_COOKIE_3__PAS_ID_MASK 0x0000FFFFL +#define IH_COOKIE_3__RESERVED_MASK 0x7FFF0000L +#define IH_COOKIE_3__PASID_SRC_MASK 0x80000000L +//IH_COOKIE_4 +#define IH_COOKIE_4__CONTEXT_ID_31_0__SHIFT 0x0 +#define IH_COOKIE_4__CONTEXT_ID_31_0_MASK 0xFFFFFFFFL +//IH_COOKIE_5 +#define IH_COOKIE_5__CONTEXT_ID_63_32__SHIFT 0x0 +#define IH_COOKIE_5__CONTEXT_ID_63_32_MASK 0xFFFFFFFFL +//IH_COOKIE_6 +#define IH_COOKIE_6__CONTEXT_ID_95_64__SHIFT 0x0 +#define IH_COOKIE_6__CONTEXT_ID_95_64_MASK 0xFFFFFFFFL +//IH_COOKIE_7 +#define IH_COOKIE_7__CONTEXT_ID_128_96__SHIFT 0x0 +#define IH_COOKIE_7__CONTEXT_ID_128_96_MASK 0xFFFFFFFFL +//IH_REGISTER_LAST_PART0 +#define IH_REGISTER_LAST_PART0__RESERVED__SHIFT 0x0 +#define IH_REGISTER_LAST_PART0__RESERVED_MASK 0xFFFFFFFFL +//SEM_REQ_INPUT_0 +#define SEM_REQ_INPUT_0__DATA__SHIFT 0x0 +#define SEM_REQ_INPUT_0__DATA_MASK 0xFFFFFFFFL +//SEM_REQ_INPUT_1 +#define SEM_REQ_INPUT_1__DATA__SHIFT 0x0 +#define SEM_REQ_INPUT_1__DATA_MASK 0xFFFFFFFFL +//SEM_REQ_INPUT_2 +#define SEM_REQ_INPUT_2__DATA__SHIFT 0x0 +#define SEM_REQ_INPUT_2__DATA_MASK 0xFFFFFFFFL +//SEM_REQ_INPUT_3 +#define SEM_REQ_INPUT_3__DATA__SHIFT 0x0 +#define SEM_REQ_INPUT_3__DATA_MASK 0xFFFFFFFFL +//SEM_REGISTER_LAST_PART0 +#define SEM_REGISTER_LAST_PART0__RESERVED__SHIFT 0x0 +#define SEM_REGISTER_LAST_PART0__RESERVED_MASK 0xFFFFFFFFL +//IH_RB_CNTL +#define IH_RB_CNTL__RB_ENABLE__SHIFT 0x0 +#define IH_RB_CNTL__RB_SIZE__SHIFT 0x1 +#define IH_RB_CNTL__RB_GPU_TS_ENABLE__SHIFT 0x7 +#define IH_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT 0x8 +#define IH_RB_CNTL__RB_FULL_DRAIN_ENABLE__SHIFT 0x9 +#define IH_RB_CNTL__FULL_DRAIN_CLEAR__SHIFT 0xa +#define IH_RB_CNTL__PAGE_RB_CLEAR__SHIFT 0xb +#define IH_RB_CNTL__RB_USED_INT_THRESHOLD__SHIFT 0xc +#define IH_RB_CNTL__WPTR_OVERFLOW_ENABLE__SHIFT 0x10 +#define IH_RB_CNTL__ENABLE_INTR__SHIFT 0x11 +#define IH_RB_CNTL__MC_SWAP__SHIFT 0x12 +#define IH_RB_CNTL__MC_SNOOP__SHIFT 0x14 +#define IH_RB_CNTL__RPTR_REARM__SHIFT 0x15 +#define IH_RB_CNTL__MC_RO__SHIFT 0x16 +#define IH_RB_CNTL__MC_VMID__SHIFT 0x18 +#define IH_RB_CNTL__MC_SPACE__SHIFT 0x1c +#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT 0x1f +#define IH_RB_CNTL__RB_ENABLE_MASK 0x00000001L +#define IH_RB_CNTL__RB_SIZE_MASK 0x0000003EL +#define IH_RB_CNTL__RB_GPU_TS_ENABLE_MASK 0x00000080L +#define IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK 0x00000100L +#define IH_RB_CNTL__RB_FULL_DRAIN_ENABLE_MASK 0x00000200L +#define IH_RB_CNTL__FULL_DRAIN_CLEAR_MASK 0x00000400L +#define IH_RB_CNTL__PAGE_RB_CLEAR_MASK 0x00000800L +#define IH_RB_CNTL__RB_USED_INT_THRESHOLD_MASK 0x0000F000L +#define IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK 0x00010000L +#define IH_RB_CNTL__ENABLE_INTR_MASK 0x00020000L +#define IH_RB_CNTL__MC_SWAP_MASK 0x000C0000L +#define IH_RB_CNTL__MC_SNOOP_MASK 0x00100000L +#define IH_RB_CNTL__RPTR_REARM_MASK 0x00200000L +#define IH_RB_CNTL__MC_RO_MASK 0x00400000L +#define IH_RB_CNTL__MC_VMID_MASK 0x0F000000L +#define IH_RB_CNTL__MC_SPACE_MASK 0x70000000L +#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK 0x80000000L +//IH_RB_BASE +#define IH_RB_BASE__ADDR__SHIFT 0x0 +#define IH_RB_BASE__ADDR_MASK 0xFFFFFFFFL +//IH_RB_BASE_HI +#define IH_RB_BASE_HI__ADDR__SHIFT 0x0 +#define IH_RB_BASE_HI__ADDR_MASK 0x000000FFL +//IH_RB_RPTR +#define IH_RB_RPTR__OFFSET__SHIFT 0x2 +#define IH_RB_RPTR__OFFSET_MASK 0x0003FFFCL +//IH_RB_WPTR +#define IH_RB_WPTR__RB_OVERFLOW__SHIFT 0x0 +#define IH_RB_WPTR__OFFSET__SHIFT 0x2 +#define IH_RB_WPTR__RB_LEFT_NONE__SHIFT 0x12 +#define IH_RB_WPTR__RB_MAY_OVERFLOW__SHIFT 0x13 +#define IH_RB_WPTR__RB_OVERFLOW_MASK 0x00000001L +#define IH_RB_WPTR__OFFSET_MASK 0x0003FFFCL +#define IH_RB_WPTR__RB_LEFT_NONE_MASK 0x00040000L +#define IH_RB_WPTR__RB_MAY_OVERFLOW_MASK 0x00080000L +//IH_RB_WPTR_ADDR_HI +#define IH_RB_WPTR_ADDR_HI__ADDR__SHIFT 0x0 +#define IH_RB_WPTR_ADDR_HI__ADDR_MASK 0x0000FFFFL +//IH_RB_WPTR_ADDR_LO +#define IH_RB_WPTR_ADDR_LO__ADDR__SHIFT 0x2 +#define IH_RB_WPTR_ADDR_LO__ADDR_MASK 0xFFFFFFFCL +//IH_DOORBELL_RPTR +#define IH_DOORBELL_RPTR__OFFSET__SHIFT 0x0 +#define IH_DOORBELL_RPTR__ENABLE__SHIFT 0x1c +#define IH_DOORBELL_RPTR__OFFSET_MASK 0x03FFFFFFL +#define IH_DOORBELL_RPTR__ENABLE_MASK 0x10000000L +//IH_RB_CNTL_RING1 +#define IH_RB_CNTL_RING1__RB_ENABLE__SHIFT 0x0 +#define IH_RB_CNTL_RING1__RB_SIZE__SHIFT 0x1 +#define IH_RB_CNTL_RING1__RB_GPU_TS_ENABLE__SHIFT 0x7 +#define IH_RB_CNTL_RING1__RB_FULL_DRAIN_ENABLE__SHIFT 0x9 +#define IH_RB_CNTL_RING1__FULL_DRAIN_CLEAR__SHIFT 0xa +#define IH_RB_CNTL_RING1__PAGE_RB_CLEAR__SHIFT 0xb +#define IH_RB_CNTL_RING1__RB_USED_INT_THRESHOLD__SHIFT 0xc +#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_ENABLE__SHIFT 0x10 +#define IH_RB_CNTL_RING1__MC_SWAP__SHIFT 0x12 +#define IH_RB_CNTL_RING1__MC_SNOOP__SHIFT 0x14 +#define IH_RB_CNTL_RING1__MC_RO__SHIFT 0x16 +#define IH_RB_CNTL_RING1__MC_VMID__SHIFT 0x18 +#define IH_RB_CNTL_RING1__MC_SPACE__SHIFT 0x1c +#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_CLEAR__SHIFT 0x1f +#define IH_RB_CNTL_RING1__RB_ENABLE_MASK 0x00000001L +#define IH_RB_CNTL_RING1__RB_SIZE_MASK 0x0000003EL +#define IH_RB_CNTL_RING1__RB_GPU_TS_ENABLE_MASK 0x00000080L +#define IH_RB_CNTL_RING1__RB_FULL_DRAIN_ENABLE_MASK 0x00000200L +#define IH_RB_CNTL_RING1__FULL_DRAIN_CLEAR_MASK 0x00000400L +#define IH_RB_CNTL_RING1__PAGE_RB_CLEAR_MASK 0x00000800L +#define IH_RB_CNTL_RING1__RB_USED_INT_THRESHOLD_MASK 0x0000F000L +#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_ENABLE_MASK 0x00010000L +#define IH_RB_CNTL_RING1__MC_SWAP_MASK 0x000C0000L +#define IH_RB_CNTL_RING1__MC_SNOOP_MASK 0x00100000L +#define IH_RB_CNTL_RING1__MC_RO_MASK 0x00400000L +#define IH_RB_CNTL_RING1__MC_VMID_MASK 0x0F000000L +#define IH_RB_CNTL_RING1__MC_SPACE_MASK 0x70000000L +#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_CLEAR_MASK 0x80000000L +//IH_RB_BASE_RING1 +#define IH_RB_BASE_RING1__ADDR__SHIFT 0x0 +#define IH_RB_BASE_RING1__ADDR_MASK 0xFFFFFFFFL +//IH_RB_BASE_HI_RING1 +#define IH_RB_BASE_HI_RING1__ADDR__SHIFT 0x0 +#define IH_RB_BASE_HI_RING1__ADDR_MASK 0x000000FFL +//IH_RB_RPTR_RING1 +#define IH_RB_RPTR_RING1__OFFSET__SHIFT 0x2 +#define IH_RB_RPTR_RING1__OFFSET_MASK 0x0003FFFCL +//IH_RB_WPTR_RING1 +#define IH_RB_WPTR_RING1__RB_OVERFLOW__SHIFT 0x0 +#define IH_RB_WPTR_RING1__OFFSET__SHIFT 0x2 +#define IH_RB_WPTR_RING1__RB_LEFT_NONE__SHIFT 0x12 +#define IH_RB_WPTR_RING1__RB_MAY_OVERFLOW__SHIFT 0x13 +#define IH_RB_WPTR_RING1__RB_OVERFLOW_MASK 0x00000001L +#define IH_RB_WPTR_RING1__OFFSET_MASK 0x0003FFFCL +#define IH_RB_WPTR_RING1__RB_LEFT_NONE_MASK 0x00040000L +#define IH_RB_WPTR_RING1__RB_MAY_OVERFLOW_MASK 0x00080000L +//IH_DOORBELL_RPTR_RING1 +#define IH_DOORBELL_RPTR_RING1__OFFSET__SHIFT 0x0 +#define IH_DOORBELL_RPTR_RING1__ENABLE__SHIFT 0x1c +#define IH_DOORBELL_RPTR_RING1__OFFSET_MASK 0x03FFFFFFL +#define IH_DOORBELL_RPTR_RING1__ENABLE_MASK 0x10000000L +//IH_RB_CNTL_RING2 +#define IH_RB_CNTL_RING2__RB_ENABLE__SHIFT 0x0 +#define IH_RB_CNTL_RING2__RB_SIZE__SHIFT 0x1 +#define IH_RB_CNTL_RING2__RB_GPU_TS_ENABLE__SHIFT 0x7 +#define IH_RB_CNTL_RING2__RB_FULL_DRAIN_ENABLE__SHIFT 0x9 +#define IH_RB_CNTL_RING2__FULL_DRAIN_CLEAR__SHIFT 0xa +#define IH_RB_CNTL_RING2__PAGE_RB_CLEAR__SHIFT 0xb +#define IH_RB_CNTL_RING2__RB_USED_INT_THRESHOLD__SHIFT 0xc +#define IH_RB_CNTL_RING2__WPTR_OVERFLOW_ENABLE__SHIFT 0x10 +#define IH_RB_CNTL_RING2__MC_SWAP__SHIFT 0x12 +#define IH_RB_CNTL_RING2__MC_SNOOP__SHIFT 0x14 +#define IH_RB_CNTL_RING2__MC_RO__SHIFT 0x16 +#define IH_RB_CNTL_RING2__MC_VMID__SHIFT 0x18 +#define IH_RB_CNTL_RING2__MC_SPACE__SHIFT 0x1c +#define IH_RB_CNTL_RING2__WPTR_OVERFLOW_CLEAR__SHIFT 0x1f +#define IH_RB_CNTL_RING2__RB_ENABLE_MASK 0x00000001L +#define IH_RB_CNTL_RING2__RB_SIZE_MASK 0x0000003EL +#define IH_RB_CNTL_RING2__RB_GPU_TS_ENABLE_MASK 0x00000080L +#define IH_RB_CNTL_RING2__RB_FULL_DRAIN_ENABLE_MASK 0x00000200L +#define IH_RB_CNTL_RING2__FULL_DRAIN_CLEAR_MASK 0x00000400L +#define IH_RB_CNTL_RING2__PAGE_RB_CLEAR_MASK 0x00000800L +#define IH_RB_CNTL_RING2__RB_USED_INT_THRESHOLD_MASK 0x0000F000L +#define IH_RB_CNTL_RING2__WPTR_OVERFLOW_ENABLE_MASK 0x00010000L +#define IH_RB_CNTL_RING2__MC_SWAP_MASK 0x000C0000L +#define IH_RB_CNTL_RING2__MC_SNOOP_MASK 0x00100000L +#define IH_RB_CNTL_RING2__MC_RO_MASK 0x00400000L +#define IH_RB_CNTL_RING2__MC_VMID_MASK 0x0F000000L +#define IH_RB_CNTL_RING2__MC_SPACE_MASK 0x70000000L +#define IH_RB_CNTL_RING2__WPTR_OVERFLOW_CLEAR_MASK 0x80000000L +//IH_RB_BASE_RING2 +#define IH_RB_BASE_RING2__ADDR__SHIFT 0x0 +#define IH_RB_BASE_RING2__ADDR_MASK 0xFFFFFFFFL +//IH_RB_BASE_HI_RING2 +#define IH_RB_BASE_HI_RING2__ADDR__SHIFT 0x0 +#define IH_RB_BASE_HI_RING2__ADDR_MASK 0x000000FFL +//IH_RB_RPTR_RING2 +#define IH_RB_RPTR_RING2__OFFSET__SHIFT 0x2 +#define IH_RB_RPTR_RING2__OFFSET_MASK 0x0003FFFCL +//IH_RB_WPTR_RING2 +#define IH_RB_WPTR_RING2__RB_OVERFLOW__SHIFT 0x0 +#define IH_RB_WPTR_RING2__OFFSET__SHIFT 0x2 +#define IH_RB_WPTR_RING2__RB_LEFT_NONE__SHIFT 0x12 +#define IH_RB_WPTR_RING2__RB_MAY_OVERFLOW__SHIFT 0x13 +#define IH_RB_WPTR_RING2__RB_OVERFLOW_MASK 0x00000001L +#define IH_RB_WPTR_RING2__OFFSET_MASK 0x0003FFFCL +#define IH_RB_WPTR_RING2__RB_LEFT_NONE_MASK 0x00040000L +#define IH_RB_WPTR_RING2__RB_MAY_OVERFLOW_MASK 0x00080000L +//IH_DOORBELL_RPTR_RING2 +#define IH_DOORBELL_RPTR_RING2__OFFSET__SHIFT 0x0 +#define IH_DOORBELL_RPTR_RING2__ENABLE__SHIFT 0x1c +#define IH_DOORBELL_RPTR_RING2__OFFSET_MASK 0x03FFFFFFL +#define IH_DOORBELL_RPTR_RING2__ENABLE_MASK 0x10000000L +//IH_VERSION +#define IH_VERSION__MINVER__SHIFT 0x0 +#define IH_VERSION__MAJVER__SHIFT 0x8 +#define IH_VERSION__REV__SHIFT 0x10 +#define IH_VERSION__MINVER_MASK 0x0000007FL +#define IH_VERSION__MAJVER_MASK 0x00007F00L +#define IH_VERSION__REV_MASK 0x003F0000L +//IH_CNTL +#define IH_CNTL__WPTR_WRITEBACK_TIMER__SHIFT 0x0 +#define IH_CNTL__IH_IDLE_HYSTERESIS_CNTL__SHIFT 0x6 +#define IH_CNTL__IH_FIFO_HIGHWATER__SHIFT 0x8 +#define IH_CNTL__MC_WR_CLEAN_CNT__SHIFT 0x14 +#define IH_CNTL__WPTR_WRITEBACK_TIMER_MASK 0x0000001FL +#define IH_CNTL__IH_IDLE_HYSTERESIS_CNTL_MASK 0x000000C0L +#define IH_CNTL__IH_FIFO_HIGHWATER_MASK 0x00007F00L +#define IH_CNTL__MC_WR_CLEAN_CNT_MASK 0x01F00000L +//IH_CNTL2 +#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_TIMEOUT__SHIFT 0x0 +#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_ENABLE__SHIFT 0x8 +#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_TIMEOUT_MASK 0x0000001FL +#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_ENABLE_MASK 0x00000100L +//IH_STATUS +#define IH_STATUS__IDLE__SHIFT 0x0 +#define IH_STATUS__INPUT_IDLE__SHIFT 0x1 +#define IH_STATUS__BUFFER_IDLE__SHIFT 0x2 +#define IH_STATUS__RB_FULL__SHIFT 0x3 +#define IH_STATUS__RB_FULL_DRAIN__SHIFT 0x4 +#define IH_STATUS__RB_OVERFLOW__SHIFT 0x5 +#define IH_STATUS__MC_WR_IDLE__SHIFT 0x6 +#define IH_STATUS__MC_WR_STALL__SHIFT 0x7 +#define IH_STATUS__MC_WR_CLEAN_PENDING__SHIFT 0x8 +#define IH_STATUS__MC_WR_CLEAN_STALL__SHIFT 0x9 +#define IH_STATUS__BIF_INTERRUPT_LINE__SHIFT 0xa +#define IH_STATUS__SWITCH_READY__SHIFT 0xb +#define IH_STATUS__RB1_FULL__SHIFT 0xc +#define IH_STATUS__RB1_FULL_DRAIN__SHIFT 0xd +#define IH_STATUS__RB1_OVERFLOW__SHIFT 0xe +#define IH_STATUS__RB2_FULL__SHIFT 0xf +#define IH_STATUS__RB2_FULL_DRAIN__SHIFT 0x10 +#define IH_STATUS__RB2_OVERFLOW__SHIFT 0x11 +#define IH_STATUS__SELF_INT_GEN_IDLE__SHIFT 0x12 +#define IH_STATUS__IDLE_MASK 0x00000001L +#define IH_STATUS__INPUT_IDLE_MASK 0x00000002L +#define IH_STATUS__BUFFER_IDLE_MASK 0x00000004L +#define IH_STATUS__RB_FULL_MASK 0x00000008L +#define IH_STATUS__RB_FULL_DRAIN_MASK 0x00000010L +#define IH_STATUS__RB_OVERFLOW_MASK 0x00000020L +#define IH_STATUS__MC_WR_IDLE_MASK 0x00000040L +#define IH_STATUS__MC_WR_STALL_MASK 0x00000080L +#define IH_STATUS__MC_WR_CLEAN_PENDING_MASK 0x00000100L +#define IH_STATUS__MC_WR_CLEAN_STALL_MASK 0x00000200L +#define IH_STATUS__BIF_INTERRUPT_LINE_MASK 0x00000400L +#define IH_STATUS__SWITCH_READY_MASK 0x00000800L +#define IH_STATUS__RB1_FULL_MASK 0x00001000L +#define IH_STATUS__RB1_FULL_DRAIN_MASK 0x00002000L +#define IH_STATUS__RB1_OVERFLOW_MASK 0x00004000L +#define IH_STATUS__RB2_FULL_MASK 0x00008000L +#define IH_STATUS__RB2_FULL_DRAIN_MASK 0x00010000L +#define IH_STATUS__RB2_OVERFLOW_MASK 0x00020000L +#define IH_STATUS__SELF_INT_GEN_IDLE_MASK 0x00040000L +//IH_PERFMON_CNTL +#define IH_PERFMON_CNTL__ENABLE0__SHIFT 0x0 +#define IH_PERFMON_CNTL__CLEAR0__SHIFT 0x1 +#define IH_PERFMON_CNTL__PERF_SEL0__SHIFT 0x2 +#define IH_PERFMON_CNTL__ENABLE1__SHIFT 0x10 +#define IH_PERFMON_CNTL__CLEAR1__SHIFT 0x11 +#define IH_PERFMON_CNTL__PERF_SEL1__SHIFT 0x12 +#define IH_PERFMON_CNTL__ENABLE0_MASK 0x00000001L +#define IH_PERFMON_CNTL__CLEAR0_MASK 0x00000002L +#define IH_PERFMON_CNTL__PERF_SEL0_MASK 0x000007FCL +#define IH_PERFMON_CNTL__ENABLE1_MASK 0x00010000L +#define IH_PERFMON_CNTL__CLEAR1_MASK 0x00020000L +#define IH_PERFMON_CNTL__PERF_SEL1_MASK 0x07FC0000L +//IH_PERFCOUNTER0_RESULT +#define IH_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT 0x0 +#define IH_PERFCOUNTER0_RESULT__PERF_COUNT_MASK 0xFFFFFFFFL +//IH_PERFCOUNTER1_RESULT +#define IH_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT 0x0 +#define IH_PERFCOUNTER1_RESULT__PERF_COUNT_MASK 0xFFFFFFFFL +//IH_DSM_MATCH_VALUE_BIT_31_0 +#define IH_DSM_MATCH_VALUE_BIT_31_0__VALUE__SHIFT 0x0 +#define IH_DSM_MATCH_VALUE_BIT_31_0__VALUE_MASK 0xFFFFFFFFL +//IH_DSM_MATCH_VALUE_BIT_63_32 +#define IH_DSM_MATCH_VALUE_BIT_63_32__VALUE__SHIFT 0x0 +#define IH_DSM_MATCH_VALUE_BIT_63_32__VALUE_MASK 0xFFFFFFFFL +//IH_DSM_MATCH_VALUE_BIT_95_64 +#define IH_DSM_MATCH_VALUE_BIT_95_64__VALUE__SHIFT 0x0 +#define IH_DSM_MATCH_VALUE_BIT_95_64__VALUE_MASK 0xFFFFFFFFL +//IH_DSM_MATCH_FIELD_CONTROL +#define IH_DSM_MATCH_FIELD_CONTROL__SRC_EN__SHIFT 0x0 +#define IH_DSM_MATCH_FIELD_CONTROL__FCNID_EN__SHIFT 0x1 +#define IH_DSM_MATCH_FIELD_CONTROL__TIMESTAMP_EN__SHIFT 0x2 +#define IH_DSM_MATCH_FIELD_CONTROL__RINGID_EN__SHIFT 0x3 +#define IH_DSM_MATCH_FIELD_CONTROL__VMID_EN__SHIFT 0x4 +#define IH_DSM_MATCH_FIELD_CONTROL__PASID_EN__SHIFT 0x5 +#define IH_DSM_MATCH_FIELD_CONTROL__CLIENT_ID_EN__SHIFT 0x6 +#define IH_DSM_MATCH_FIELD_CONTROL__SRC_EN_MASK 0x00000001L +#define IH_DSM_MATCH_FIELD_CONTROL__FCNID_EN_MASK 0x00000002L +#define IH_DSM_MATCH_FIELD_CONTROL__TIMESTAMP_EN_MASK 0x00000004L +#define IH_DSM_MATCH_FIELD_CONTROL__RINGID_EN_MASK 0x00000008L +#define IH_DSM_MATCH_FIELD_CONTROL__VMID_EN_MASK 0x00000010L +#define IH_DSM_MATCH_FIELD_CONTROL__PASID_EN_MASK 0x00000020L +#define IH_DSM_MATCH_FIELD_CONTROL__CLIENT_ID_EN_MASK 0x00000040L +//IH_DSM_MATCH_DATA_CONTROL +#define IH_DSM_MATCH_DATA_CONTROL__VALUE__SHIFT 0x0 +#define IH_DSM_MATCH_DATA_CONTROL__VALUE_MASK 0x0FFFFFFFL +//IH_DSM_MATCH_FCN_ID +#define IH_DSM_MATCH_FCN_ID__PF_VF__SHIFT 0x0 +#define IH_DSM_MATCH_FCN_ID__VF_ID__SHIFT 0x1 +#define IH_DSM_MATCH_FCN_ID__PF_VF_MASK 0x00000001L +#define IH_DSM_MATCH_FCN_ID__VF_ID_MASK 0x0000001EL +//IH_LIMIT_INT_RATE_CNTL +#define IH_LIMIT_INT_RATE_CNTL__LIMIT_ENABLE__SHIFT 0x0 +#define IH_LIMIT_INT_RATE_CNTL__PERF_INTERVAL__SHIFT 0x1 +#define IH_LIMIT_INT_RATE_CNTL__PERF_THRESHOLD__SHIFT 0x5 +#define IH_LIMIT_INT_RATE_CNTL__RETURN_DELAY__SHIFT 0x11 +#define IH_LIMIT_INT_RATE_CNTL__PERF_RESULT__SHIFT 0x15 +#define IH_LIMIT_INT_RATE_CNTL__LIMIT_ENABLE_MASK 0x00000001L +#define IH_LIMIT_INT_RATE_CNTL__PERF_INTERVAL_MASK 0x0000001EL +#define IH_LIMIT_INT_RATE_CNTL__PERF_THRESHOLD_MASK 0x0000FFE0L +#define IH_LIMIT_INT_RATE_CNTL__RETURN_DELAY_MASK 0x001E0000L +#define IH_LIMIT_INT_RATE_CNTL__PERF_RESULT_MASK 0xFFE00000L +//IH_VF_RB_STATUS +#define IH_VF_RB_STATUS__RB_FULL_DRAIN_VF__SHIFT 0x0 +#define IH_VF_RB_STATUS__RB_OVERFLOW_VF__SHIFT 0x10 +#define IH_VF_RB_STATUS__RB_FULL_DRAIN_VF_MASK 0x0000FFFFL +#define IH_VF_RB_STATUS__RB_OVERFLOW_VF_MASK 0xFFFF0000L +//IH_VF_RB_STATUS2 +#define IH_VF_RB_STATUS2__RB_FULL_VF__SHIFT 0x0 +#define IH_VF_RB_STATUS2__BIF_INTERRUPT_LINE_VF__SHIFT 0x10 +#define IH_VF_RB_STATUS2__RB_FULL_VF_MASK 0x0000FFFFL +#define IH_VF_RB_STATUS2__BIF_INTERRUPT_LINE_VF_MASK 0xFFFF0000L +//IH_VF_RB1_STATUS +#define IH_VF_RB1_STATUS__RB_FULL_DRAIN_VF__SHIFT 0x0 +#define IH_VF_RB1_STATUS__RB_OVERFLOW_VF__SHIFT 0x10 +#define IH_VF_RB1_STATUS__RB_FULL_DRAIN_VF_MASK 0x0000FFFFL +#define IH_VF_RB1_STATUS__RB_OVERFLOW_VF_MASK 0xFFFF0000L +//IH_VF_RB1_STATUS2 +#define IH_VF_RB1_STATUS2__RB_FULL_VF__SHIFT 0x0 +#define IH_VF_RB1_STATUS2__RB_FULL_VF_MASK 0x0000FFFFL +//IH_VF_RB2_STATUS +#define IH_VF_RB2_STATUS__RB_FULL_DRAIN_VF__SHIFT 0x0 +#define IH_VF_RB2_STATUS__RB_OVERFLOW_VF__SHIFT 0x10 +#define IH_VF_RB2_STATUS__RB_FULL_DRAIN_VF_MASK 0x0000FFFFL +#define IH_VF_RB2_STATUS__RB_OVERFLOW_VF_MASK 0xFFFF0000L +//IH_VF_RB2_STATUS2 +#define IH_VF_RB2_STATUS2__RB_FULL_VF__SHIFT 0x0 +#define IH_VF_RB2_STATUS2__RB_FULL_VF_MASK 0x0000FFFFL +//IH_INT_FLOOD_CNTL +#define IH_INT_FLOOD_CNTL__HIGHWATER__SHIFT 0x0 +#define IH_INT_FLOOD_CNTL__FLOOD_CNTL_ENABLE__SHIFT 0x3 +#define IH_INT_FLOOD_CNTL__CLEAR_INT_FLOOD_STATUS__SHIFT 0x4 +#define IH_INT_FLOOD_CNTL__HIGHWATER_MASK 0x00000007L +#define IH_INT_FLOOD_CNTL__FLOOD_CNTL_ENABLE_MASK 0x00000008L +#define IH_INT_FLOOD_CNTL__CLEAR_INT_FLOOD_STATUS_MASK 0x00000010L +//IH_RB0_INT_FLOOD_STATUS +#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_VF__SHIFT 0x0 +#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED__SHIFT 0x1f +#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_VF_MASK 0x0000FFFFL +#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_MASK 0x80000000L +//IH_RB1_INT_FLOOD_STATUS +#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_VF__SHIFT 0x0 +#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED__SHIFT 0x1f +#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_VF_MASK 0x0000FFFFL +#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_MASK 0x80000000L +//IH_RB2_INT_FLOOD_STATUS +#define IH_RB2_INT_FLOOD_STATUS__RB_INT_DROPPED_VF__SHIFT 0x0 +#define IH_RB2_INT_FLOOD_STATUS__RB_INT_DROPPED__SHIFT 0x1f +#define IH_RB2_INT_FLOOD_STATUS__RB_INT_DROPPED_VF_MASK 0x0000FFFFL +#define IH_RB2_INT_FLOOD_STATUS__RB_INT_DROPPED_MASK 0x80000000L +//IH_INT_FLOOD_STATUS +#define IH_INT_FLOOD_STATUS__INT_DROP_CNT__SHIFT 0x0 +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_CLIENT_ID__SHIFT 0x8 +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_SOURCE_ID__SHIFT 0x10 +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_ID__SHIFT 0x18 +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF__SHIFT 0x1c +#define IH_INT_FLOOD_STATUS__INT_DROPPED__SHIFT 0x1e +#define IH_INT_FLOOD_STATUS__INT_DROP_CNT_MASK 0x000000FFL +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_CLIENT_ID_MASK 0x0000FF00L +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_SOURCE_ID_MASK 0x00FF0000L +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_ID_MASK 0x0F000000L +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_MASK 0x10000000L +#define IH_INT_FLOOD_STATUS__INT_DROPPED_MASK 0x40000000L +//IH_STORM_CLIENT_LIST_CNTL +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT1_IS_STORM_CLIENT__SHIFT 0x1 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT2_IS_STORM_CLIENT__SHIFT 0x2 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT3_IS_STORM_CLIENT__SHIFT 0x3 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT4_IS_STORM_CLIENT__SHIFT 0x4 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT5_IS_STORM_CLIENT__SHIFT 0x5 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT6_IS_STORM_CLIENT__SHIFT 0x6 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT7_IS_STORM_CLIENT__SHIFT 0x7 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT8_IS_STORM_CLIENT__SHIFT 0x8 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT9_IS_STORM_CLIENT__SHIFT 0x9 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT10_IS_STORM_CLIENT__SHIFT 0xa +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT11_IS_STORM_CLIENT__SHIFT 0xb +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT12_IS_STORM_CLIENT__SHIFT 0xc +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT13_IS_STORM_CLIENT__SHIFT 0xd +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT14_IS_STORM_CLIENT__SHIFT 0xe +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT15_IS_STORM_CLIENT__SHIFT 0xf +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT16_IS_STORM_CLIENT__SHIFT 0x10 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT17_IS_STORM_CLIENT__SHIFT 0x11 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT18_IS_STORM_CLIENT__SHIFT 0x12 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT19_IS_STORM_CLIENT__SHIFT 0x13 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT20_IS_STORM_CLIENT__SHIFT 0x14 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT21_IS_STORM_CLIENT__SHIFT 0x15 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT22_IS_STORM_CLIENT__SHIFT 0x16 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT23_IS_STORM_CLIENT__SHIFT 0x17 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT24_IS_STORM_CLIENT__SHIFT 0x18 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT25_IS_STORM_CLIENT__SHIFT 0x19 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT26_IS_STORM_CLIENT__SHIFT 0x1a +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT27_IS_STORM_CLIENT__SHIFT 0x1b +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT28_IS_STORM_CLIENT__SHIFT 0x1c +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT29_IS_STORM_CLIENT__SHIFT 0x1d +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT30_IS_STORM_CLIENT__SHIFT 0x1e +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT31_IS_STORM_CLIENT__SHIFT 0x1f +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT1_IS_STORM_CLIENT_MASK 0x00000002L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT2_IS_STORM_CLIENT_MASK 0x00000004L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT3_IS_STORM_CLIENT_MASK 0x00000008L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT4_IS_STORM_CLIENT_MASK 0x00000010L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT5_IS_STORM_CLIENT_MASK 0x00000020L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT6_IS_STORM_CLIENT_MASK 0x00000040L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT7_IS_STORM_CLIENT_MASK 0x00000080L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT8_IS_STORM_CLIENT_MASK 0x00000100L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT9_IS_STORM_CLIENT_MASK 0x00000200L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT10_IS_STORM_CLIENT_MASK 0x00000400L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT11_IS_STORM_CLIENT_MASK 0x00000800L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT12_IS_STORM_CLIENT_MASK 0x00001000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT13_IS_STORM_CLIENT_MASK 0x00002000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT14_IS_STORM_CLIENT_MASK 0x00004000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT15_IS_STORM_CLIENT_MASK 0x00008000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT16_IS_STORM_CLIENT_MASK 0x00010000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT17_IS_STORM_CLIENT_MASK 0x00020000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT18_IS_STORM_CLIENT_MASK 0x00040000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT19_IS_STORM_CLIENT_MASK 0x00080000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT20_IS_STORM_CLIENT_MASK 0x00100000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT21_IS_STORM_CLIENT_MASK 0x00200000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT22_IS_STORM_CLIENT_MASK 0x00400000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT23_IS_STORM_CLIENT_MASK 0x00800000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT24_IS_STORM_CLIENT_MASK 0x01000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT25_IS_STORM_CLIENT_MASK 0x02000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT26_IS_STORM_CLIENT_MASK 0x04000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT27_IS_STORM_CLIENT_MASK 0x08000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT28_IS_STORM_CLIENT_MASK 0x10000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT29_IS_STORM_CLIENT_MASK 0x20000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT30_IS_STORM_CLIENT_MASK 0x40000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT31_IS_STORM_CLIENT_MASK 0x80000000L +//IH_CLK_CTRL +#define IH_CLK_CTRL__IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE__SHIFT 0x19 +#define IH_CLK_CTRL__IH_BUFFER_MEM_CLK_SOFT_OVERRIDE__SHIFT 0x1a +#define IH_CLK_CTRL__DBUS_MUX_CLK_SOFT_OVERRIDE__SHIFT 0x1b +#define IH_CLK_CTRL__OSSSYS_SHARE_CLK_SOFT_OVERRIDE__SHIFT 0x1c +#define IH_CLK_CTRL__LIMIT_SMN_CLK_SOFT_OVERRIDE__SHIFT 0x1d +#define IH_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE__SHIFT 0x1e +#define IH_CLK_CTRL__REG_CLK_SOFT_OVERRIDE__SHIFT 0x1f +#define IH_CLK_CTRL__IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE_MASK 0x02000000L +#define IH_CLK_CTRL__IH_BUFFER_MEM_CLK_SOFT_OVERRIDE_MASK 0x04000000L +#define IH_CLK_CTRL__DBUS_MUX_CLK_SOFT_OVERRIDE_MASK 0x08000000L +#define IH_CLK_CTRL__OSSSYS_SHARE_CLK_SOFT_OVERRIDE_MASK 0x10000000L +#define IH_CLK_CTRL__LIMIT_SMN_CLK_SOFT_OVERRIDE_MASK 0x20000000L +#define IH_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE_MASK 0x40000000L +#define IH_CLK_CTRL__REG_CLK_SOFT_OVERRIDE_MASK 0x80000000L +//IH_INT_FLAGS +#define IH_INT_FLAGS__CLIENT_0_FLAG__SHIFT 0x0 +#define IH_INT_FLAGS__CLIENT_1_FLAG__SHIFT 0x1 +#define IH_INT_FLAGS__CLIENT_2_FLAG__SHIFT 0x2 +#define IH_INT_FLAGS__CLIENT_3_FLAG__SHIFT 0x3 +#define IH_INT_FLAGS__CLIENT_4_FLAG__SHIFT 0x4 +#define IH_INT_FLAGS__CLIENT_5_FLAG__SHIFT 0x5 +#define IH_INT_FLAGS__CLIENT_6_FLAG__SHIFT 0x6 +#define IH_INT_FLAGS__CLIENT_7_FLAG__SHIFT 0x7 +#define IH_INT_FLAGS__CLIENT_8_FLAG__SHIFT 0x8 +#define IH_INT_FLAGS__CLIENT_9_FLAG__SHIFT 0x9 +#define IH_INT_FLAGS__CLIENT_10_FLAG__SHIFT 0xa +#define IH_INT_FLAGS__CLIENT_11_FLAG__SHIFT 0xb +#define IH_INT_FLAGS__CLIENT_12_FLAG__SHIFT 0xc +#define IH_INT_FLAGS__CLIENT_13_FLAG__SHIFT 0xd +#define IH_INT_FLAGS__CLIENT_14_FLAG__SHIFT 0xe +#define IH_INT_FLAGS__CLIENT_15_FLAG__SHIFT 0xf +#define IH_INT_FLAGS__CLIENT_16_FLAG__SHIFT 0x10 +#define IH_INT_FLAGS__CLIENT_17_FLAG__SHIFT 0x11 +#define IH_INT_FLAGS__CLIENT_18_FLAG__SHIFT 0x12 +#define IH_INT_FLAGS__CLIENT_19_FLAG__SHIFT 0x13 +#define IH_INT_FLAGS__CLIENT_20_FLAG__SHIFT 0x14 +#define IH_INT_FLAGS__CLIENT_21_FLAG__SHIFT 0x15 +#define IH_INT_FLAGS__CLIENT_22_FLAG__SHIFT 0x16 +#define IH_INT_FLAGS__CLIENT_23_FLAG__SHIFT 0x17 +#define IH_INT_FLAGS__CLIENT_24_FLAG__SHIFT 0x18 +#define IH_INT_FLAGS__CLIENT_25_FLAG__SHIFT 0x19 +#define IH_INT_FLAGS__CLIENT_26_FLAG__SHIFT 0x1a +#define IH_INT_FLAGS__CLIENT_27_FLAG__SHIFT 0x1b +#define IH_INT_FLAGS__CLIENT_28_FLAG__SHIFT 0x1c +#define IH_INT_FLAGS__CLIENT_29_FLAG__SHIFT 0x1d +#define IH_INT_FLAGS__CLIENT_30_FLAG__SHIFT 0x1e +#define IH_INT_FLAGS__CLIENT_31_FLAG__SHIFT 0x1f +#define IH_INT_FLAGS__CLIENT_0_FLAG_MASK 0x00000001L +#define IH_INT_FLAGS__CLIENT_1_FLAG_MASK 0x00000002L +#define IH_INT_FLAGS__CLIENT_2_FLAG_MASK 0x00000004L +#define IH_INT_FLAGS__CLIENT_3_FLAG_MASK 0x00000008L +#define IH_INT_FLAGS__CLIENT_4_FLAG_MASK 0x00000010L +#define IH_INT_FLAGS__CLIENT_5_FLAG_MASK 0x00000020L +#define IH_INT_FLAGS__CLIENT_6_FLAG_MASK 0x00000040L +#define IH_INT_FLAGS__CLIENT_7_FLAG_MASK 0x00000080L +#define IH_INT_FLAGS__CLIENT_8_FLAG_MASK 0x00000100L +#define IH_INT_FLAGS__CLIENT_9_FLAG_MASK 0x00000200L +#define IH_INT_FLAGS__CLIENT_10_FLAG_MASK 0x00000400L +#define IH_INT_FLAGS__CLIENT_11_FLAG_MASK 0x00000800L +#define IH_INT_FLAGS__CLIENT_12_FLAG_MASK 0x00001000L +#define IH_INT_FLAGS__CLIENT_13_FLAG_MASK 0x00002000L +#define IH_INT_FLAGS__CLIENT_14_FLAG_MASK 0x00004000L +#define IH_INT_FLAGS__CLIENT_15_FLAG_MASK 0x00008000L +#define IH_INT_FLAGS__CLIENT_16_FLAG_MASK 0x00010000L +#define IH_INT_FLAGS__CLIENT_17_FLAG_MASK 0x00020000L +#define IH_INT_FLAGS__CLIENT_18_FLAG_MASK 0x00040000L +#define IH_INT_FLAGS__CLIENT_19_FLAG_MASK 0x00080000L +#define IH_INT_FLAGS__CLIENT_20_FLAG_MASK 0x00100000L +#define IH_INT_FLAGS__CLIENT_21_FLAG_MASK 0x00200000L +#define IH_INT_FLAGS__CLIENT_22_FLAG_MASK 0x00400000L +#define IH_INT_FLAGS__CLIENT_23_FLAG_MASK 0x00800000L +#define IH_INT_FLAGS__CLIENT_24_FLAG_MASK 0x01000000L +#define IH_INT_FLAGS__CLIENT_25_FLAG_MASK 0x02000000L +#define IH_INT_FLAGS__CLIENT_26_FLAG_MASK 0x04000000L +#define IH_INT_FLAGS__CLIENT_27_FLAG_MASK 0x08000000L +#define IH_INT_FLAGS__CLIENT_28_FLAG_MASK 0x10000000L +#define IH_INT_FLAGS__CLIENT_29_FLAG_MASK 0x20000000L +#define IH_INT_FLAGS__CLIENT_30_FLAG_MASK 0x40000000L +#define IH_INT_FLAGS__CLIENT_31_FLAG_MASK 0x80000000L +//IH_LAST_INT_INFO0 +#define IH_LAST_INT_INFO0__CLIENT_ID__SHIFT 0x0 +#define IH_LAST_INT_INFO0__SOURCE_ID__SHIFT 0x8 +#define IH_LAST_INT_INFO0__RING_ID__SHIFT 0x10 +#define IH_LAST_INT_INFO0__VM_ID__SHIFT 0x18 +#define IH_LAST_INT_INFO0__VMID_TYPE__SHIFT 0x1f +#define IH_LAST_INT_INFO0__CLIENT_ID_MASK 0x000000FFL +#define IH_LAST_INT_INFO0__SOURCE_ID_MASK 0x0000FF00L +#define IH_LAST_INT_INFO0__RING_ID_MASK 0x00FF0000L +#define IH_LAST_INT_INFO0__VM_ID_MASK 0x0F000000L +#define IH_LAST_INT_INFO0__VMID_TYPE_MASK 0x80000000L +//IH_LAST_INT_INFO1 +#define IH_LAST_INT_INFO1__CONTEXT_ID__SHIFT 0x0 +#define IH_LAST_INT_INFO1__CONTEXT_ID_MASK 0xFFFFFFFFL +//IH_LAST_INT_INFO2 +#define IH_LAST_INT_INFO2__PAS_ID__SHIFT 0x0 +#define IH_LAST_INT_INFO2__VF_ID__SHIFT 0x10 +#define IH_LAST_INT_INFO2__VF__SHIFT 0x14 +#define IH_LAST_INT_INFO2__PAS_ID_MASK 0x0000FFFFL +#define IH_LAST_INT_INFO2__VF_ID_MASK 0x000F0000L +#define IH_LAST_INT_INFO2__VF_MASK 0x00100000L +//IH_SCRATCH +#define IH_SCRATCH__DATA__SHIFT 0x0 +#define IH_SCRATCH__DATA_MASK 0xFFFFFFFFL +//IH_CLIENT_CREDIT_ERROR +#define IH_CLIENT_CREDIT_ERROR__CLEAR__SHIFT 0x0 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_1_ERROR__SHIFT 0x1 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_2_ERROR__SHIFT 0x2 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_3_ERROR__SHIFT 0x3 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_4_ERROR__SHIFT 0x4 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_5_ERROR__SHIFT 0x5 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_6_ERROR__SHIFT 0x6 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_7_ERROR__SHIFT 0x7 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_8_ERROR__SHIFT 0x8 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_9_ERROR__SHIFT 0x9 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_10_ERROR__SHIFT 0xa +#define IH_CLIENT_CREDIT_ERROR__CLIENT_11_ERROR__SHIFT 0xb +#define IH_CLIENT_CREDIT_ERROR__CLIENT_12_ERROR__SHIFT 0xc +#define IH_CLIENT_CREDIT_ERROR__CLIENT_13_ERROR__SHIFT 0xd +#define IH_CLIENT_CREDIT_ERROR__CLIENT_14_ERROR__SHIFT 0xe +#define IH_CLIENT_CREDIT_ERROR__CLIENT_15_ERROR__SHIFT 0xf +#define IH_CLIENT_CREDIT_ERROR__CLIENT_16_ERROR__SHIFT 0x10 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_17_ERROR__SHIFT 0x11 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_18_ERROR__SHIFT 0x12 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_19_ERROR__SHIFT 0x13 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_20_ERROR__SHIFT 0x14 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_21_ERROR__SHIFT 0x15 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_22_ERROR__SHIFT 0x16 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_23_ERROR__SHIFT 0x17 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_24_ERROR__SHIFT 0x18 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_25_ERROR__SHIFT 0x19 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_26_ERROR__SHIFT 0x1a +#define IH_CLIENT_CREDIT_ERROR__CLIENT_27_ERROR__SHIFT 0x1b +#define IH_CLIENT_CREDIT_ERROR__CLIENT_28_ERROR__SHIFT 0x1c +#define IH_CLIENT_CREDIT_ERROR__CLIENT_29_ERROR__SHIFT 0x1d +#define IH_CLIENT_CREDIT_ERROR__CLIENT_30_ERROR__SHIFT 0x1e +#define IH_CLIENT_CREDIT_ERROR__CLIENT_31_ERROR__SHIFT 0x1f +#define IH_CLIENT_CREDIT_ERROR__CLEAR_MASK 0x00000001L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_1_ERROR_MASK 0x00000002L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_2_ERROR_MASK 0x00000004L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_3_ERROR_MASK 0x00000008L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_4_ERROR_MASK 0x00000010L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_5_ERROR_MASK 0x00000020L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_6_ERROR_MASK 0x00000040L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_7_ERROR_MASK 0x00000080L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_8_ERROR_MASK 0x00000100L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_9_ERROR_MASK 0x00000200L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_10_ERROR_MASK 0x00000400L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_11_ERROR_MASK 0x00000800L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_12_ERROR_MASK 0x00001000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_13_ERROR_MASK 0x00002000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_14_ERROR_MASK 0x00004000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_15_ERROR_MASK 0x00008000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_16_ERROR_MASK 0x00010000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_17_ERROR_MASK 0x00020000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_18_ERROR_MASK 0x00040000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_19_ERROR_MASK 0x00080000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_20_ERROR_MASK 0x00100000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_21_ERROR_MASK 0x00200000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_22_ERROR_MASK 0x00400000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_23_ERROR_MASK 0x00800000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_24_ERROR_MASK 0x01000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_25_ERROR_MASK 0x02000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_26_ERROR_MASK 0x04000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_27_ERROR_MASK 0x08000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_28_ERROR_MASK 0x10000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_29_ERROR_MASK 0x20000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_30_ERROR_MASK 0x40000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_31_ERROR_MASK 0x80000000L +//IH_GPU_IOV_VIOLATION_LOG +#define IH_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT 0x0 +#define IH_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT 0x1 +#define IH_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT 0x2 +#define IH_GPU_IOV_VIOLATION_LOG__OPCODE__SHIFT 0x12 +#define IH_GPU_IOV_VIOLATION_LOG__VF__SHIFT 0x13 +#define IH_GPU_IOV_VIOLATION_LOG__VF_ID__SHIFT 0x14 +#define IH_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT 0x18 +#define IH_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK 0x00000001L +#define IH_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK 0x00000002L +#define IH_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK 0x0003FFFCL +#define IH_GPU_IOV_VIOLATION_LOG__OPCODE_MASK 0x00040000L +#define IH_GPU_IOV_VIOLATION_LOG__VF_MASK 0x00080000L +#define IH_GPU_IOV_VIOLATION_LOG__VF_ID_MASK 0x00F00000L +#define IH_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK 0xFF000000L +//IH_COOKIE_REC_VIOLATION_LOG +#define IH_COOKIE_REC_VIOLATION_LOG__VIOLATION_STATUS__SHIFT 0x0 +#define IH_COOKIE_REC_VIOLATION_LOG__CLIENT_ID__SHIFT 0x10 +#define IH_COOKIE_REC_VIOLATION_LOG__INITIATOR_ID__SHIFT 0x18 +#define IH_COOKIE_REC_VIOLATION_LOG__VIOLATION_STATUS_MASK 0x00000001L +#define IH_COOKIE_REC_VIOLATION_LOG__CLIENT_ID_MASK 0x00FF0000L +#define IH_COOKIE_REC_VIOLATION_LOG__INITIATOR_ID_MASK 0xFF000000L +//IH_CREDIT_STATUS +#define IH_CREDIT_STATUS__CLIENT_1_CREDIT_RETURNED__SHIFT 0x1 +#define IH_CREDIT_STATUS__CLIENT_2_CREDIT_RETURNED__SHIFT 0x2 +#define IH_CREDIT_STATUS__CLIENT_3_CREDIT_RETURNED__SHIFT 0x3 +#define IH_CREDIT_STATUS__CLIENT_4_CREDIT_RETURNED__SHIFT 0x4 +#define IH_CREDIT_STATUS__CLIENT_5_CREDIT_RETURNED__SHIFT 0x5 +#define IH_CREDIT_STATUS__CLIENT_6_CREDIT_RETURNED__SHIFT 0x6 +#define IH_CREDIT_STATUS__CLIENT_7_CREDIT_RETURNED__SHIFT 0x7 +#define IH_CREDIT_STATUS__CLIENT_8_CREDIT_RETURNED__SHIFT 0x8 +#define IH_CREDIT_STATUS__CLIENT_9_CREDIT_RETURNED__SHIFT 0x9 +#define IH_CREDIT_STATUS__CLIENT_10_CREDIT_RETURNED__SHIFT 0xa +#define IH_CREDIT_STATUS__CLIENT_11_CREDIT_RETURNED__SHIFT 0xb +#define IH_CREDIT_STATUS__CLIENT_12_CREDIT_RETURNED__SHIFT 0xc +#define IH_CREDIT_STATUS__CLIENT_13_CREDIT_RETURNED__SHIFT 0xd +#define IH_CREDIT_STATUS__CLIENT_14_CREDIT_RETURNED__SHIFT 0xe +#define IH_CREDIT_STATUS__CLIENT_15_CREDIT_RETURNED__SHIFT 0xf +#define IH_CREDIT_STATUS__CLIENT_16_CREDIT_RETURNED__SHIFT 0x10 +#define IH_CREDIT_STATUS__CLIENT_17_CREDIT_RETURNED__SHIFT 0x11 +#define IH_CREDIT_STATUS__CLIENT_18_CREDIT_RETURNED__SHIFT 0x12 +#define IH_CREDIT_STATUS__CLIENT_19_CREDIT_RETURNED__SHIFT 0x13 +#define IH_CREDIT_STATUS__CLIENT_20_CREDIT_RETURNED__SHIFT 0x14 +#define IH_CREDIT_STATUS__CLIENT_21_CREDIT_RETURNED__SHIFT 0x15 +#define IH_CREDIT_STATUS__CLIENT_22_CREDIT_RETURNED__SHIFT 0x16 +#define IH_CREDIT_STATUS__CLIENT_23_CREDIT_RETURNED__SHIFT 0x17 +#define IH_CREDIT_STATUS__CLIENT_24_CREDIT_RETURNED__SHIFT 0x18 +#define IH_CREDIT_STATUS__CLIENT_25_CREDIT_RETURNED__SHIFT 0x19 +#define IH_CREDIT_STATUS__CLIENT_26_CREDIT_RETURNED__SHIFT 0x1a +#define IH_CREDIT_STATUS__CLIENT_27_CREDIT_RETURNED__SHIFT 0x1b +#define IH_CREDIT_STATUS__CLIENT_28_CREDIT_RETURNED__SHIFT 0x1c +#define IH_CREDIT_STATUS__CLIENT_29_CREDIT_RETURNED__SHIFT 0x1d +#define IH_CREDIT_STATUS__CLIENT_30_CREDIT_RETURNED__SHIFT 0x1e +#define IH_CREDIT_STATUS__CLIENT_31_CREDIT_RETURNED__SHIFT 0x1f +#define IH_CREDIT_STATUS__CLIENT_1_CREDIT_RETURNED_MASK 0x00000002L +#define IH_CREDIT_STATUS__CLIENT_2_CREDIT_RETURNED_MASK 0x00000004L +#define IH_CREDIT_STATUS__CLIENT_3_CREDIT_RETURNED_MASK 0x00000008L +#define IH_CREDIT_STATUS__CLIENT_4_CREDIT_RETURNED_MASK 0x00000010L +#define IH_CREDIT_STATUS__CLIENT_5_CREDIT_RETURNED_MASK 0x00000020L +#define IH_CREDIT_STATUS__CLIENT_6_CREDIT_RETURNED_MASK 0x00000040L +#define IH_CREDIT_STATUS__CLIENT_7_CREDIT_RETURNED_MASK 0x00000080L +#define IH_CREDIT_STATUS__CLIENT_8_CREDIT_RETURNED_MASK 0x00000100L +#define IH_CREDIT_STATUS__CLIENT_9_CREDIT_RETURNED_MASK 0x00000200L +#define IH_CREDIT_STATUS__CLIENT_10_CREDIT_RETURNED_MASK 0x00000400L +#define IH_CREDIT_STATUS__CLIENT_11_CREDIT_RETURNED_MASK 0x00000800L +#define IH_CREDIT_STATUS__CLIENT_12_CREDIT_RETURNED_MASK 0x00001000L +#define IH_CREDIT_STATUS__CLIENT_13_CREDIT_RETURNED_MASK 0x00002000L +#define IH_CREDIT_STATUS__CLIENT_14_CREDIT_RETURNED_MASK 0x00004000L +#define IH_CREDIT_STATUS__CLIENT_15_CREDIT_RETURNED_MASK 0x00008000L +#define IH_CREDIT_STATUS__CLIENT_16_CREDIT_RETURNED_MASK 0x00010000L +#define IH_CREDIT_STATUS__CLIENT_17_CREDIT_RETURNED_MASK 0x00020000L +#define IH_CREDIT_STATUS__CLIENT_18_CREDIT_RETURNED_MASK 0x00040000L +#define IH_CREDIT_STATUS__CLIENT_19_CREDIT_RETURNED_MASK 0x00080000L +#define IH_CREDIT_STATUS__CLIENT_20_CREDIT_RETURNED_MASK 0x00100000L +#define IH_CREDIT_STATUS__CLIENT_21_CREDIT_RETURNED_MASK 0x00200000L +#define IH_CREDIT_STATUS__CLIENT_22_CREDIT_RETURNED_MASK 0x00400000L +#define IH_CREDIT_STATUS__CLIENT_23_CREDIT_RETURNED_MASK 0x00800000L +#define IH_CREDIT_STATUS__CLIENT_24_CREDIT_RETURNED_MASK 0x01000000L +#define IH_CREDIT_STATUS__CLIENT_25_CREDIT_RETURNED_MASK 0x02000000L +#define IH_CREDIT_STATUS__CLIENT_26_CREDIT_RETURNED_MASK 0x04000000L +#define IH_CREDIT_STATUS__CLIENT_27_CREDIT_RETURNED_MASK 0x08000000L +#define IH_CREDIT_STATUS__CLIENT_28_CREDIT_RETURNED_MASK 0x10000000L +#define IH_CREDIT_STATUS__CLIENT_29_CREDIT_RETURNED_MASK 0x20000000L +#define IH_CREDIT_STATUS__CLIENT_30_CREDIT_RETURNED_MASK 0x40000000L +#define IH_CREDIT_STATUS__CLIENT_31_CREDIT_RETURNED_MASK 0x80000000L +//IH_MMHUB_ERROR +#define IH_MMHUB_ERROR__IH_BRESP_01__SHIFT 0x1 +#define IH_MMHUB_ERROR__IH_BRESP_10__SHIFT 0x2 +#define IH_MMHUB_ERROR__IH_BRESP_11__SHIFT 0x3 +#define IH_MMHUB_ERROR__IH_BUSER_NACK_01__SHIFT 0x5 +#define IH_MMHUB_ERROR__IH_BUSER_NACK_10__SHIFT 0x6 +#define IH_MMHUB_ERROR__IH_BUSER_NACK_11__SHIFT 0x7 +#define IH_MMHUB_ERROR__IH_BRESP_01_MASK 0x00000002L +#define IH_MMHUB_ERROR__IH_BRESP_10_MASK 0x00000004L +#define IH_MMHUB_ERROR__IH_BRESP_11_MASK 0x00000008L +#define IH_MMHUB_ERROR__IH_BUSER_NACK_01_MASK 0x00000020L +#define IH_MMHUB_ERROR__IH_BUSER_NACK_10_MASK 0x00000040L +#define IH_MMHUB_ERROR__IH_BUSER_NACK_11_MASK 0x00000080L +//IH_MEM_POWER_CTRL +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_CTRL_EN__SHIFT 0x0 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_LS_EN__SHIFT 0x1 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DS_EN__SHIFT 0x2 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_SD_EN__SHIFT 0x3 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_IDLE_HYSTERESIS__SHIFT 0x4 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_UP_RECOVER_DELAY__SHIFT 0x8 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DOWN_LS_ENTER_DELAY__SHIFT 0xe +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_CTRL_EN_MASK 0x00000001L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_LS_EN_MASK 0x00000002L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DS_EN_MASK 0x00000004L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_SD_EN_MASK 0x00000008L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_IDLE_HYSTERESIS_MASK 0x00000070L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_UP_RECOVER_DELAY_MASK 0x00003F00L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DOWN_LS_ENTER_DELAY_MASK 0x0000C000L +//IH_REGISTER_LAST_PART2 +#define IH_REGISTER_LAST_PART2__RESERVED__SHIFT 0x0 +#define IH_REGISTER_LAST_PART2__RESERVED_MASK 0xFFFFFFFFL +//SEM_CLK_CTRL +#define SEM_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define SEM_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define SEM_CLK_CTRL__RESERVED__SHIFT 0xc +#define SEM_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define SEM_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define SEM_CLK_CTRL__MEM_CLK_SOFT_OVERRIDE__SHIFT 0x1a +#define SEM_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define SEM_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define SEM_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define SEM_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE__SHIFT 0x1e +#define SEM_CLK_CTRL__REG_CLK_SOFT_OVERRIDE__SHIFT 0x1f +#define SEM_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define SEM_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define SEM_CLK_CTRL__RESERVED_MASK 0x00FFF000L +#define SEM_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define SEM_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define SEM_CLK_CTRL__MEM_CLK_SOFT_OVERRIDE_MASK 0x04000000L +#define SEM_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define SEM_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define SEM_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define SEM_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE_MASK 0x40000000L +#define SEM_CLK_CTRL__REG_CLK_SOFT_OVERRIDE_MASK 0x80000000L +//SEM_UTC_CREDIT +#define SEM_UTC_CREDIT__UTCL2_CREDIT__SHIFT 0x0 +#define SEM_UTC_CREDIT__WATERMARK__SHIFT 0x8 +#define SEM_UTC_CREDIT__UTCL2_CREDIT_MASK 0x0000001FL +#define SEM_UTC_CREDIT__WATERMARK_MASK 0x00000F00L +//SEM_UTC_CONFIG +#define SEM_UTC_CONFIG__USE_MTYPE__SHIFT 0x0 +#define SEM_UTC_CONFIG__FORCE_SNOOP__SHIFT 0x3 +#define SEM_UTC_CONFIG__FORCE_GCC__SHIFT 0x4 +#define SEM_UTC_CONFIG__USE_PT_SNOOP__SHIFT 0x5 +#define SEM_UTC_CONFIG__USE_MTYPE_MASK 0x00000007L +#define SEM_UTC_CONFIG__FORCE_SNOOP_MASK 0x00000008L +#define SEM_UTC_CONFIG__FORCE_GCC_MASK 0x00000010L +#define SEM_UTC_CONFIG__USE_PT_SNOOP_MASK 0x00000020L +//SEM_UTCL2_TRAN_EN_LUT +#define SEM_UTCL2_TRAN_EN_LUT__SDMA0_UTCL2_EN__SHIFT 0x0 +#define SEM_UTCL2_TRAN_EN_LUT__SDMA1_UTCL2_EN__SHIFT 0x1 +#define SEM_UTCL2_TRAN_EN_LUT__UVD_UTCL2_EN__SHIFT 0x2 +#define SEM_UTCL2_TRAN_EN_LUT__VCE0_UTCL2_EN__SHIFT 0x3 +#define SEM_UTCL2_TRAN_EN_LUT__ACP_UTCL2_EN__SHIFT 0x4 +#define SEM_UTCL2_TRAN_EN_LUT__ISP_UTCL2_EN__SHIFT 0x5 +#define SEM_UTCL2_TRAN_EN_LUT__VCE1_UTCL2_EN__SHIFT 0x6 +#define SEM_UTCL2_TRAN_EN_LUT__VP8_UTCL2_EN__SHIFT 0x7 +#define SEM_UTCL2_TRAN_EN_LUT__UVD1_UTCL2_EN__SHIFT 0x8 +#define SEM_UTCL2_TRAN_EN_LUT__RESERVED__SHIFT 0x9 +#define SEM_UTCL2_TRAN_EN_LUT__CP_UTCL2_EN__SHIFT 0x1f +#define SEM_UTCL2_TRAN_EN_LUT__SDMA0_UTCL2_EN_MASK 0x00000001L +#define SEM_UTCL2_TRAN_EN_LUT__SDMA1_UTCL2_EN_MASK 0x00000002L +#define SEM_UTCL2_TRAN_EN_LUT__UVD_UTCL2_EN_MASK 0x00000004L +#define SEM_UTCL2_TRAN_EN_LUT__VCE0_UTCL2_EN_MASK 0x00000008L +#define SEM_UTCL2_TRAN_EN_LUT__ACP_UTCL2_EN_MASK 0x00000010L +#define SEM_UTCL2_TRAN_EN_LUT__ISP_UTCL2_EN_MASK 0x00000020L +#define SEM_UTCL2_TRAN_EN_LUT__VCE1_UTCL2_EN_MASK 0x00000040L +#define SEM_UTCL2_TRAN_EN_LUT__VP8_UTCL2_EN_MASK 0x00000080L +#define SEM_UTCL2_TRAN_EN_LUT__UVD1_UTCL2_EN_MASK 0x00000100L +#define SEM_UTCL2_TRAN_EN_LUT__RESERVED_MASK 0x7FFFFE00L +#define SEM_UTCL2_TRAN_EN_LUT__CP_UTCL2_EN_MASK 0x80000000L +//SEM_MCIF_CONFIG +#define SEM_MCIF_CONFIG__MC_REQ_SWAP__SHIFT 0x0 +#define SEM_MCIF_CONFIG__MC_WRREQ_CREDIT__SHIFT 0x2 +#define SEM_MCIF_CONFIG__MC_RDREQ_CREDIT__SHIFT 0x8 +#define SEM_MCIF_CONFIG__MC_REQ_SWAP_MASK 0x00000003L +#define SEM_MCIF_CONFIG__MC_WRREQ_CREDIT_MASK 0x000000FCL +#define SEM_MCIF_CONFIG__MC_RDREQ_CREDIT_MASK 0x00003F00L +//SEM_PERFMON_CNTL +#define SEM_PERFMON_CNTL__PERF_ENABLE0__SHIFT 0x0 +#define SEM_PERFMON_CNTL__PERF_CLEAR0__SHIFT 0x1 +#define SEM_PERFMON_CNTL__PERF_SEL0__SHIFT 0x2 +#define SEM_PERFMON_CNTL__PERF_ENABLE1__SHIFT 0xa +#define SEM_PERFMON_CNTL__PERF_CLEAR1__SHIFT 0xb +#define SEM_PERFMON_CNTL__PERF_SEL1__SHIFT 0xc +#define SEM_PERFMON_CNTL__PERF_ENABLE0_MASK 0x00000001L +#define SEM_PERFMON_CNTL__PERF_CLEAR0_MASK 0x00000002L +#define SEM_PERFMON_CNTL__PERF_SEL0_MASK 0x000003FCL +#define SEM_PERFMON_CNTL__PERF_ENABLE1_MASK 0x00000400L +#define SEM_PERFMON_CNTL__PERF_CLEAR1_MASK 0x00000800L +#define SEM_PERFMON_CNTL__PERF_SEL1_MASK 0x000FF000L +//SEM_PERFCOUNTER0_RESULT +#define SEM_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT 0x0 +#define SEM_PERFCOUNTER0_RESULT__PERF_COUNT_MASK 0xFFFFFFFFL +//SEM_PERFCOUNTER1_RESULT +#define SEM_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT 0x0 +#define SEM_PERFCOUNTER1_RESULT__PERF_COUNT_MASK 0xFFFFFFFFL +//SEM_STATUS +#define SEM_STATUS__SEM_IDLE__SHIFT 0x0 +#define SEM_STATUS__SEM_INTERNAL_IDLE__SHIFT 0x1 +#define SEM_STATUS__MC_RDREQ_FIFO_FULL__SHIFT 0x2 +#define SEM_STATUS__MC_WRREQ_FIFO_FULL__SHIFT 0x3 +#define SEM_STATUS__WRITE1_FIFO_FULL__SHIFT 0x4 +#define SEM_STATUS__CHECK0_FIFO_FULL__SHIFT 0x5 +#define SEM_STATUS__MC_RDREQ_PENDING__SHIFT 0x6 +#define SEM_STATUS__MC_WRREQ_PENDING__SHIFT 0x7 +#define SEM_STATUS__SDMA0_MAILBOX_PENDING__SHIFT 0x8 +#define SEM_STATUS__SDMA1_MAILBOX_PENDING__SHIFT 0x9 +#define SEM_STATUS__UVD_MAILBOX_PENDING__SHIFT 0xa +#define SEM_STATUS__VCE_MAILBOX_PENDING__SHIFT 0xb +#define SEM_STATUS__CPG1_MAILBOX_PENDING__SHIFT 0xc +#define SEM_STATUS__CPG2_MAILBOX_PENDING__SHIFT 0xd +#define SEM_STATUS__VCE1_MAILBOX_PENDING__SHIFT 0xe +#define SEM_STATUS__ATC_REQ_PENDING__SHIFT 0xf +#define SEM_STATUS__OUTSTANDING_CLEAN__SHIFT 0x10 +#define SEM_STATUS__INVREQ_FLUSH_VF_MISMATCH__SHIFT 0x11 +#define SEM_STATUS__INVREQ_NONFLUSH_VF_MISMATCH__SHIFT 0x12 +#define SEM_STATUS__INVREQ_CNT_IDLE__SHIFT 0x13 +#define SEM_STATUS__ENTRYLIST_IDLE__SHIFT 0x14 +#define SEM_STATUS__MIF_IDLE__SHIFT 0x15 +#define SEM_STATUS__REGISTER_IDLE__SHIFT 0x16 +#define SEM_STATUS__ATCL2_INVREQ_IDLE__SHIFT 0x17 +#define SEM_STATUS__UVD1_MAILBOX_PENDING__SHIFT 0x18 +#define SEM_STATUS__SWITCH_READY__SHIFT 0x1f +#define SEM_STATUS__SEM_IDLE_MASK 0x00000001L +#define SEM_STATUS__SEM_INTERNAL_IDLE_MASK 0x00000002L +#define SEM_STATUS__MC_RDREQ_FIFO_FULL_MASK 0x00000004L +#define SEM_STATUS__MC_WRREQ_FIFO_FULL_MASK 0x00000008L +#define SEM_STATUS__WRITE1_FIFO_FULL_MASK 0x00000010L +#define SEM_STATUS__CHECK0_FIFO_FULL_MASK 0x00000020L +#define SEM_STATUS__MC_RDREQ_PENDING_MASK 0x00000040L +#define SEM_STATUS__MC_WRREQ_PENDING_MASK 0x00000080L +#define SEM_STATUS__SDMA0_MAILBOX_PENDING_MASK 0x00000100L +#define SEM_STATUS__SDMA1_MAILBOX_PENDING_MASK 0x00000200L +#define SEM_STATUS__UVD_MAILBOX_PENDING_MASK 0x00000400L +#define SEM_STATUS__VCE_MAILBOX_PENDING_MASK 0x00000800L +#define SEM_STATUS__CPG1_MAILBOX_PENDING_MASK 0x00001000L +#define SEM_STATUS__CPG2_MAILBOX_PENDING_MASK 0x00002000L +#define SEM_STATUS__VCE1_MAILBOX_PENDING_MASK 0x00004000L +#define SEM_STATUS__ATC_REQ_PENDING_MASK 0x00008000L +#define SEM_STATUS__OUTSTANDING_CLEAN_MASK 0x00010000L +#define SEM_STATUS__INVREQ_FLUSH_VF_MISMATCH_MASK 0x00020000L +#define SEM_STATUS__INVREQ_NONFLUSH_VF_MISMATCH_MASK 0x00040000L +#define SEM_STATUS__INVREQ_CNT_IDLE_MASK 0x00080000L +#define SEM_STATUS__ENTRYLIST_IDLE_MASK 0x00100000L +#define SEM_STATUS__MIF_IDLE_MASK 0x00200000L +#define SEM_STATUS__REGISTER_IDLE_MASK 0x00400000L +#define SEM_STATUS__ATCL2_INVREQ_IDLE_MASK 0x00800000L +#define SEM_STATUS__UVD1_MAILBOX_PENDING_MASK 0x01000000L +#define SEM_STATUS__SWITCH_READY_MASK 0x80000000L +//SEM_MAILBOX_CLIENTCONFIG +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT0__SHIFT 0x0 +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT1__SHIFT 0x3 +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT2__SHIFT 0x6 +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT3__SHIFT 0x9 +#define SEM_MAILBOX_CLIENTCONFIG__SDMA_CLIENT0__SHIFT 0xc +#define SEM_MAILBOX_CLIENTCONFIG__UVD_CLIENT0__SHIFT 0xf +#define SEM_MAILBOX_CLIENTCONFIG__SDMA1_CLIENT0__SHIFT 0x12 +#define SEM_MAILBOX_CLIENTCONFIG__VCE_CLIENT0__SHIFT 0x15 +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT0_MASK 0x00000007L +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT1_MASK 0x00000038L +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT2_MASK 0x000001C0L +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT3_MASK 0x00000E00L +#define SEM_MAILBOX_CLIENTCONFIG__SDMA_CLIENT0_MASK 0x00007000L +#define SEM_MAILBOX_CLIENTCONFIG__UVD_CLIENT0_MASK 0x00038000L +#define SEM_MAILBOX_CLIENTCONFIG__SDMA1_CLIENT0_MASK 0x001C0000L +#define SEM_MAILBOX_CLIENTCONFIG__VCE_CLIENT0_MASK 0x00E00000L +//SEM_MAILBOX +#define SEM_MAILBOX__HOSTPORT__SHIFT 0x0 +#define SEM_MAILBOX__RESERVED__SHIFT 0x10 +#define SEM_MAILBOX__HOSTPORT_MASK 0x0000FFFFL +#define SEM_MAILBOX__RESERVED_MASK 0xFFFF0000L +//SEM_MAILBOX_CONTROL +#define SEM_MAILBOX_CONTROL__HOSTPORT_ENABLE__SHIFT 0x0 +#define SEM_MAILBOX_CONTROL__RESERVED__SHIFT 0x10 +#define SEM_MAILBOX_CONTROL__HOSTPORT_ENABLE_MASK 0x0000FFFFL +#define SEM_MAILBOX_CONTROL__RESERVED_MASK 0xFFFF0000L +//SEM_CHICKEN_BITS +#define SEM_CHICKEN_BITS__VMID_PIPELINE_EN__SHIFT 0x0 +#define SEM_CHICKEN_BITS__ENTRY_PIPELINE_EN__SHIFT 0x1 +#define SEM_CHICKEN_BITS__CHECK_COUNTER_EN__SHIFT 0x2 +#define SEM_CHICKEN_BITS__ECC_BEHAVIOR__SHIFT 0x3 +#define SEM_CHICKEN_BITS__PHY_TRAN_EN__SHIFT 0x6 +#define SEM_CHICKEN_BITS__ADDR_CMP_UNTRAN_EN__SHIFT 0x7 +#define SEM_CHICKEN_BITS__IDLE_COUNTER_INDEX__SHIFT 0x8 +#define SEM_CHICKEN_BITS__OUTSTANDING_CLEAN_COUNTER_INDEX__SHIFT 0xa +#define SEM_CHICKEN_BITS__ATCL2_BUS_ID__SHIFT 0xc +#define SEM_CHICKEN_BITS__ATOMIC_EN__SHIFT 0xe +#define SEM_CHICKEN_BITS__EXTERNAL_ATOMIC_CHECK__SHIFT 0xf +#define SEM_CHICKEN_BITS__CLEAR_MAILBOX__SHIFT 0x10 +#define SEM_CHICKEN_BITS__INVACK_AFTER_OUTSTANDING_CLEAN__SHIFT 0x12 +#define SEM_CHICKEN_BITS__UTC_TAG_CONFLICT_CHECK__SHIFT 0x13 +#define SEM_CHICKEN_BITS__VMID_PIPELINE_EN_MASK 0x00000001L +#define SEM_CHICKEN_BITS__ENTRY_PIPELINE_EN_MASK 0x00000002L +#define SEM_CHICKEN_BITS__CHECK_COUNTER_EN_MASK 0x00000004L +#define SEM_CHICKEN_BITS__ECC_BEHAVIOR_MASK 0x00000018L +#define SEM_CHICKEN_BITS__PHY_TRAN_EN_MASK 0x00000040L +#define SEM_CHICKEN_BITS__ADDR_CMP_UNTRAN_EN_MASK 0x00000080L +#define SEM_CHICKEN_BITS__IDLE_COUNTER_INDEX_MASK 0x00000300L +#define SEM_CHICKEN_BITS__OUTSTANDING_CLEAN_COUNTER_INDEX_MASK 0x00000C00L +#define SEM_CHICKEN_BITS__ATCL2_BUS_ID_MASK 0x00003000L +#define SEM_CHICKEN_BITS__ATOMIC_EN_MASK 0x00004000L +#define SEM_CHICKEN_BITS__EXTERNAL_ATOMIC_CHECK_MASK 0x00008000L +#define SEM_CHICKEN_BITS__CLEAR_MAILBOX_MASK 0x00030000L +#define SEM_CHICKEN_BITS__INVACK_AFTER_OUTSTANDING_CLEAN_MASK 0x00040000L +#define SEM_CHICKEN_BITS__UTC_TAG_CONFLICT_CHECK_MASK 0x00080000L +//SEM_MAILBOX_CLIENTCONFIG_EXTRA +#define SEM_MAILBOX_CLIENTCONFIG_EXTRA__VCE1_CLIENT0__SHIFT 0x0 +#define SEM_MAILBOX_CLIENTCONFIG_EXTRA__UVD1_CLIENT0__SHIFT 0x4 +#define SEM_MAILBOX_CLIENTCONFIG_EXTRA__VCE1_CLIENT0_MASK 0x0000000FL +#define SEM_MAILBOX_CLIENTCONFIG_EXTRA__UVD1_CLIENT0_MASK 0x000000F0L +//SEM_GPU_IOV_VIOLATION_LOG +#define SEM_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT 0x0 +#define SEM_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT 0x1 +#define SEM_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT 0x2 +#define SEM_GPU_IOV_VIOLATION_LOG__OPCODE__SHIFT 0x12 +#define SEM_GPU_IOV_VIOLATION_LOG__VF__SHIFT 0x13 +#define SEM_GPU_IOV_VIOLATION_LOG__VF_ID__SHIFT 0x14 +#define SEM_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT 0x18 +#define SEM_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK 0x00000001L +#define SEM_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK 0x00000002L +#define SEM_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK 0x0003FFFCL +#define SEM_GPU_IOV_VIOLATION_LOG__OPCODE_MASK 0x00040000L +#define SEM_GPU_IOV_VIOLATION_LOG__VF_MASK 0x00080000L +#define SEM_GPU_IOV_VIOLATION_LOG__VF_ID_MASK 0x00F00000L +#define SEM_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK 0xFF000000L +//SEM_OUTSTANDING_THRESHOLD +#define SEM_OUTSTANDING_THRESHOLD__VALUE__SHIFT 0x0 +#define SEM_OUTSTANDING_THRESHOLD__VALUE_MASK 0x000000FFL +//SEM_MEM_POWER_CTRL +#define SEM_MEM_POWER_CTRL__MEM_POWER_CTRL_EN__SHIFT 0x0 +#define SEM_MEM_POWER_CTRL__MEM_POWER_LS_EN__SHIFT 0x1 +#define SEM_MEM_POWER_CTRL__MEM_POWER_DS_EN__SHIFT 0x2 +#define SEM_MEM_POWER_CTRL__MEM_POWER_SD_EN__SHIFT 0x3 +#define SEM_MEM_POWER_CTRL__MEM_IDLE_HYSTERESIS__SHIFT 0x4 +#define SEM_MEM_POWER_CTRL__MEM_POWER_UP_RECOVER_DELAY__SHIFT 0x8 +#define SEM_MEM_POWER_CTRL__MEM_POWER_DOWN_LS_ENTER_DELAY__SHIFT 0xe +#define SEM_MEM_POWER_CTRL__MEM_POWER_CTRL_EN_MASK 0x00000001L +#define SEM_MEM_POWER_CTRL__MEM_POWER_LS_EN_MASK 0x00000002L +#define SEM_MEM_POWER_CTRL__MEM_POWER_DS_EN_MASK 0x00000004L +#define SEM_MEM_POWER_CTRL__MEM_POWER_SD_EN_MASK 0x00000008L +#define SEM_MEM_POWER_CTRL__MEM_IDLE_HYSTERESIS_MASK 0x00000070L +#define SEM_MEM_POWER_CTRL__MEM_POWER_UP_RECOVER_DELAY_MASK 0x00003F00L +#define SEM_MEM_POWER_CTRL__MEM_POWER_DOWN_LS_ENTER_DELAY_MASK 0x0000C000L +//SEM_REGISTER_LAST_PART2 +#define SEM_REGISTER_LAST_PART2__RESERVED__SHIFT 0x0 +#define SEM_REGISTER_LAST_PART2__RESERVED_MASK 0xFFFFFFFFL +//IH_ACTIVE_FCN_ID +#define IH_ACTIVE_FCN_ID__VF_ID__SHIFT 0x0 +#define IH_ACTIVE_FCN_ID__RESERVED__SHIFT 0x4 +#define IH_ACTIVE_FCN_ID__PF_VF__SHIFT 0x1f +#define IH_ACTIVE_FCN_ID__VF_ID_MASK 0x0000000FL +#define IH_ACTIVE_FCN_ID__RESERVED_MASK 0x7FFFFFF0L +#define IH_ACTIVE_FCN_ID__PF_VF_MASK 0x80000000L +//IH_VIRT_RESET_REQ +#define IH_VIRT_RESET_REQ__VF__SHIFT 0x0 +#define IH_VIRT_RESET_REQ__PF__SHIFT 0x1f +#define IH_VIRT_RESET_REQ__VF_MASK 0x0000FFFFL +#define IH_VIRT_RESET_REQ__PF_MASK 0x80000000L +//IH_CLIENT_CFG +#define IH_CLIENT_CFG__TOTAL_CLIENT_NUM__SHIFT 0x0 +#define IH_CLIENT_CFG__TOTAL_CLIENT_NUM_MASK 0x0000001FL +//IH_CLIENT_CFG_INDEX +#define IH_CLIENT_CFG_INDEX__INDEX__SHIFT 0x0 +#define IH_CLIENT_CFG_INDEX__INDEX_MASK 0x0000001FL +//IH_CLIENT_CFG_DATA +#define IH_CLIENT_CFG_DATA__CREDIT_RETURN_ADDR__SHIFT 0x0 +#define IH_CLIENT_CFG_DATA__CLIENT_TYPE__SHIFT 0x12 +#define IH_CLIENT_CFG_DATA__RING_ID__SHIFT 0x14 +#define IH_CLIENT_CFG_DATA__VF_RB_SELECT__SHIFT 0x16 +#define IH_CLIENT_CFG_DATA__OVERWRITE_RING_ID_WITH_ACTIVE_FCN_ID__SHIFT 0x18 +#define IH_CLIENT_CFG_DATA__CREDIT_RETURN_ADDR_MASK 0x0003FFFFL +#define IH_CLIENT_CFG_DATA__CLIENT_TYPE_MASK 0x000C0000L +#define IH_CLIENT_CFG_DATA__RING_ID_MASK 0x00300000L +#define IH_CLIENT_CFG_DATA__VF_RB_SELECT_MASK 0x00C00000L +#define IH_CLIENT_CFG_DATA__OVERWRITE_RING_ID_WITH_ACTIVE_FCN_ID_MASK 0x01000000L +//IH_CID_REMAP_INDEX +#define IH_CID_REMAP_INDEX__INDEX__SHIFT 0x0 +#define IH_CID_REMAP_INDEX__INDEX_MASK 0x00000003L +//IH_CID_REMAP_DATA +#define IH_CID_REMAP_DATA__CLIENT_ID__SHIFT 0x0 +#define IH_CID_REMAP_DATA__INITIATOR_ID__SHIFT 0x8 +#define IH_CID_REMAP_DATA__CLIENT_ID_REMAP__SHIFT 0x10 +#define IH_CID_REMAP_DATA__CLIENT_ID_MASK 0x000000FFL +#define IH_CID_REMAP_DATA__INITIATOR_ID_MASK 0x0000FF00L +#define IH_CID_REMAP_DATA__CLIENT_ID_REMAP_MASK 0x00FF0000L +//IH_CHICKEN +#define IH_CHICKEN__ACTIVE_FCN_ID_PROT_ENABLE__SHIFT 0x0 +#define IH_CHICKEN__MC_SPACE_FBPA_ENABLE__SHIFT 0x3 +#define IH_CHICKEN__MC_SPACE_GPA_ENABLE__SHIFT 0x4 +#define IH_CHICKEN__ACTIVE_FCN_ID_PROT_ENABLE_MASK 0x00000001L +#define IH_CHICKEN__MC_SPACE_FBPA_ENABLE_MASK 0x00000008L +#define IH_CHICKEN__MC_SPACE_GPA_ENABLE_MASK 0x00000010L +//IH_MMHUB_CNTL +#define IH_MMHUB_CNTL__UNITID__SHIFT 0x0 +#define IH_MMHUB_CNTL__IV_TLVL__SHIFT 0x8 +#define IH_MMHUB_CNTL__WPTR_WB_TLVL__SHIFT 0xc +#define IH_MMHUB_CNTL__UNITID_MASK 0x0000003FL +#define IH_MMHUB_CNTL__IV_TLVL_MASK 0x00000700L +#define IH_MMHUB_CNTL__WPTR_WB_TLVL_MASK 0x00007000L +//IH_INT_DROP_CNTL +#define IH_INT_DROP_CNTL__INT_DROP_EN__SHIFT 0x0 +#define IH_INT_DROP_CNTL__CLIENT_ID_MATCH_EN__SHIFT 0x1 +#define IH_INT_DROP_CNTL__SOURCE_ID_MATCH_EN__SHIFT 0x2 +#define IH_INT_DROP_CNTL__VF_ID_MATCH_EN__SHIFT 0x3 +#define IH_INT_DROP_CNTL__VF_MATCH_EN__SHIFT 0x4 +#define IH_INT_DROP_CNTL__CONTEXT_ID_MATCH_EN__SHIFT 0x5 +#define IH_INT_DROP_CNTL__INT_DROP_MODE__SHIFT 0x6 +#define IH_INT_DROP_CNTL__UTCL2_RETRY_INT_DROP_EN__SHIFT 0x8 +#define IH_INT_DROP_CNTL__INT_DROPPED__SHIFT 0x10 +#define IH_INT_DROP_CNTL__INT_DROP_EN_MASK 0x00000001L +#define IH_INT_DROP_CNTL__CLIENT_ID_MATCH_EN_MASK 0x00000002L +#define IH_INT_DROP_CNTL__SOURCE_ID_MATCH_EN_MASK 0x00000004L +#define IH_INT_DROP_CNTL__VF_ID_MATCH_EN_MASK 0x00000008L +#define IH_INT_DROP_CNTL__VF_MATCH_EN_MASK 0x00000010L +#define IH_INT_DROP_CNTL__CONTEXT_ID_MATCH_EN_MASK 0x00000020L +#define IH_INT_DROP_CNTL__INT_DROP_MODE_MASK 0x000000C0L +#define IH_INT_DROP_CNTL__UTCL2_RETRY_INT_DROP_EN_MASK 0x00000100L +#define IH_INT_DROP_CNTL__INT_DROPPED_MASK 0x00010000L +//IH_INT_DROP_MATCH_VALUE0 +#define IH_INT_DROP_MATCH_VALUE0__CLIENT_ID_MATCH_VALUE__SHIFT 0x0 +#define IH_INT_DROP_MATCH_VALUE0__SOURCE_ID_MATCH_VALUE__SHIFT 0x8 +#define IH_INT_DROP_MATCH_VALUE0__VF_ID_MATCH_VALUE__SHIFT 0x10 +#define IH_INT_DROP_MATCH_VALUE0__VF_MATCH_VALUE__SHIFT 0x17 +#define IH_INT_DROP_MATCH_VALUE0__CONTEXT_ID_39_32_MATCH_VALUE__SHIFT 0x18 +#define IH_INT_DROP_MATCH_VALUE0__CLIENT_ID_MATCH_VALUE_MASK 0x000000FFL +#define IH_INT_DROP_MATCH_VALUE0__SOURCE_ID_MATCH_VALUE_MASK 0x0000FF00L +#define IH_INT_DROP_MATCH_VALUE0__VF_ID_MATCH_VALUE_MASK 0x000F0000L +#define IH_INT_DROP_MATCH_VALUE0__VF_MATCH_VALUE_MASK 0x00800000L +#define IH_INT_DROP_MATCH_VALUE0__CONTEXT_ID_39_32_MATCH_VALUE_MASK 0xFF000000L +//IH_INT_DROP_MATCH_VALUE1 +#define IH_INT_DROP_MATCH_VALUE1__CONTEXT_ID_31_0_MATCH_VALUE__SHIFT 0x0 +#define IH_INT_DROP_MATCH_VALUE1__CONTEXT_ID_31_0_MATCH_VALUE_MASK 0xFFFFFFFFL +//IH_INT_DROP_MATCH_MASK0 +#define IH_INT_DROP_MATCH_MASK0__CLIENT_ID_MATCH_MASK__SHIFT 0x0 +#define IH_INT_DROP_MATCH_MASK0__SOURCE_ID_MATCH_MASK__SHIFT 0x8 +#define IH_INT_DROP_MATCH_MASK0__VF_ID_MATCH_MASK__SHIFT 0x10 +#define IH_INT_DROP_MATCH_MASK0__VF_MATCH_MASK__SHIFT 0x17 +#define IH_INT_DROP_MATCH_MASK0__CONTEXT_ID_39_32_MATCH_MASK__SHIFT 0x18 +#define IH_INT_DROP_MATCH_MASK0__CLIENT_ID_MATCH_MASK_MASK 0x000000FFL +#define IH_INT_DROP_MATCH_MASK0__SOURCE_ID_MATCH_MASK_MASK 0x0000FF00L +#define IH_INT_DROP_MATCH_MASK0__VF_ID_MATCH_MASK_MASK 0x000F0000L +#define IH_INT_DROP_MATCH_MASK0__VF_MATCH_MASK_MASK 0x00800000L +#define IH_INT_DROP_MATCH_MASK0__CONTEXT_ID_39_32_MATCH_MASK_MASK 0xFF000000L +//IH_INT_DROP_MATCH_MASK1 +#define IH_INT_DROP_MATCH_MASK1__CONTEXT_ID_31_0_MATCH_MASK__SHIFT 0x0 +#define IH_INT_DROP_MATCH_MASK1__CONTEXT_ID_31_0_MATCH_MASK_MASK 0xFFFFFFFFL +//IH_REGISTER_LAST_PART1 +#define IH_REGISTER_LAST_PART1__RESERVED__SHIFT 0x0 +#define IH_REGISTER_LAST_PART1__RESERVED_MASK 0xFFFFFFFFL +//SEM_ACTIVE_FCN_ID +#define SEM_ACTIVE_FCN_ID__VFID__SHIFT 0x0 +#define SEM_ACTIVE_FCN_ID__VF__SHIFT 0x1f +#define SEM_ACTIVE_FCN_ID__VFID_MASK 0x0000000FL +#define SEM_ACTIVE_FCN_ID__VF_MASK 0x80000000L +//SEM_VIRT_RESET_REQ +#define SEM_VIRT_RESET_REQ__VF__SHIFT 0x0 +#define SEM_VIRT_RESET_REQ__PF__SHIFT 0x1f +#define SEM_VIRT_RESET_REQ__VF_MASK 0x0000FFFFL +#define SEM_VIRT_RESET_REQ__PF_MASK 0x80000000L +//SEM_RESP_SDMA0 +#define SEM_RESP_SDMA0__ADDR__SHIFT 0x2 +#define SEM_RESP_SDMA0__ADDR_MASK 0x000FFFFCL +//SEM_RESP_SDMA1 +#define SEM_RESP_SDMA1__ADDR__SHIFT 0x2 +#define SEM_RESP_SDMA1__ADDR_MASK 0x000FFFFCL +//SEM_RESP_UVD +#define SEM_RESP_UVD__ADDR__SHIFT 0x2 +#define SEM_RESP_UVD__ADDR_MASK 0x000FFFFCL +//SEM_RESP_VCE_0 +#define SEM_RESP_VCE_0__ADDR__SHIFT 0x2 +#define SEM_RESP_VCE_0__ADDR_MASK 0x000FFFFCL +//SEM_RESP_ACP +#define SEM_RESP_ACP__ADDR__SHIFT 0x2 +#define SEM_RESP_ACP__ADDR_MASK 0x000FFFFCL +//SEM_RESP_ISP +#define SEM_RESP_ISP__ADDR__SHIFT 0x2 +#define SEM_RESP_ISP__ADDR_MASK 0x000FFFFCL +//SEM_RESP_VCE_1 +#define SEM_RESP_VCE_1__ADDR__SHIFT 0x2 +#define SEM_RESP_VCE_1__ADDR_MASK 0x000FFFFCL +//SEM_RESP_VP8 +#define SEM_RESP_VP8__ADDR__SHIFT 0x2 +#define SEM_RESP_VP8__ADDR_MASK 0x000FFFFCL +//SEM_RESP_GC +#define SEM_RESP_GC__ADDR__SHIFT 0x2 +#define SEM_RESP_GC__ADDR_MASK 0x000FFFFCL +//SEM_RESP_UVD_1 +#define SEM_RESP_UVD_1__ADDR__SHIFT 0x2 +#define SEM_RESP_UVD_1__ADDR_MASK 0x000FFFFCL +//SEM_CID_REMAP_INDEX +#define SEM_CID_REMAP_INDEX__INDEX__SHIFT 0x0 +#define SEM_CID_REMAP_INDEX__INDEX_MASK 0x00000003L +//SEM_CID_REMAP_DATA +#define SEM_CID_REMAP_DATA__CLIENT_ID__SHIFT 0x0 +#define SEM_CID_REMAP_DATA__INITIATOR_ID__SHIFT 0x8 +#define SEM_CID_REMAP_DATA__CLIENT_ID_REMAP__SHIFT 0x10 +#define SEM_CID_REMAP_DATA__CLIENT_ID_MASK 0x000000FFL +#define SEM_CID_REMAP_DATA__INITIATOR_ID_MASK 0x0000FF00L +#define SEM_CID_REMAP_DATA__CLIENT_ID_REMAP_MASK 0x00FF0000L +//SEM_ATOMIC_OP_LUT +#define SEM_ATOMIC_OP_LUT__SIGNAL_NORMAL__SHIFT 0x0 +#define SEM_ATOMIC_OP_LUT__SIGNAL_WRITE1__SHIFT 0x7 +#define SEM_ATOMIC_OP_LUT__WAIT_NORMAL__SHIFT 0xe +#define SEM_ATOMIC_OP_LUT__WAIT_CHECK0__SHIFT 0x15 +#define SEM_ATOMIC_OP_LUT__SIGNAL_NORMAL_MASK 0x0000007FL +#define SEM_ATOMIC_OP_LUT__SIGNAL_WRITE1_MASK 0x00003F80L +#define SEM_ATOMIC_OP_LUT__WAIT_NORMAL_MASK 0x001FC000L +#define SEM_ATOMIC_OP_LUT__WAIT_CHECK0_MASK 0x0FE00000L +//SEM_EDC_CONFIG +#define SEM_EDC_CONFIG__WRITE_DIS__SHIFT 0x0 +#define SEM_EDC_CONFIG__DIS_EDC__SHIFT 0x1 +#define SEM_EDC_CONFIG__WRITE_DIS_MASK 0x00000001L +#define SEM_EDC_CONFIG__DIS_EDC_MASK 0x00000002L +//SEM_CHICKEN_BITS2 +#define SEM_CHICKEN_BITS2__ACTIVE_FCN_ID_PROT_ENABLE__SHIFT 0x0 +#define SEM_CHICKEN_BITS2__MM_CLIENT_USE_CONFIG_VFID__SHIFT 0x1 +#define SEM_CHICKEN_BITS2__ACTIVE_FCN_ID_PROT_ENABLE_MASK 0x00000001L +#define SEM_CHICKEN_BITS2__MM_CLIENT_USE_CONFIG_VFID_MASK 0x00000002L +//SEM_MMHUB_CNTL +#define SEM_MMHUB_CNTL__UNIT_ID__SHIFT 0x0 +#define SEM_MMHUB_CNTL__TLVL_VALUE__SHIFT 0x8 +#define SEM_MMHUB_CNTL__UNIT_ID_MASK 0x0000003FL +#define SEM_MMHUB_CNTL__TLVL_VALUE_MASK 0x00000700L +//SEM_REGISTER_LAST_PART1 +#define SEM_REGISTER_LAST_PART1__RESERVED__SHIFT 0x0 +#define SEM_REGISTER_LAST_PART1__RESERVED_MASK 0xFFFFFFFFL + +#endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_offset.h b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_offset.h new file mode 100644 index 000000000000..55facadea54b --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_offset.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef _smuio_11_0_6_OFFSET_HEADER +#define _smuio_11_0_6_OFFSET_HEADER + + + +// addressBlock: smuio_smuio_SmuSmuioDec +// base address: 0x5a000 +#define mmCGTT_ROM_CLK_CTRL0 0x00e4 +#define mmCGTT_ROM_CLK_CTRL0_BASE_IDX 0 +#define mmROM_INDEX 0x00e5 +#define mmROM_INDEX_BASE_IDX 0 +#define mmROM_DATA 0x00e6 +#define mmROM_DATA_BASE_IDX 0 + +#endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_sh_mask.h new file mode 100644 index 000000000000..7d6a2fac2839 --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_sh_mask.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef _smuio_11_0_6_SH_MASK_HEADER +#define _smuio_11_0_6_SH_MASK_HEADER + + +//CGTT_ROM_CLK_CTRL0 +#define CGTT_ROM_CLK_CTRL0__ON_DELAY__SHIFT 0x0 +#define CGTT_ROM_CLK_CTRL0__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_ROM_CLK_CTRL0__ON_DELAY_MASK 0x0000000FL +#define CGTT_ROM_CLK_CTRL0__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK 0x40000000L +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK 0x80000000L +//ROM_INDEX +#define ROM_INDEX__ROM_INDEX__SHIFT 0x0 +#define ROM_INDEX__ROM_INDEX_MASK 0x01FFFFFFL +//ROM_DATA +#define ROM_DATA__ROM_DATA__SHIFT 0x0 +#define ROM_DATA__ROM_DATA_MASK 0xFFFFFFFFL + +#endif diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h index f775aac6c1bd..a41875ac5dfb 100644 --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h @@ -103,6 +103,7 @@ enum pp_clock_type { enum amd_pp_sensors { AMDGPU_PP_SENSOR_GFX_SCLK = 0, + AMDGPU_PP_SENSOR_CPU_CLK, AMDGPU_PP_SENSOR_VDDNB, AMDGPU_PP_SENSOR_VDDGFX, AMDGPU_PP_SENSOR_UVD_VCLK, @@ -155,9 +156,11 @@ enum { enum PP_OD_DPM_TABLE_COMMAND { PP_OD_EDIT_SCLK_VDDC_TABLE, PP_OD_EDIT_MCLK_VDDC_TABLE, + PP_OD_EDIT_CCLK_VDDC_TABLE, PP_OD_EDIT_VDDC_CURVE, PP_OD_RESTORE_DEFAULT_TABLE, - PP_OD_COMMIT_DPM_TABLE + PP_OD_COMMIT_DPM_TABLE, + PP_OD_EDIT_VDDGFX_OFFSET }; struct pp_states_info { diff --git a/drivers/gpu/drm/amd/include/renoir_ip_offset.h b/drivers/gpu/drm/amd/include/renoir_ip_offset.h index 07633e22e99a..7dff85c81e5a 100644 --- a/drivers/gpu/drm/amd/include/renoir_ip_offset.h +++ b/drivers/gpu/drm/amd/include/renoir_ip_offset.h @@ -33,7 +33,7 @@ struct IP_BASE_INSTANCE struct IP_BASE { struct IP_BASE_INSTANCE instance[MAX_INSTANCE]; -}; +} __maybe_unused; static const struct IP_BASE ACP_BASE ={ { { { 0x02403800, 0x00480000, 0, 0, 0 } }, diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index 7b6ef05a1d35..5fa65f191a37 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -36,6 +36,7 @@ #include #include #include +#include #include "hwmgr.h" static const struct cg_flag_name clocks[] = { @@ -730,11 +731,18 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, * * - minimum and maximum engine clock labeled OD_SCLK * - * - maximum memory clock labeled OD_MCLK + * - minimum(not available for Vega20 and Navi1x) and maximum memory + * clock labeled OD_MCLK * * - three points labeled OD_VDDC_CURVE. * They can be used to calibrate the sclk voltage curve. * + * - voltage offset(in mV) applied on target voltage calculation. + * This is available for Sienna Cichlid, Navy Flounder and Dimgrey + * Cavefish. For these ASICs, the target voltage calculation can be + * illustrated by "voltage = voltage calculated from v/f curve + + * overdrive vddgfx offset" + * * - a list of valid ranges for sclk, mclk, and voltage curve points * labeled OD_RANGE * @@ -755,6 +763,11 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, * 600mV. "vc 2 1000 1000" will update point3 with clock set * as 1000Mhz and voltage 1000mV. * + * To update the voltage offset applied for gfxclk/voltage calculation, + * enter the new value by writing a string that contains "vo offset". + * This is supported by Sienna Cichlid, Navy Flounder and Dimgrey Cavefish. + * And the offset can be a positive or negative value. + * * - When you have edited all of the states as needed, write "c" (commit) * to the file to commit your changes * @@ -787,6 +800,8 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, if (*buf == 's') type = PP_OD_EDIT_SCLK_VDDC_TABLE; + else if (*buf == 'p') + type = PP_OD_EDIT_CCLK_VDDC_TABLE; else if (*buf == 'm') type = PP_OD_EDIT_MCLK_VDDC_TABLE; else if(*buf == 'r') @@ -795,6 +810,8 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, type = PP_OD_COMMIT_DPM_TABLE; else if (!strncmp(buf, "vc", 2)) type = PP_OD_EDIT_VDDC_CURVE; + else if (!strncmp(buf, "vo", 2)) + type = PP_OD_EDIT_VDDGFX_OFFSET; else return -EINVAL; @@ -802,12 +819,14 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, tmp_str = buf_cpy; - if (type == PP_OD_EDIT_VDDC_CURVE) + if ((type == PP_OD_EDIT_VDDC_CURVE) || + (type == PP_OD_EDIT_VDDGFX_OFFSET)) tmp_str++; while (isspace(*++tmp_str)); - while (tmp_str[0]) { - sub_str = strsep(&tmp_str, delimiter); + while ((sub_str = strsep(&tmp_str, delimiter)) != NULL) { + if (strlen(sub_str) == 0) + continue; ret = kstrtol(sub_str, 0, ¶meter[parameter_size]); if (ret) return -EINVAL; @@ -898,7 +917,9 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev, size = smu_print_clk_levels(&adev->smu, SMU_OD_SCLK, buf); size += smu_print_clk_levels(&adev->smu, SMU_OD_MCLK, buf+size); size += smu_print_clk_levels(&adev->smu, SMU_OD_VDDC_CURVE, buf+size); + size += smu_print_clk_levels(&adev->smu, SMU_OD_VDDGFX_OFFSET, buf+size); size += smu_print_clk_levels(&adev->smu, SMU_OD_RANGE, buf+size); + size += smu_print_clk_levels(&adev->smu, SMU_OD_CCLK, buf+size); } else if (adev->powerplay.pp_funcs->print_clock_levels) { size = amdgpu_dpm_print_clock_levels(adev, OD_SCLK, buf); size += amdgpu_dpm_print_clock_levels(adev, OD_MCLK, buf+size); @@ -1074,7 +1095,7 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev, static ssize_t amdgpu_read_mask(const char *buf, size_t count, uint32_t *mask) { int ret; - long level; + unsigned long level; char *sub_str = NULL; char *tmp; char buf_cpy[AMDGPU_MASK_BUF_MAX + 1]; @@ -1087,11 +1108,10 @@ static ssize_t amdgpu_read_mask(const char *buf, size_t count, uint32_t *mask) memcpy(buf_cpy, buf, bytes); buf_cpy[bytes] = '\0'; tmp = buf_cpy; - while (tmp[0]) { - sub_str = strsep(&tmp, delimiter); + while ((sub_str = strsep(&tmp, delimiter)) != NULL) { if (strlen(sub_str)) { - ret = kstrtol(sub_str, 0, &level); - if (ret) + ret = kstrtoul(sub_str, 0, &level); + if (ret || level > 31) return -EINVAL; *mask |= 1 << level; } else @@ -1346,6 +1366,138 @@ static ssize_t amdgpu_set_pp_dpm_fclk(struct device *dev, return count; } +static ssize_t amdgpu_get_pp_dpm_vclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + ssize_t size; + int ret; + + if (amdgpu_in_reset(adev)) + return -EPERM; + + ret = pm_runtime_get_sync(ddev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(ddev->dev); + return ret; + } + + if (is_support_sw_smu(adev)) + size = smu_print_clk_levels(&adev->smu, SMU_VCLK, buf); + else + size = snprintf(buf, PAGE_SIZE, "\n"); + + pm_runtime_mark_last_busy(ddev->dev); + pm_runtime_put_autosuspend(ddev->dev); + + return size; +} + +static ssize_t amdgpu_set_pp_dpm_vclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + int ret; + uint32_t mask = 0; + + if (amdgpu_in_reset(adev)) + return -EPERM; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + ret = pm_runtime_get_sync(ddev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(ddev->dev); + return ret; + } + + if (is_support_sw_smu(adev)) + ret = smu_force_clk_levels(&adev->smu, SMU_VCLK, mask); + else + ret = 0; + + pm_runtime_mark_last_busy(ddev->dev); + pm_runtime_put_autosuspend(ddev->dev); + + if (ret) + return -EINVAL; + + return count; +} + +static ssize_t amdgpu_get_pp_dpm_dclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + ssize_t size; + int ret; + + if (amdgpu_in_reset(adev)) + return -EPERM; + + ret = pm_runtime_get_sync(ddev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(ddev->dev); + return ret; + } + + if (is_support_sw_smu(adev)) + size = smu_print_clk_levels(&adev->smu, SMU_DCLK, buf); + else + size = snprintf(buf, PAGE_SIZE, "\n"); + + pm_runtime_mark_last_busy(ddev->dev); + pm_runtime_put_autosuspend(ddev->dev); + + return size; +} + +static ssize_t amdgpu_set_pp_dpm_dclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + int ret; + uint32_t mask = 0; + + if (amdgpu_in_reset(adev)) + return -EPERM; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + ret = pm_runtime_get_sync(ddev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(ddev->dev); + return ret; + } + + if (is_support_sw_smu(adev)) + ret = smu_force_clk_levels(&adev->smu, SMU_DCLK, mask); + else + ret = 0; + + pm_runtime_mark_last_busy(ddev->dev); + pm_runtime_put_autosuspend(ddev->dev); + + if (ret) + return -EINVAL; + + return count; +} + static ssize_t amdgpu_get_pp_dpm_dcefclk(struct device *dev, struct device_attribute *attr, char *buf) @@ -1717,8 +1869,9 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev, i++; memcpy(buf_cpy, buf, count-i); tmp_str = buf_cpy; - while (tmp_str[0]) { - sub_str = strsep(&tmp_str, delimiter); + while ((sub_str = strsep(&tmp_str, delimiter)) != NULL) { + if (strlen(sub_str) == 0) + continue; ret = kstrtol(sub_str, 0, ¶meter[parameter_size]); if (ret) return -EINVAL; @@ -2025,6 +2178,8 @@ static struct amdgpu_device_attr amdgpu_device_attrs[] = { AMDGPU_DEVICE_ATTR_RW(pp_dpm_mclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(pp_dpm_socclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(pp_dpm_fclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), + AMDGPU_DEVICE_ATTR_RW(pp_dpm_vclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), + AMDGPU_DEVICE_ATTR_RW(pp_dpm_dclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(pp_dpm_dcefclk, ATTR_FLAG_BASIC), AMDGPU_DEVICE_ATTR_RW(pp_dpm_pcie, ATTR_FLAG_BASIC), AMDGPU_DEVICE_ATTR_RW(pp_sclk_od, ATTR_FLAG_BASIC), @@ -2067,7 +2222,8 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_ } else if (DEVICE_ATTR_IS(pp_od_clk_voltage)) { *states = ATTR_STATE_UNSUPPORTED; if ((is_support_sw_smu(adev) && adev->smu.od_enabled) || - (!is_support_sw_smu(adev) && hwmgr->od_enabled)) + (is_support_sw_smu(adev) && adev->smu.is_apu) || + (!is_support_sw_smu(adev) && hwmgr->od_enabled)) *states = ATTR_STATE_SUPPORTED; } else if (DEVICE_ATTR_IS(mem_busy_percent)) { if (adev->flags & AMD_IS_APU || asic_type == CHIP_VEGA10) @@ -2087,6 +2243,12 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_ } else if (DEVICE_ATTR_IS(gpu_metrics)) { if (asic_type < CHIP_VEGA12) *states = ATTR_STATE_UNSUPPORTED; + } else if (DEVICE_ATTR_IS(pp_dpm_vclk)) { + if (!(asic_type == CHIP_VANGOGH)) + *states = ATTR_STATE_UNSUPPORTED; + } else if (DEVICE_ATTR_IS(pp_dpm_dclk)) { + if (!(asic_type == CHIP_VANGOGH)) + *states = ATTR_STATE_UNSUPPORTED; } if (asic_type == CHIP_ARCTURUS) { @@ -2897,7 +3059,8 @@ static ssize_t amdgpu_hwmon_show_power_cap_max(struct device *dev, char *buf) { struct amdgpu_device *adev = dev_get_drvdata(dev); - uint32_t limit = 0; + int limit_type = to_sensor_dev_attr(attr)->index; + uint32_t limit = limit_type << 24; ssize_t size; int r; @@ -2911,7 +3074,7 @@ static ssize_t amdgpu_hwmon_show_power_cap_max(struct device *dev, } if (is_support_sw_smu(adev)) { - smu_get_power_limit(&adev->smu, &limit, true); + smu_get_power_limit(&adev->smu, &limit, SMU_PPT_LIMIT_MAX); size = snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000); } else if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_power_limit) { adev->powerplay.pp_funcs->get_power_limit(adev->powerplay.pp_handle, &limit, true); @@ -2931,7 +3094,8 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev, char *buf) { struct amdgpu_device *adev = dev_get_drvdata(dev); - uint32_t limit = 0; + int limit_type = to_sensor_dev_attr(attr)->index; + uint32_t limit = limit_type << 24; ssize_t size; int r; @@ -2945,7 +3109,7 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev, } if (is_support_sw_smu(adev)) { - smu_get_power_limit(&adev->smu, &limit, false); + smu_get_power_limit(&adev->smu, &limit, SMU_PPT_LIMIT_CURRENT); size = snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000); } else if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_power_limit) { adev->powerplay.pp_funcs->get_power_limit(adev->powerplay.pp_handle, &limit, false); @@ -2960,6 +3124,15 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev, return size; } +static ssize_t amdgpu_hwmon_show_power_label(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int limit_type = to_sensor_dev_attr(attr)->index; + + return snprintf(buf, PAGE_SIZE, "%s\n", + limit_type == SMU_FAST_PPT_LIMIT ? "fastPPT" : "slowPPT"); +} static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, struct device_attribute *attr, @@ -2967,6 +3140,7 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, size_t count) { struct amdgpu_device *adev = dev_get_drvdata(dev); + int limit_type = to_sensor_dev_attr(attr)->index; int err; u32 value; @@ -2981,7 +3155,7 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, return err; value = value / 1000000; /* convert to Watt */ - + value |= limit_type << 24; err = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (err < 0) { @@ -3193,6 +3367,12 @@ static SENSOR_DEVICE_ATTR(power1_average, S_IRUGO, amdgpu_hwmon_show_power_avg, static SENSOR_DEVICE_ATTR(power1_cap_max, S_IRUGO, amdgpu_hwmon_show_power_cap_max, NULL, 0); static SENSOR_DEVICE_ATTR(power1_cap_min, S_IRUGO, amdgpu_hwmon_show_power_cap_min, NULL, 0); static SENSOR_DEVICE_ATTR(power1_cap, S_IRUGO | S_IWUSR, amdgpu_hwmon_show_power_cap, amdgpu_hwmon_set_power_cap, 0); +static SENSOR_DEVICE_ATTR(power1_label, S_IRUGO, amdgpu_hwmon_show_power_label, NULL, 0); +static SENSOR_DEVICE_ATTR(power2_average, S_IRUGO, amdgpu_hwmon_show_power_avg, NULL, 1); +static SENSOR_DEVICE_ATTR(power2_cap_max, S_IRUGO, amdgpu_hwmon_show_power_cap_max, NULL, 1); +static SENSOR_DEVICE_ATTR(power2_cap_min, S_IRUGO, amdgpu_hwmon_show_power_cap_min, NULL, 1); +static SENSOR_DEVICE_ATTR(power2_cap, S_IRUGO | S_IWUSR, amdgpu_hwmon_show_power_cap, amdgpu_hwmon_set_power_cap, 1); +static SENSOR_DEVICE_ATTR(power2_label, S_IRUGO, amdgpu_hwmon_show_power_label, NULL, 1); static SENSOR_DEVICE_ATTR(freq1_input, S_IRUGO, amdgpu_hwmon_show_sclk, NULL, 0); static SENSOR_DEVICE_ATTR(freq1_label, S_IRUGO, amdgpu_hwmon_show_sclk_label, NULL, 0); static SENSOR_DEVICE_ATTR(freq2_input, S_IRUGO, amdgpu_hwmon_show_mclk, NULL, 0); @@ -3231,6 +3411,12 @@ static struct attribute *hwmon_attributes[] = { &sensor_dev_attr_power1_cap_max.dev_attr.attr, &sensor_dev_attr_power1_cap_min.dev_attr.attr, &sensor_dev_attr_power1_cap.dev_attr.attr, + &sensor_dev_attr_power1_label.dev_attr.attr, + &sensor_dev_attr_power2_average.dev_attr.attr, + &sensor_dev_attr_power2_cap_max.dev_attr.attr, + &sensor_dev_attr_power2_cap_min.dev_attr.attr, + &sensor_dev_attr_power2_cap.dev_attr.attr, + &sensor_dev_attr_power2_label.dev_attr.attr, &sensor_dev_attr_freq1_input.dev_attr.attr, &sensor_dev_attr_freq1_label.dev_attr.attr, &sensor_dev_attr_freq2_input.dev_attr.attr, @@ -3323,8 +3509,9 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj, effective_mode &= ~S_IWUSR; } - if (((adev->flags & AMD_IS_APU) || - adev->family == AMDGPU_FAMILY_SI) && /* not implemented yet */ + if (((adev->family == AMDGPU_FAMILY_SI) || + ((adev->flags & AMD_IS_APU) && + (adev->asic_type != CHIP_VANGOGH))) && /* not implemented yet */ (attr == &sensor_dev_attr_power1_cap_max.dev_attr.attr || attr == &sensor_dev_attr_power1_cap_min.dev_attr.attr|| attr == &sensor_dev_attr_power1_cap.dev_attr.attr)) @@ -3387,6 +3574,16 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj, attr == &sensor_dev_attr_temp3_label.dev_attr.attr)) return 0; + /* only Vangogh has fast PPT limit and power labels */ + if (!(adev->asic_type == CHIP_VANGOGH) && + (attr == &sensor_dev_attr_power2_average.dev_attr.attr || + attr == &sensor_dev_attr_power2_cap_max.dev_attr.attr || + attr == &sensor_dev_attr_power2_cap_min.dev_attr.attr || + attr == &sensor_dev_attr_power2_cap.dev_attr.attr || + attr == &sensor_dev_attr_power2_label.dev_attr.attr || + attr == &sensor_dev_attr_power1_label.dev_attr.attr)) + return 0; + return effective_mode; } @@ -3465,6 +3662,27 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev) */ #if defined(CONFIG_DEBUG_FS) +static void amdgpu_debugfs_prints_cpu_info(struct seq_file *m, + struct amdgpu_device *adev) { + uint16_t *p_val; + uint32_t size; + int i; + + if (is_support_cclk_dpm(adev)) { + p_val = kcalloc(adev->smu.cpu_core_num, sizeof(uint16_t), + GFP_KERNEL); + + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_CPU_CLK, + (void *)p_val, &size)) { + for (i = 0; i < adev->smu.cpu_core_num; i++) + seq_printf(m, "\t%u MHz (CPU%d)\n", + *(p_val + i), i); + } + + kfree(p_val); + } +} + static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *adev) { uint32_t value; @@ -3475,6 +3693,9 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a /* GPU Clocks */ size = sizeof(value); seq_printf(m, "GFX Clocks and Power:\n"); + + amdgpu_debugfs_prints_cpu_info(m, adev); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_MCLK, (void *)&value, &size)) seq_printf(m, "\t%u MHz (MCLK)\n", value/100); if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_SCLK, (void *)&value, &size)) diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h index 0d797fa9f5cc..10b0624ade65 100644 --- a/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h +++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h @@ -33,6 +33,8 @@ #define SMU_TEMPERATURE_UNITS_PER_CENTIGRADES 1000 #define SMU_FW_NAME_LEN 0x24 +#define SMU_DPM_USER_PROFILE_RESTORE (1 << 0) + struct smu_hw_power_state { unsigned int magic; }; @@ -159,6 +161,19 @@ enum smu_power_src_type SMU_POWER_SOURCE_COUNT, }; +enum smu_ppt_limit_type +{ + SMU_DEFAULT_PPT_LIMIT = 0, + SMU_FAST_PPT_LIMIT, +}; + +enum smu_ppt_limit_level +{ + SMU_PPT_LIMIT_MIN = -1, + SMU_PPT_LIMIT_CURRENT, + SMU_PPT_LIMIT_MAX, +}; + enum smu_memory_pool_size { SMU_MEMORY_POOL_SIZE_ZERO = 0, @@ -168,6 +183,17 @@ enum smu_memory_pool_size SMU_MEMORY_POOL_SIZE_2_GB = 0x80000000, }; +struct smu_user_dpm_profile { + uint32_t fan_mode; + uint32_t power_limit; + uint32_t fan_speed_percent; + uint32_t flags; + + /* user clock state information */ + uint32_t clk_mask[SMU_CLK_COUNT]; + uint32_t clk_dependency; +}; + #define SMU_TABLE_INIT(tables, table_id, s, a, d) \ do { \ tables[table_id].size = s; \ @@ -459,131 +485,672 @@ struct smu_context struct work_struct interrupt_work; unsigned fan_max_rpm; - unsigned manual_fan_speed_rpm; + unsigned manual_fan_speed_percent; uint32_t gfx_default_hard_min_freq; uint32_t gfx_default_soft_max_freq; uint32_t gfx_actual_hard_min_freq; uint32_t gfx_actual_soft_max_freq; + + /* APU only */ + uint32_t cpu_default_soft_min_freq; + uint32_t cpu_default_soft_max_freq; + uint32_t cpu_actual_soft_min_freq; + uint32_t cpu_actual_soft_max_freq; + uint32_t cpu_core_id_select; + uint16_t cpu_core_num; + + struct smu_user_dpm_profile user_dpm_profile; }; struct i2c_adapter; +/** + * struct pptable_funcs - Callbacks used to interact with the SMU. + */ struct pptable_funcs { + /** + * @run_btc: Calibrate voltage/frequency curve to fit the system's + * power delivery and voltage margins. Required for adaptive + * voltage frequency scaling (AVFS). + */ int (*run_btc)(struct smu_context *smu); + + /** + * @get_allowed_feature_mask: Get allowed feature mask. + * &feature_mask: Array to store feature mask. + * &num: Elements in &feature_mask. + */ int (*get_allowed_feature_mask)(struct smu_context *smu, uint32_t *feature_mask, uint32_t num); + + /** + * @get_current_power_state: Get the current power state. + * + * Return: Current power state on success, negative errno on failure. + */ enum amd_pm_state_type (*get_current_power_state)(struct smu_context *smu); + + /** + * @set_default_dpm_table: Retrieve the default overdrive settings from + * the SMU. + */ int (*set_default_dpm_table)(struct smu_context *smu); + int (*set_power_state)(struct smu_context *smu); + + /** + * @populate_umd_state_clk: Populate the UMD power state table with + * defaults. + */ int (*populate_umd_state_clk)(struct smu_context *smu); + + /** + * @print_clk_levels: Print DPM clock levels for a clock domain + * to buffer. Star current level. + * + * Used for sysfs interfaces. + */ int (*print_clk_levels)(struct smu_context *smu, enum smu_clk_type clk_type, char *buf); + + /** + * @force_clk_levels: Set a range of allowed DPM levels for a clock + * domain. + * &clk_type: Clock domain. + * &mask: Range of allowed DPM levels. + */ int (*force_clk_levels)(struct smu_context *smu, enum smu_clk_type clk_type, uint32_t mask); + + /** + * @od_edit_dpm_table: Edit the custom overdrive DPM table. + * &type: Type of edit. + * &input: Edit parameters. + * &size: Size of &input. + */ int (*od_edit_dpm_table)(struct smu_context *smu, enum PP_OD_DPM_TABLE_COMMAND type, long *input, uint32_t size); + + /** + * @get_clock_by_type_with_latency: Get the speed and latency of a clock + * domain. + */ int (*get_clock_by_type_with_latency)(struct smu_context *smu, enum smu_clk_type clk_type, struct pp_clock_levels_with_latency *clocks); + /** + * @get_clock_by_type_with_voltage: Get the speed and voltage of a clock + * domain. + */ + int (*get_clock_by_type_with_voltage)(struct smu_context *smu, + enum amd_pp_clock_type type, + struct + pp_clock_levels_with_voltage + *clocks); + + /** + * @get_power_profile_mode: Print all power profile modes to + * buffer. Star current mode. + */ int (*get_power_profile_mode)(struct smu_context *smu, char *buf); + + /** + * @set_power_profile_mode: Set a power profile mode. Also used to + * create/set custom power profile modes. + * &input: Power profile mode parameters. + * &size: Size of &input. + */ int (*set_power_profile_mode)(struct smu_context *smu, long *input, uint32_t size); + + /** + * @dpm_set_vcn_enable: Enable/disable VCN engine dynamic power + * management. + */ int (*dpm_set_vcn_enable)(struct smu_context *smu, bool enable); + + /** + * @dpm_set_jpeg_enable: Enable/disable JPEG engine dynamic power + * management. + */ int (*dpm_set_jpeg_enable)(struct smu_context *smu, bool enable); + + /** + * @read_sensor: Read data from a sensor. + * &sensor: Sensor to read data from. + * &data: Sensor reading. + * &size: Size of &data. + */ int (*read_sensor)(struct smu_context *smu, enum amd_pp_sensors sensor, void *data, uint32_t *size); + + /** + * @pre_display_config_changed: Prepare GPU for a display configuration + * change. + * + * Disable display tracking and pin memory clock speed to maximum. Used + * in display component synchronization. + */ int (*pre_display_config_changed)(struct smu_context *smu); + + /** + * @display_config_changed: Notify the SMU of the current display + * configuration. + * + * Allows SMU to properly track blanking periods for memory clock + * adjustment. Used in display component synchronization. + */ int (*display_config_changed)(struct smu_context *smu); + int (*apply_clocks_adjust_rules)(struct smu_context *smu); + + /** + * @notify_smc_display_config: Applies display requirements to the + * current power state. + * + * Optimize deep sleep DCEFclk and mclk for the current display + * configuration. Used in display component synchronization. + */ int (*notify_smc_display_config)(struct smu_context *smu); + + /** + * @is_dpm_running: Check if DPM is running. + * + * Return: True if DPM is running, false otherwise. + */ bool (*is_dpm_running)(struct smu_context *smu); - int (*get_fan_speed_rpm)(struct smu_context *smu, uint32_t *speed); + + /** + * @get_fan_speed_percent: Get the current fan speed in percent. + */ + int (*get_fan_speed_percent)(struct smu_context *smu, uint32_t *speed); + + /** + * @set_watermarks_table: Configure and upload the watermarks tables to + * the SMU. + */ int (*set_watermarks_table)(struct smu_context *smu, struct pp_smu_wm_range_sets *clock_ranges); + + /** + * @get_thermal_temperature_range: Get safe thermal limits in Celcius. + */ int (*get_thermal_temperature_range)(struct smu_context *smu, struct smu_temperature_range *range); + + /** + * @get_uclk_dpm_states: Get memory clock DPM levels in kHz. + * &clocks_in_khz: Array of DPM levels. + * &num_states: Elements in &clocks_in_khz. + */ int (*get_uclk_dpm_states)(struct smu_context *smu, uint32_t *clocks_in_khz, uint32_t *num_states); + + /** + * @set_default_od_settings: Set the overdrive tables to defaults. + */ int (*set_default_od_settings)(struct smu_context *smu); + + /** + * @set_performance_level: Set a performance level. + */ int (*set_performance_level)(struct smu_context *smu, enum amd_dpm_forced_level level); + + /** + * @display_disable_memory_clock_switch: Enable/disable dynamic memory + * clock switching. + * + * Disabling this feature forces memory clock speed to maximum. + * Enabling sets the minimum memory clock capable of driving the + * current display configuration. + */ int (*display_disable_memory_clock_switch)(struct smu_context *smu, bool disable_memory_clock_switch); + + /** + * @dump_pptable: Print the power play table to the system log. + */ void (*dump_pptable)(struct smu_context *smu); + + /** + * @get_power_limit: Get the device's power limits. + */ int (*get_power_limit)(struct smu_context *smu); + + /** + * @get_ppt_limit: Get the device's ppt limits. + */ + int (*get_ppt_limit)(struct smu_context *smu, uint32_t *ppt_limit, + enum smu_ppt_limit_type limit_type, enum smu_ppt_limit_level limit_level); + + /** + * @set_df_cstate: Set data fabric cstate. + */ int (*set_df_cstate)(struct smu_context *smu, enum pp_df_cstate state); + + /** + * @allow_xgmi_power_down: Enable/disable external global memory + * interconnect power down. + */ int (*allow_xgmi_power_down)(struct smu_context *smu, bool en); + + /** + * @update_pcie_parameters: Update and upload the system's PCIe + * capabilites to the SMU. + * &pcie_gen_cap: Maximum allowed PCIe generation. + * &pcie_width_cap: Maximum allowed PCIe width. + */ int (*update_pcie_parameters)(struct smu_context *smu, uint32_t pcie_gen_cap, uint32_t pcie_width_cap); + + /** + * @i2c_init: Initialize i2c. + * + * The i2c bus is used internally by the SMU voltage regulators and + * other devices. The i2c's EEPROM also stores bad page tables on boards + * with ECC. + */ int (*i2c_init)(struct smu_context *smu, struct i2c_adapter *control); + + /** + * @i2c_fini: Tear down i2c. + */ void (*i2c_fini)(struct smu_context *smu, struct i2c_adapter *control); + + /** + * @get_unique_id: Get the GPU's unique id. Used for asset tracking. + */ void (*get_unique_id)(struct smu_context *smu); + + /** + * @get_dpm_clock_table: Get a copy of the DPM clock table. + * + * Used by display component in bandwidth and watermark calculations. + */ int (*get_dpm_clock_table)(struct smu_context *smu, struct dpm_clocks *clock_table); + + /** + * @init_microcode: Request the SMU's firmware from the kernel. + */ int (*init_microcode)(struct smu_context *smu); + + /** + * @load_microcode: Load firmware onto the SMU. + */ int (*load_microcode)(struct smu_context *smu); + + /** + * @fini_microcode: Release the SMU's firmware. + */ void (*fini_microcode)(struct smu_context *smu); + + /** + * @init_smc_tables: Initialize the SMU tables. + */ int (*init_smc_tables)(struct smu_context *smu); + + /** + * @fini_smc_tables: Release the SMU tables. + */ int (*fini_smc_tables)(struct smu_context *smu); + + /** + * @init_power: Initialize the power gate table context. + */ int (*init_power)(struct smu_context *smu); + + /** + * @fini_power: Release the power gate table context. + */ int (*fini_power)(struct smu_context *smu); + + /** + * @check_fw_status: Check the SMU's firmware status. + * + * Return: Zero if check passes, negative errno on failure. + */ int (*check_fw_status)(struct smu_context *smu); + + /** + * @setup_pptable: Initialize the power play table and populate it with + * default values. + */ int (*setup_pptable)(struct smu_context *smu); + + /** + * @get_vbios_bootup_values: Get default boot values from the VBIOS. + */ int (*get_vbios_bootup_values)(struct smu_context *smu); + + /** + * @check_fw_version: Print driver and SMU interface versions to the + * system log. + * + * Interface mismatch is not a critical failure. + */ int (*check_fw_version)(struct smu_context *smu); + + /** + * @powergate_sdma: Power up/down system direct memory access. + */ int (*powergate_sdma)(struct smu_context *smu, bool gate); + + /** + * @set_gfx_cgpg: Enable/disable graphics engine course grain power + * gating. + */ int (*set_gfx_cgpg)(struct smu_context *smu, bool enable); + + /** + * @write_pptable: Write the power play table to the SMU. + */ int (*write_pptable)(struct smu_context *smu); + + /** + * @set_driver_table_location: Send the location of the driver table to + * the SMU. + */ int (*set_driver_table_location)(struct smu_context *smu); + + /** + * @set_tool_table_location: Send the location of the tool table to the + * SMU. + */ int (*set_tool_table_location)(struct smu_context *smu); + + /** + * @notify_memory_pool_location: Send the location of the memory pool to + * the SMU. + */ int (*notify_memory_pool_location)(struct smu_context *smu); + + /** + * @system_features_control: Enable/disable all SMU features. + */ int (*system_features_control)(struct smu_context *smu, bool en); + + /** + * @send_smc_msg_with_param: Send a message with a parameter to the SMU. + * &msg: Type of message. + * ¶m: Message parameter. + * &read_arg: SMU response (optional). + */ int (*send_smc_msg_with_param)(struct smu_context *smu, enum smu_message_type msg, uint32_t param, uint32_t *read_arg); + + /** + * @send_smc_msg: Send a message to the SMU. + * &msg: Type of message. + * &read_arg: SMU response (optional). + */ int (*send_smc_msg)(struct smu_context *smu, enum smu_message_type msg, uint32_t *read_arg); + + /** + * @init_display_count: Notify the SMU of the number of display + * components in current display configuration. + */ int (*init_display_count)(struct smu_context *smu, uint32_t count); + + /** + * @set_allowed_mask: Notify the SMU of the features currently allowed + * by the driver. + */ int (*set_allowed_mask)(struct smu_context *smu); + + /** + * @get_enabled_mask: Get a mask of features that are currently enabled + * on the SMU. + * &feature_mask: Array representing enabled feature mask. + * &num: Elements in &feature_mask. + */ int (*get_enabled_mask)(struct smu_context *smu, uint32_t *feature_mask, uint32_t num); + + /** + * @feature_is_enabled: Test if a feature is enabled. + * + * Return: One if enabled, zero if disabled. + */ int (*feature_is_enabled)(struct smu_context *smu, enum smu_feature_mask mask); + + /** + * @disable_all_features_with_exception: Disable all features with + * exception to those in &mask. + */ int (*disable_all_features_with_exception)(struct smu_context *smu, enum smu_feature_mask mask); + + /** + * @notify_display_change: Enable fast memory clock switching. + * + * Allows for fine grained memory clock switching but has more stringent + * timing requirements. + */ int (*notify_display_change)(struct smu_context *smu); + + /** + * @set_power_limit: Set power limit in watts. + */ int (*set_power_limit)(struct smu_context *smu, uint32_t n); + + /** + * @init_max_sustainable_clocks: Populate max sustainable clock speed + * table with values from the SMU. + */ int (*init_max_sustainable_clocks)(struct smu_context *smu); + + /** + * @enable_thermal_alert: Enable thermal alert interrupts. + */ int (*enable_thermal_alert)(struct smu_context *smu); + + /** + * @disable_thermal_alert: Disable thermal alert interrupts. + */ int (*disable_thermal_alert)(struct smu_context *smu); + + /** + * @set_min_dcef_deep_sleep: Set a minimum display fabric deep sleep + * clock speed in MHz. + */ int (*set_min_dcef_deep_sleep)(struct smu_context *smu, uint32_t clk); + + /** + * @display_clock_voltage_request: Set a hard minimum frequency + * for a clock domain. + */ int (*display_clock_voltage_request)(struct smu_context *smu, struct pp_display_clock_request *clock_req); + + /** + * @get_fan_control_mode: Get the current fan control mode. + */ uint32_t (*get_fan_control_mode)(struct smu_context *smu); + + /** + * @set_fan_control_mode: Set the fan control mode. + */ int (*set_fan_control_mode)(struct smu_context *smu, uint32_t mode); + + /** + * @set_fan_speed_percent: Set a static fan speed in percent. + */ int (*set_fan_speed_percent)(struct smu_context *smu, uint32_t speed); - int (*set_fan_speed_rpm)(struct smu_context *smu, uint32_t speed); + + /** + * @set_xgmi_pstate: Set inter-chip global memory interconnect pstate. + * &pstate: Pstate to set. D0 if Nonzero, D3 otherwise. + */ int (*set_xgmi_pstate)(struct smu_context *smu, uint32_t pstate); + + /** + * @gfx_off_control: Enable/disable graphics engine poweroff. + */ int (*gfx_off_control)(struct smu_context *smu, bool enable); + + + /** + * @get_gfx_off_status: Get graphics engine poweroff status. + * + * Return: + * 0 - GFXOFF(default). + * 1 - Transition out of GFX State. + * 2 - Not in GFXOFF. + * 3 - Transition into GFXOFF. + */ uint32_t (*get_gfx_off_status)(struct smu_context *smu); + + /** + * @register_irq_handler: Register interupt request handlers. + */ int (*register_irq_handler)(struct smu_context *smu); + + /** + * @set_azalia_d3_pme: Wake the audio decode engine from d3 sleep. + */ int (*set_azalia_d3_pme)(struct smu_context *smu); + + /** + * @get_max_sustainable_clocks_by_dc: Get a copy of the max sustainable + * clock speeds table. + * + * Provides a way for the display component (DC) to get the max + * sustainable clocks from the SMU. + */ int (*get_max_sustainable_clocks_by_dc)(struct smu_context *smu, struct pp_smu_nv_clock_table *max_clocks); + + /** + * @baco_is_support: Check if GPU supports BACO (Bus Active, Chip Off). + */ bool (*baco_is_support)(struct smu_context *smu); + + /** + * @baco_get_state: Get the current BACO state. + * + * Return: Current BACO state. + */ enum smu_baco_state (*baco_get_state)(struct smu_context *smu); + + /** + * @baco_set_state: Enter/exit BACO. + */ int (*baco_set_state)(struct smu_context *smu, enum smu_baco_state state); + + /** + * @baco_enter: Enter BACO. + */ int (*baco_enter)(struct smu_context *smu); + + /** + * @baco_exit: Exit Baco. + */ int (*baco_exit)(struct smu_context *smu); + + /** + * @mode1_reset_is_support: Check if GPU supports mode1 reset. + */ bool (*mode1_reset_is_support)(struct smu_context *smu); + + /** + * @mode1_reset: Perform mode1 reset. + * + * Complete GPU reset. + */ int (*mode1_reset)(struct smu_context *smu); + + /** + * @mode2_reset: Perform mode2 reset. + * + * Mode2 reset generally does not reset as many IPs as mode1 reset. The + * IPs reset varies by asic. + */ int (*mode2_reset)(struct smu_context *smu); + + /** + * @get_dpm_ultimate_freq: Get the hard frequency range of a clock + * domain in MHz. + */ int (*get_dpm_ultimate_freq)(struct smu_context *smu, enum smu_clk_type clk_type, uint32_t *min, uint32_t *max); + + /** + * @set_soft_freq_limited_range: Set the soft frequency range of a clock + * domain in MHz. + */ int (*set_soft_freq_limited_range)(struct smu_context *smu, enum smu_clk_type clk_type, uint32_t min, uint32_t max); + + /** + * @set_power_source: Notify the SMU of the current power source. + */ int (*set_power_source)(struct smu_context *smu, enum smu_power_src_type power_src); + + /** + * @log_thermal_throttling_event: Print a thermal throttling warning to + * the system's log. + */ void (*log_thermal_throttling_event)(struct smu_context *smu); + + /** + * @get_pp_feature_mask: Print a human readable table of enabled + * features to buffer. + */ size_t (*get_pp_feature_mask)(struct smu_context *smu, char *buf); + + /** + * @set_pp_feature_mask: Request the SMU enable/disable features to + * match those enabled in &new_mask. + */ int (*set_pp_feature_mask)(struct smu_context *smu, uint64_t new_mask); + + /** + * @get_gpu_metrics: Get a copy of the GPU metrics table from the SMU. + * + * Return: Size of &table + */ ssize_t (*get_gpu_metrics)(struct smu_context *smu, void **table); + + /** + * @enable_mgpu_fan_boost: Enable multi-GPU fan boost. + */ int (*enable_mgpu_fan_boost)(struct smu_context *smu); + + /** + * @gfx_ulv_control: Enable/disable ultra low voltage. + */ int (*gfx_ulv_control)(struct smu_context *smu, bool enablement); + + /** + * @deep_sleep_control: Enable/disable deep sleep. + */ int (*deep_sleep_control)(struct smu_context *smu, bool enablement); + + /** + * @get_fan_parameters: Get fan parameters. + * + * Get maximum fan speed from the power play table. + */ int (*get_fan_parameters)(struct smu_context *smu); + + /** + * @post_init: Helper function for asic specific workarounds. + */ int (*post_init)(struct smu_context *smu); + + /** + * @interrupt_work: Work task scheduled from SMU interrupt handler. + */ void (*interrupt_work)(struct smu_context *smu); + + /** + * @gpo_control: Enable/disable graphics power optimization if supported. + */ int (*gpo_control)(struct smu_context *smu, bool enablement); + + /** + * @gfx_state_change_set: Send the current graphics state to the SMU. + */ int (*gfx_state_change_set)(struct smu_context *smu, uint32_t state); + + /** + * @set_fine_grain_gfx_freq_parameters: Set fine grain graphics clock + * parameters to defaults. + */ int (*set_fine_grain_gfx_freq_parameters)(struct smu_context *smu); }; @@ -597,6 +1164,7 @@ typedef enum { METRICS_CURR_DCLK1, METRICS_CURR_FCLK, METRICS_CURR_DCEFCLK, + METRICS_AVERAGE_CPUCLK, METRICS_AVERAGE_GFXCLK, METRICS_AVERAGE_SOCCLK, METRICS_AVERAGE_FCLK, @@ -637,6 +1205,12 @@ enum smu_cmn2asic_mapping_type { #define FEA_MAP(fea) \ [SMU_FEATURE_##fea##_BIT] = {1, FEATURE_##fea##_BIT} +#define FEA_MAP_REVERSE(fea) \ + [SMU_FEATURE_DPM_##fea##_BIT] = {1, FEATURE_##fea##_DPM_BIT} + +#define FEA_MAP_HALF_REVERSE(fea) \ + [SMU_FEATURE_DPM_##fea##CLK_BIT] = {1, FEATURE_##fea##_DPM_BIT} + #define TAB_MAP(tab) \ [SMU_TABLE_##tab] = {1, TABLE_##tab} @@ -663,7 +1237,7 @@ int smu_set_fan_speed_rpm(struct smu_context *smu, uint32_t speed); int smu_get_power_limit(struct smu_context *smu, uint32_t *limit, - bool max_setting); + enum smu_ppt_limit_level limit_level); int smu_set_power_limit(struct smu_context *smu, uint32_t limit); int smu_print_clk_levels(struct smu_context *smu, enum smu_clk_type clk_type, char *buf); @@ -719,6 +1293,7 @@ extern const struct amdgpu_ip_block_version smu_v11_0_ip_block; extern const struct amdgpu_ip_block_version smu_v12_0_ip_block; bool is_support_sw_smu(struct amdgpu_device *adev); +bool is_support_cclk_dpm(struct amdgpu_device *adev); int smu_reset(struct smu_context *smu); int smu_sys_get_pp_table(struct smu_context *smu, void **table); int smu_sys_set_pp_table(struct smu_context *smu, void *buf, size_t size); diff --git a/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_vangogh.h b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_vangogh.h index 1c19eae93ff1..6e23a3f803a7 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_vangogh.h +++ b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_vangogh.h @@ -141,7 +141,6 @@ typedef struct { uint32_t MaxGfxClk; uint8_t NumDfPstatesEnabled; - uint8_t NumDpmLevelsEnabled; uint8_t NumDcfclkLevelsEnabled; uint8_t NumDispClkLevelsEnabled; //applies to both dispclk and dppclk uint8_t NumSocClkLevelsEnabled; diff --git a/drivers/gpu/drm/amd/pm/inc/smu_types.h b/drivers/gpu/drm/amd/pm/inc/smu_types.h index 720d15612fe1..aa4822202587 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_types.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_types.h @@ -133,8 +133,6 @@ __SMU_DUMMY_MAP(PowerUpSdma), \ __SMU_DUMMY_MAP(SetHardMinIspclkByFreq), \ __SMU_DUMMY_MAP(SetHardMinVcn), \ - __SMU_DUMMY_MAP(Spare1), \ - __SMU_DUMMY_MAP(Spare2), \ __SMU_DUMMY_MAP(SetAllowFclkSwitch), \ __SMU_DUMMY_MAP(SetMinVideoGfxclkFreq), \ __SMU_DUMMY_MAP(ActiveProcessNotify), \ @@ -211,6 +209,11 @@ __SMU_DUMMY_MAP(SetGpoFeaturePMask), \ __SMU_DUMMY_MAP(DisallowGpo), \ __SMU_DUMMY_MAP(Enable2ndUSB20Port), \ + __SMU_DUMMY_MAP(RequestActiveWgp), \ + __SMU_DUMMY_MAP(SetFastPPTLimit), \ + __SMU_DUMMY_MAP(SetSlowPPTLimit), \ + __SMU_DUMMY_MAP(GetFastPPTLimit), \ + __SMU_DUMMY_MAP(GetSlowPPTLimit), \ #undef __SMU_DUMMY_MAP #define __SMU_DUMMY_MAP(type) SMU_MSG_##type @@ -236,10 +239,12 @@ enum smu_clk_type { SMU_SCLK, SMU_MCLK, SMU_PCIE, + SMU_OD_CCLK, SMU_OD_SCLK, SMU_OD_MCLK, SMU_OD_VDDC_CURVE, SMU_OD_RANGE, + SMU_OD_VDDGFX_OFFSET, SMU_CLK_COUNT, }; diff --git a/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h index 5d0b29653ffa..d4cddd2390a2 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h @@ -31,7 +31,7 @@ #define SMU11_DRIVER_IF_VERSION_NV12 0x36 #define SMU11_DRIVER_IF_VERSION_NV14 0x36 #define SMU11_DRIVER_IF_VERSION_Sienna_Cichlid 0x3D -#define SMU11_DRIVER_IF_VERSION_Navy_Flounder 0xC +#define SMU11_DRIVER_IF_VERSION_Navy_Flounder 0xE #define SMU11_DRIVER_IF_VERSION_VANGOGH 0x02 #define SMU11_DRIVER_IF_VERSION_Dimgrey_Cavefish 0xF @@ -129,6 +129,15 @@ struct smu_11_0_power_context { enum smu_11_0_power_state power_state; }; +struct smu_11_5_power_context { + uint32_t power_source; + uint8_t in_power_limit_boost_mode; + enum smu_11_0_power_state power_state; + + uint32_t current_fast_ppt_limit; + uint32_t max_fast_ppt_limit; +}; + enum smu_v11_0_baco_seq { BACO_SEQ_BACO = 0, BACO_SEQ_MSR, @@ -203,14 +212,8 @@ int smu_v11_0_set_fan_control_mode(struct smu_context *smu, uint32_t mode); -int -smu_v11_0_set_fan_speed_percent(struct smu_context *smu, uint32_t speed); - -int smu_v11_0_set_fan_speed_rpm(struct smu_context *smu, - uint32_t speed); - -int smu_v11_0_get_fan_speed_rpm(struct smu_context *smu, - uint32_t *speed); +int smu_v11_0_set_fan_speed_percent(struct smu_context *smu, + uint32_t speed); int smu_v11_0_set_xgmi_pstate(struct smu_context *smu, uint32_t pstate); @@ -278,10 +281,6 @@ int smu_v11_0_get_current_pcie_link_speed_level(struct smu_context *smu); int smu_v11_0_get_current_pcie_link_speed(struct smu_context *smu); -void smu_v11_0_init_gpu_metrics_v1_0(struct gpu_metrics_v1_0 *gpu_metrics); - -void smu_v11_0_init_gpu_metrics_v2_0(struct gpu_metrics_v2_0 *gpu_metrics); - int smu_v11_0_gfx_ulv_control(struct smu_context *smu, bool enablement); diff --git a/drivers/gpu/drm/amd/pm/inc/smu_v11_5_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_5_ppsmc.h index 55d7892e4e0e..fe130a497d6c 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_v11_5_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_5_ppsmc.h @@ -104,7 +104,11 @@ #define PPSMC_MSG_DramLogSetDramBufferSize 0x46 #define PPSMC_MSG_RequestActiveWgp 0x47 #define PPSMC_MSG_QueryActiveWgp 0x48 -#define PPSMC_Message_Count 0x49 +#define PPSMC_MSG_SetFastPPTLimit 0x49 +#define PPSMC_MSG_SetSlowPPTLimit 0x4A +#define PPSMC_MSG_GetFastPPTLimit 0x4B +#define PPSMC_MSG_GetSlowPPTLimit 0x4C +#define PPSMC_Message_Count 0x4D //Argument for PPSMC_MSG_GfxDeviceDriverReset enum { diff --git a/drivers/gpu/drm/amd/pm/inc/smu_v12_0.h b/drivers/gpu/drm/amd/pm/inc/smu_v12_0.h index fa2e8cb07967..02de3b6199e5 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_v12_0.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v12_0.h @@ -60,7 +60,5 @@ int smu_v12_0_set_soft_freq_limited_range(struct smu_context *smu, enum smu_clk_ int smu_v12_0_set_driver_table_location(struct smu_context *smu); -void smu_v12_0_init_gpu_metrics_v2_0(struct gpu_metrics_v2_0 *gpu_metrics); - #endif #endif diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c index 6a7de8b898fa..f2cef0930aa9 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c @@ -33,6 +33,7 @@ #include "ppsmc.h" #include "amd_acpi.h" #include "pp_psm.h" +#include "vega10_hwmgr.h" extern const struct pp_smumgr_func ci_smu_funcs; extern const struct pp_smumgr_func smu8_smu_funcs; @@ -46,7 +47,6 @@ extern const struct pp_smumgr_func vega12_smu_funcs; extern const struct pp_smumgr_func smu10_smu_funcs; extern const struct pp_smumgr_func vega20_smu_funcs; -extern int vega10_hwmgr_init(struct pp_hwmgr *hwmgr); extern int smu10_init_function_pointers(struct pp_hwmgr *hwmgr); static int polaris_set_asic_special_caps(struct pp_hwmgr *hwmgr); diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c index 83a6504e093c..b1038d30c8dc 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c @@ -279,7 +279,7 @@ static const ATOM_VOLTAGE_OBJECT_V3 *atomctrl_lookup_voltage_type_v3( * * @hwmgr: input parameter: pointer to HwMgr * @clock_value: input parameter: memory clock - * @dividers: output parameter: memory PLL dividers + * @mpll_param: output parameter: memory clock parameters * @strobe_mode: input parameter: 1 for strobe mode, 0 for performance mode */ int atomctrl_get_memory_pll_dividers_si( @@ -332,7 +332,7 @@ int atomctrl_get_memory_pll_dividers_si( * * @hwmgr: input parameter: pointer to HwMgr * @clock_value: input parameter: memory clock - * @dividers: output parameter: memory PLL dividers + * @mpll_param: output parameter: memory clock parameters */ int atomctrl_get_memory_pll_dividers_vi(struct pp_hwmgr *hwmgr, uint32_t clock_value, pp_atomctrl_memory_clock_param *mpll_param) diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c index 741e03ad5311..f2a55c1413f5 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c @@ -1362,6 +1362,7 @@ static int ppt_get_vce_state_table_entry_v1_0(struct pp_hwmgr *hwmgr, uint32_t i * @hwmgr: Pointer to the hardware manager. * @entry_index: The index of the entry to be extracted from the table. * @power_state: The address of the PowerState instance being created. + * @call_back_func: The function to call into to fill power state * Return: -1 if the entry cannot be retrieved. */ int get_powerplay_table_entry_v1_0(struct pp_hwmgr *hwmgr, diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c index 88322781e447..ed05a30d1139 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c @@ -1487,7 +1487,7 @@ static int smu10_set_fine_grain_clk_vol(struct pp_hwmgr *hwmgr, } if (!smu10_data->fine_grain_enabled) { - pr_err("Fine grain not started\n"); + pr_err("pp_od_clk_voltage is not accessible if power_dpm_force_perfomance_level is not in manual mode!\n"); return -EINVAL; } diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index 82676c086ce4..c57dc9ae81f2 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -235,7 +235,7 @@ static int smu7_get_current_pcie_lane_number(struct pp_hwmgr *hwmgr) /** * smu7_enable_smc_voltage_controller - Enable voltage control * - * @hwmgr the address of the powerplay hardware manager. + * @hwmgr: the address of the powerplay hardware manager. * Return: always PP_Result_OK */ static int smu7_enable_smc_voltage_controller(struct pp_hwmgr *hwmgr) @@ -4501,7 +4501,7 @@ static int smu7_display_configuration_changed_task(struct pp_hwmgr *hwmgr) * smu7_set_max_fan_rpm_output - Set maximum target operating fan output RPM * * @hwmgr: the address of the powerplay hardware manager. - * @usMaxFanRpm: max operating fan RPM value. + * @us_max_fan_rpm: max operating fan RPM value. * Return: The response that came from the SMC. */ static int smu7_set_max_fan_rpm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_rpm) diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c index 1b47f94e0331..29c99642d22d 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c @@ -542,11 +542,11 @@ static int vega10_get_socclk_for_voltage_evv(struct pp_hwmgr *hwmgr, #define ATOM_VIRTUAL_VOLTAGE_ID0 0xff01 /** -* Get Leakage VDDC based on leakage ID. -* -* @param hwmgr the address of the powerplay hardware manager. -* @return always 0. -*/ + * Get Leakage VDDC based on leakage ID. + * + * @hwmgr: the address of the powerplay hardware manager. + * return: always 0. + */ static int vega10_get_evv_voltages(struct pp_hwmgr *hwmgr) { struct vega10_hwmgr *data = hwmgr->backend; @@ -600,9 +600,9 @@ static int vega10_get_evv_voltages(struct pp_hwmgr *hwmgr) /** * Change virtual leakage voltage to actual value. * - * @param hwmgr the address of the powerplay hardware manager. - * @param pointer to changing voltage - * @param pointer to leakage table + * @hwmgr: the address of the powerplay hardware manager. + * @voltage: pointer to changing voltage + * @leakage_table: pointer to leakage table */ static void vega10_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr, uint16_t *voltage, struct vega10_leakage_voltage *leakage_table) @@ -624,13 +624,13 @@ static void vega10_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr, } /** -* Patch voltage lookup table by EVV leakages. -* -* @param hwmgr the address of the powerplay hardware manager. -* @param pointer to voltage lookup table -* @param pointer to leakage table -* @return always 0 -*/ + * Patch voltage lookup table by EVV leakages. + * + * @hwmgr: the address of the powerplay hardware manager. + * @lookup_table: pointer to voltage lookup table + * @leakage_table: pointer to leakage table + * return: always 0 + */ static int vega10_patch_lookup_table_with_leakage(struct pp_hwmgr *hwmgr, phm_ppt_v1_voltage_lookup_table *lookup_table, struct vega10_leakage_voltage *leakage_table) @@ -1001,13 +1001,12 @@ static int vega10_setup_asic_task(struct pp_hwmgr *hwmgr) } /** -* Remove repeated voltage values and create table with unique values. -* -* @param hwmgr the address of the powerplay hardware manager. -* @param vol_table the pointer to changing voltage table -* @return 0 in success -*/ - + * Remove repeated voltage values and create table with unique values. + * + * @hwmgr: the address of the powerplay hardware manager. + * @vol_table: the pointer to changing voltage table + * return: 0 in success + */ static int vega10_trim_voltage_table(struct pp_hwmgr *hwmgr, struct pp_atomfwctrl_voltage_table *vol_table) { @@ -1151,11 +1150,11 @@ static void vega10_trim_voltage_table_to_fit_state_table( } /** -* Create Voltage Tables. -* -* @param hwmgr the address of the powerplay hardware manager. -* @return always 0 -*/ + * Create Voltage Tables. + * + * @hwmgr: the address of the powerplay hardware manager. + * return: always 0 + */ static int vega10_construct_voltage_tables(struct pp_hwmgr *hwmgr) { struct vega10_hwmgr *data = hwmgr->backend; @@ -1212,11 +1211,11 @@ static int vega10_construct_voltage_tables(struct pp_hwmgr *hwmgr) } /* - * @fn vega10_init_dpm_state - * @brief Function to initialize all Soft Min/Max and Hard Min/Max to 0xff. + * vega10_init_dpm_state + * Function to initialize all Soft Min/Max and Hard Min/Max to 0xff. * - * @param dpm_state - the address of the DPM Table to initiailize. - * @return None. + * @dpm_state: - the address of the DPM Table to initiailize. + * return: None. */ static void vega10_init_dpm_state(struct vega10_dpm_state *dpm_state) { @@ -1460,11 +1459,11 @@ static int vega10_setup_default_dpm_tables(struct pp_hwmgr *hwmgr) } /* - * @fn vega10_populate_ulv_state - * @brief Function to provide parameters for Utral Low Voltage state to SMC. + * vega10_populate_ulv_state + * Function to provide parameters for Utral Low Voltage state to SMC. * - * @param hwmgr - the address of the hardware manager. - * @return Always 0. + * @hwmgr: - the address of the hardware manager. + * return: Always 0. */ static int vega10_populate_ulv_state(struct pp_hwmgr *hwmgr) { @@ -1545,13 +1544,13 @@ static int vega10_populate_smc_link_levels(struct pp_hwmgr *hwmgr) } /** -* Populates single SMC GFXSCLK structure using the provided engine clock -* -* @param hwmgr the address of the hardware manager -* @param gfx_clock the GFX clock to use to populate the structure. -* @param current_gfxclk_level location in PPTable for the SMC GFXCLK structure. -*/ - + * Populates single SMC GFXSCLK structure using the provided engine clock + * + * @hwmgr: the address of the hardware manager + * @gfx_clock: the GFX clock to use to populate the structure. + * @current_gfxclk_level: location in PPTable for the SMC GFXCLK structure. + * @acg_freq: ACG frequenty to return (MHz) + */ static int vega10_populate_single_gfx_level(struct pp_hwmgr *hwmgr, uint32_t gfx_clock, PllSetting_t *current_gfxclk_level, uint32_t *acg_freq) @@ -1610,12 +1609,13 @@ static int vega10_populate_single_gfx_level(struct pp_hwmgr *hwmgr, } /** - * @brief Populates single SMC SOCCLK structure using the provided clock. + * Populates single SMC SOCCLK structure using the provided clock. * - * @param hwmgr - the address of the hardware manager. - * @param soc_clock - the SOC clock to use to populate the structure. - * @param current_socclk_level - location in PPTable for the SMC SOCCLK structure. - * @return 0 on success.. + * @hwmgr: the address of the hardware manager. + * @soc_clock: the SOC clock to use to populate the structure. + * @current_soc_did: DFS divider to pass back to caller + * @current_vol_index: index of current VDD to pass back to caller + * return: 0 on success */ static int vega10_populate_single_soc_level(struct pp_hwmgr *hwmgr, uint32_t soc_clock, uint8_t *current_soc_did, @@ -1659,10 +1659,10 @@ static int vega10_populate_single_soc_level(struct pp_hwmgr *hwmgr, } /** -* Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states -* -* @param hwmgr the address of the hardware manager -*/ + * Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states + * + * @hwmgr: the address of the hardware manager + */ static int vega10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr) { struct vega10_hwmgr *data = hwmgr->backend; @@ -1746,12 +1746,12 @@ static void vega10_populate_vddc_soc_levels(struct pp_hwmgr *hwmgr) } } -/** - * @brief Populates single SMC GFXCLK structure using the provided clock. +/* + * Populates single SMC GFXCLK structure using the provided clock. * - * @param hwmgr - the address of the hardware manager. - * @param mem_clock - the memory clock to use to populate the structure. - * @return 0 on success.. + * @hwmgr: the address of the hardware manager. + * @mem_clock: the memory clock to use to populate the structure. + * return: 0 on success.. */ static int vega10_populate_single_memory_level(struct pp_hwmgr *hwmgr, uint32_t mem_clock, uint8_t *current_mem_vid, @@ -1808,10 +1808,10 @@ static int vega10_populate_single_memory_level(struct pp_hwmgr *hwmgr, } /** - * @brief Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states. + * Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states. * - * @param pHwMgr - the address of the hardware manager. - * @return PP_Result_OK on success. + * @hwmgr: the address of the hardware manager. + * return: PP_Result_OK on success. */ static int vega10_populate_all_memory_levels(struct pp_hwmgr *hwmgr) { @@ -2486,12 +2486,11 @@ static void vega10_check_dpm_table_updated(struct pp_hwmgr *hwmgr) } /** -* Initializes the SMC table and uploads it -* -* @param hwmgr the address of the powerplay hardware manager. -* @param pInput the pointer to input data (PowerState) -* @return always 0 -*/ + * Initializes the SMC table and uploads it + * + * @hwmgr: the address of the powerplay hardware manager. + * return: always 0 + */ static int vega10_init_smc_table(struct pp_hwmgr *hwmgr) { int result; @@ -2864,11 +2863,11 @@ static int vega10_stop_dpm(struct pp_hwmgr *hwmgr, uint32_t bitmap) } /** - * @brief Tell SMC to enabled the supported DPMs. + * Tell SMC to enabled the supported DPMs. * - * @param hwmgr - the address of the powerplay hardware manager. - * @Param bitmap - bitmap for the features to enabled. - * @return 0 on at least one DPM is successfully enabled. + * @hwmgr: the address of the powerplay hardware manager. + * @bitmap: bitmap for the features to enabled. + * return: 0 on at least one DPM is successfully enabled. */ static int vega10_start_dpm(struct pp_hwmgr *hwmgr, uint32_t bitmap) { diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.h index f752b4ad0c8a..07c06f8c90b0 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.h @@ -442,5 +442,6 @@ int vega10_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate); int vega10_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate); int vega10_update_acp_dpm(struct pp_hwmgr *hwmgr, bool bgate); int vega10_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable); +int vega10_hwmgr_init(struct pp_hwmgr *hwmgr); #endif /* _VEGA10_HWMGR_H_ */ diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c index dc206fa88c5e..c0753029a8e2 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c @@ -718,12 +718,11 @@ static int vega12_save_default_power_profile(struct pp_hwmgr *hwmgr) #endif /** -* Initializes the SMC table and uploads it -* -* @param hwmgr the address of the powerplay hardware manager. -* @param pInput the pointer to input data (PowerState) -* @return always 0 -*/ + * Initializes the SMC table and uploads it + * + * @hwmgr: the address of the powerplay hardware manager. + * return: always 0 + */ static int vega12_init_smc_table(struct pp_hwmgr *hwmgr) { int result; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c index da84012b7fd5..87811b005b85 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c @@ -771,12 +771,11 @@ static int vega20_setup_default_dpm_tables(struct pp_hwmgr *hwmgr) } /** -* Initializes the SMC table and uploads it -* -* @param hwmgr the address of the powerplay hardware manager. -* @param pInput the pointer to input data (PowerState) -* @return always 0 -*/ + * Initializes the SMC table and uploads it + * + * @hwmgr: the address of the powerplay hardware manager. + * return: always 0 + */ static int vega20_init_smc_table(struct pp_hwmgr *hwmgr) { int result; diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index e84c737e3967..d143ef1b460b 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -266,6 +266,119 @@ int smu_dpm_set_power_gate(struct smu_context *smu, uint32_t block_type, return ret; } +/** + * smu_set_user_clk_dependencies - set user profile clock dependencies + * + * @smu: smu_context pointer + * @clk: enum smu_clk_type type + * + * Enable/Disable the clock dependency for the @clk type. + */ +static void smu_set_user_clk_dependencies(struct smu_context *smu, enum smu_clk_type clk) +{ + if (smu->adev->in_suspend) + return; + + /* + * mclk, fclk and socclk are interdependent + * on each other + */ + if (clk == SMU_MCLK) { + /* reset clock dependency */ + smu->user_dpm_profile.clk_dependency = 0; + /* set mclk dependent clocks(fclk and socclk) */ + smu->user_dpm_profile.clk_dependency = BIT(SMU_FCLK) | BIT(SMU_SOCCLK); + } else if (clk == SMU_FCLK) { + /* give priority to mclk, if mclk dependent clocks are set */ + if (smu->user_dpm_profile.clk_dependency == (BIT(SMU_FCLK) | BIT(SMU_SOCCLK))) + return; + + /* reset clock dependency */ + smu->user_dpm_profile.clk_dependency = 0; + /* set fclk dependent clocks(mclk and socclk) */ + smu->user_dpm_profile.clk_dependency = BIT(SMU_MCLK) | BIT(SMU_SOCCLK); + } else if (clk == SMU_SOCCLK) { + /* give priority to mclk, if mclk dependent clocks are set */ + if (smu->user_dpm_profile.clk_dependency == (BIT(SMU_FCLK) | BIT(SMU_SOCCLK))) + return; + + /* reset clock dependency */ + smu->user_dpm_profile.clk_dependency = 0; + /* set socclk dependent clocks(mclk and fclk) */ + smu->user_dpm_profile.clk_dependency = BIT(SMU_MCLK) | BIT(SMU_FCLK); + } else + /* add clk dependencies here, if any */ + return; +} + +/** + * smu_restore_dpm_user_profile - reinstate user dpm profile + * + * @smu: smu_context pointer + * + * Restore the saved user power configurations include power limit, + * clock frequencies, fan control mode and fan speed. + */ +static void smu_restore_dpm_user_profile(struct smu_context *smu) +{ + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); + int ret = 0; + + if (!smu->adev->in_suspend) + return; + + if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) + return; + + /* Enable restore flag */ + smu->user_dpm_profile.flags = SMU_DPM_USER_PROFILE_RESTORE; + + /* set the user dpm power limit */ + if (smu->user_dpm_profile.power_limit) { + ret = smu_set_power_limit(smu, smu->user_dpm_profile.power_limit); + if (ret) + dev_err(smu->adev->dev, "Failed to set power limit value\n"); + } + + /* set the user dpm clock configurations */ + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { + enum smu_clk_type clk_type; + + for (clk_type = 0; clk_type < SMU_CLK_COUNT; clk_type++) { + /* + * Iterate over smu clk type and force the saved user clk + * configs, skip if clock dependency is enabled + */ + if (!(smu->user_dpm_profile.clk_dependency & BIT(clk_type)) && + smu->user_dpm_profile.clk_mask[clk_type]) { + ret = smu_force_clk_levels(smu, clk_type, + smu->user_dpm_profile.clk_mask[clk_type]); + if (ret) + dev_err(smu->adev->dev, "Failed to set clock type = %d\n", + clk_type); + } + } + } + + /* set the user dpm fan configurations */ + if (smu->user_dpm_profile.fan_mode == AMD_FAN_CTRL_MANUAL) { + ret = smu_set_fan_control_mode(smu, smu->user_dpm_profile.fan_mode); + if (ret) { + dev_err(smu->adev->dev, "Failed to set manual fan control mode\n"); + return; + } + + if (!ret && smu->user_dpm_profile.fan_speed_percent) { + ret = smu_set_fan_speed_percent(smu, smu->user_dpm_profile.fan_speed_percent); + if (ret) + dev_err(smu->adev->dev, "Failed to set manual fan speed\n"); + } + } + + /* Disable restore flag */ + smu->user_dpm_profile.flags &= ~SMU_DPM_USER_PROFILE_RESTORE; +} + int smu_get_power_num_states(struct smu_context *smu, struct pp_states_info *state_info) { @@ -288,6 +401,20 @@ bool is_support_sw_smu(struct amdgpu_device *adev) return false; } +bool is_support_cclk_dpm(struct amdgpu_device *adev) +{ + struct smu_context *smu = &adev->smu; + + if (!is_support_sw_smu(adev)) + return false; + + if (!smu_feature_is_enabled(smu, SMU_FEATURE_CCLK_DPM_BIT)) + return false; + + return true; +} + + int smu_sys_get_pp_table(struct smu_context *smu, void **table) { struct smu_table_context *smu_table = &smu->smu_table; @@ -405,8 +532,6 @@ static int smu_set_funcs(struct amdgpu_device *adev) break; case CHIP_VANGOGH: vangogh_set_ppt_funcs(smu); - /* enable the OD by default to allow the fine grain tuning function */ - smu->od_enabled = true; break; default: return -EINVAL; @@ -478,9 +603,6 @@ static int smu_late_init(void *handle) smu_set_fine_grain_gfx_freq_parameters(smu); - if (adev->asic_type == CHIP_VANGOGH) - return 0; - if (!smu->pm_enabled) return 0; @@ -517,6 +639,8 @@ static int smu_late_init(void *handle) AMD_PP_TASK_COMPLETE_INIT, false); + smu_restore_dpm_user_profile(smu); + return 0; } @@ -1610,6 +1734,12 @@ int smu_force_performance_level(struct smu_context *smu, enum amd_dpm_forced_lev mutex_unlock(&smu->mutex); + /* reset user dpm clock state */ + if (!ret && smu_dpm_ctx->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL) { + memset(smu->user_dpm_profile.clk_mask, 0, sizeof(smu->user_dpm_profile.clk_mask)); + smu->user_dpm_profile.clk_dependency = 0; + } + return ret; } @@ -1644,8 +1774,13 @@ int smu_force_clk_levels(struct smu_context *smu, mutex_lock(&smu->mutex); - if (smu->ppt_funcs && smu->ppt_funcs->force_clk_levels) + if (smu->ppt_funcs && smu->ppt_funcs->force_clk_levels) { ret = smu->ppt_funcs->force_clk_levels(smu, clk_type, mask); + if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) { + smu->user_dpm_profile.clk_mask[clk_type] = mask; + smu_set_user_clk_dependencies(smu, clk_type); + } + } mutex_unlock(&smu->mutex); @@ -1887,6 +2022,7 @@ int smu_set_gfx_cgpg(struct smu_context *smu, bool enabled) int smu_set_fan_speed_rpm(struct smu_context *smu, uint32_t speed) { + u32 percent; int ret = 0; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) @@ -1894,8 +2030,12 @@ int smu_set_fan_speed_rpm(struct smu_context *smu, uint32_t speed) mutex_lock(&smu->mutex); - if (smu->ppt_funcs->set_fan_speed_rpm) - ret = smu->ppt_funcs->set_fan_speed_rpm(smu, speed); + if (smu->ppt_funcs->set_fan_speed_percent) { + percent = speed * 100 / smu->fan_max_rpm; + ret = smu->ppt_funcs->set_fan_speed_percent(smu, percent); + if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + smu->user_dpm_profile.fan_speed_percent = percent; + } mutex_unlock(&smu->mutex); @@ -1904,22 +2044,9 @@ int smu_set_fan_speed_rpm(struct smu_context *smu, uint32_t speed) int smu_get_power_limit(struct smu_context *smu, uint32_t *limit, - bool max_setting) -{ - if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) - return -EOPNOTSUPP; - - mutex_lock(&smu->mutex); - - *limit = (max_setting ? smu->max_power_limit : smu->current_power_limit); - - mutex_unlock(&smu->mutex); - - return 0; -} - -int smu_set_power_limit(struct smu_context *smu, uint32_t limit) + enum smu_ppt_limit_level limit_level) { + uint32_t limit_type = *limit >> 24; int ret = 0; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) @@ -1927,6 +2054,43 @@ int smu_set_power_limit(struct smu_context *smu, uint32_t limit) mutex_lock(&smu->mutex); + if (limit_type != SMU_DEFAULT_PPT_LIMIT) { + if (smu->ppt_funcs->get_ppt_limit) + ret = smu->ppt_funcs->get_ppt_limit(smu, limit, limit_type, limit_level); + } else { + switch (limit_level) { + case SMU_PPT_LIMIT_CURRENT: + *limit = smu->current_power_limit; + break; + case SMU_PPT_LIMIT_MAX: + *limit = smu->max_power_limit; + break; + default: + break; + } + } + + mutex_unlock(&smu->mutex); + + return ret; +} + +int smu_set_power_limit(struct smu_context *smu, uint32_t limit) +{ + uint32_t limit_type = limit >> 24; + int ret = 0; + + if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) + return -EOPNOTSUPP; + + mutex_lock(&smu->mutex); + + if (limit_type != SMU_DEFAULT_PPT_LIMIT) + if (smu->ppt_funcs->set_power_limit) { + ret = smu->ppt_funcs->set_power_limit(smu, limit); + goto out; + } + if (limit > smu->max_power_limit) { dev_err(smu->adev->dev, "New power limit (%d) is over the max allowed %d\n", @@ -1937,8 +2101,11 @@ int smu_set_power_limit(struct smu_context *smu, uint32_t limit) if (!limit) limit = smu->current_power_limit; - if (smu->ppt_funcs->set_power_limit) + if (smu->ppt_funcs->set_power_limit) { ret = smu->ppt_funcs->set_power_limit(smu, limit); + if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + smu->user_dpm_profile.power_limit = limit; + } out: mutex_unlock(&smu->mutex); @@ -2115,11 +2282,19 @@ int smu_set_fan_control_mode(struct smu_context *smu, int value) mutex_lock(&smu->mutex); - if (smu->ppt_funcs->set_fan_control_mode) + if (smu->ppt_funcs->set_fan_control_mode) { ret = smu->ppt_funcs->set_fan_control_mode(smu, value); + if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + smu->user_dpm_profile.fan_mode = value; + } mutex_unlock(&smu->mutex); + /* reset user dpm fan speed */ + if (!ret && value != AMD_FAN_CTRL_MANUAL && + smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + smu->user_dpm_profile.fan_speed_percent = 0; + return ret; } @@ -2127,17 +2302,15 @@ int smu_get_fan_speed_percent(struct smu_context *smu, uint32_t *speed) { int ret = 0; uint32_t percent; - uint32_t current_rpm; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) return -EOPNOTSUPP; mutex_lock(&smu->mutex); - if (smu->ppt_funcs->get_fan_speed_rpm) { - ret = smu->ppt_funcs->get_fan_speed_rpm(smu, ¤t_rpm); + if (smu->ppt_funcs->get_fan_speed_percent) { + ret = smu->ppt_funcs->get_fan_speed_percent(smu, &percent); if (!ret) { - percent = current_rpm * 100 / smu->fan_max_rpm; *speed = percent > 100 ? 100 : percent; } } @@ -2157,8 +2330,13 @@ int smu_set_fan_speed_percent(struct smu_context *smu, uint32_t speed) mutex_lock(&smu->mutex); - if (smu->ppt_funcs->set_fan_speed_percent) + if (smu->ppt_funcs->set_fan_speed_percent) { + if (speed > 100) + speed = 100; ret = smu->ppt_funcs->set_fan_speed_percent(smu, speed); + if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + smu->user_dpm_profile.fan_speed_percent = speed; + } mutex_unlock(&smu->mutex); @@ -2168,14 +2346,17 @@ int smu_set_fan_speed_percent(struct smu_context *smu, uint32_t speed) int smu_get_fan_speed_rpm(struct smu_context *smu, uint32_t *speed) { int ret = 0; + u32 percent; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) return -EOPNOTSUPP; mutex_lock(&smu->mutex); - if (smu->ppt_funcs->get_fan_speed_rpm) - ret = smu->ppt_funcs->get_fan_speed_rpm(smu, speed); + if (smu->ppt_funcs->get_fan_speed_percent) { + ret = smu->ppt_funcs->get_fan_speed_percent(smu, &percent); + *speed = percent * smu->fan_max_rpm / 100; + } mutex_unlock(&smu->mutex); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c index 16db0b506b0d..45564a776e9b 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c @@ -1080,15 +1080,27 @@ static int arcturus_read_sensor(struct smu_context *smu, return ret; } -static int arcturus_get_fan_speed_rpm(struct smu_context *smu, - uint32_t *speed) +static int arcturus_get_fan_speed_percent(struct smu_context *smu, + uint32_t *speed) { + int ret; + u32 rpm; + if (!speed) return -EINVAL; - return arcturus_get_smu_metrics_data(smu, - METRICS_CURR_FANSPEED, - speed); + switch (smu_v11_0_get_fan_control_mode(smu)) { + case AMD_FAN_CTRL_AUTO: + ret = arcturus_get_smu_metrics_data(smu, + METRICS_CURR_FANSPEED, + &rpm); + if (!ret && smu->fan_max_rpm) + *speed = rpm * 100 / smu->fan_max_rpm; + return ret; + default: + *speed = smu->user_dpm_profile.fan_speed_percent; + return 0; + } } static int arcturus_get_fan_parameters(struct smu_context *smu) @@ -2227,7 +2239,7 @@ static ssize_t arcturus_get_gpu_metrics(struct smu_context *smu, if (ret) return ret; - smu_v11_0_init_gpu_metrics_v1_0(gpu_metrics); + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 0); gpu_metrics->temperature_edge = metrics.TemperatureEdge; gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot; @@ -2264,6 +2276,8 @@ static ssize_t arcturus_get_gpu_metrics(struct smu_context *smu, gpu_metrics->pcie_link_speed = arcturus_get_current_pcie_link_speed(smu); + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + *table = (void *)gpu_metrics; return sizeof(struct gpu_metrics_v1_0); @@ -2281,7 +2295,7 @@ static const struct pptable_funcs arcturus_ppt_funcs = { .print_clk_levels = arcturus_print_clk_levels, .force_clk_levels = arcturus_force_clk_levels, .read_sensor = arcturus_read_sensor, - .get_fan_speed_rpm = arcturus_get_fan_speed_rpm, + .get_fan_speed_percent = arcturus_get_fan_speed_percent, .get_power_profile_mode = arcturus_get_power_profile_mode, .set_power_profile_mode = arcturus_set_power_profile_mode, .set_performance_level = arcturus_set_performance_level, @@ -2327,7 +2341,6 @@ static const struct pptable_funcs arcturus_ppt_funcs = { .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, - .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, .register_irq_handler = smu_v11_0_register_irq_handler, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c index cd7efa923195..6e641f1513d8 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c @@ -1317,15 +1317,27 @@ static bool navi10_is_dpm_running(struct smu_context *smu) return !!(feature_enabled & SMC_DPM_FEATURE); } -static int navi10_get_fan_speed_rpm(struct smu_context *smu, - uint32_t *speed) +static int navi10_get_fan_speed_percent(struct smu_context *smu, + uint32_t *speed) { + int ret; + u32 rpm; + if (!speed) return -EINVAL; - return navi10_get_smu_metrics_data(smu, - METRICS_CURR_FANSPEED, - speed); + switch (smu_v11_0_get_fan_control_mode(smu)) { + case AMD_FAN_CTRL_AUTO: + ret = navi10_get_smu_metrics_data(smu, + METRICS_CURR_FANSPEED, + &rpm); + if (!ret && smu->fan_max_rpm) + *speed = rpm * 100 / smu->fan_max_rpm; + return ret; + default: + *speed = smu->user_dpm_profile.fan_speed_percent; + return 0; + } } static int navi10_get_fan_parameters(struct smu_context *smu) @@ -1673,7 +1685,7 @@ static int navi10_read_sensor(struct smu_context *smu, *size = 4; break; case AMDGPU_PP_SENSOR_GFX_SCLK: - ret = navi10_get_current_clk_freq_by_table(smu, SMU_GFXCLK, (uint32_t *)data); + ret = navi10_get_smu_metrics_data(smu, METRICS_AVERAGE_GFXCLK, (uint32_t *)data); *(uint32_t *)data *= 100; *size = 4; break; @@ -2302,7 +2314,7 @@ static ssize_t navi10_get_gpu_metrics(struct smu_context *smu, mutex_unlock(&smu->metrics_lock); - smu_v11_0_init_gpu_metrics_v1_0(gpu_metrics); + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 0); gpu_metrics->temperature_edge = metrics.TemperatureEdge; gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot; @@ -2342,6 +2354,8 @@ static ssize_t navi10_get_gpu_metrics(struct smu_context *smu, gpu_metrics->pcie_link_speed = smu_v11_0_get_current_pcie_link_speed(smu); + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + *table = (void *)gpu_metrics; return sizeof(struct gpu_metrics_v1_0); @@ -2413,7 +2427,7 @@ static const struct pptable_funcs navi10_ppt_funcs = { .display_config_changed = navi10_display_config_changed, .notify_smc_display_config = navi10_notify_smc_display_config, .is_dpm_running = navi10_is_dpm_running, - .get_fan_speed_rpm = navi10_get_fan_speed_rpm, + .get_fan_speed_percent = navi10_get_fan_speed_percent, .get_power_profile_mode = navi10_get_power_profile_mode, .set_power_profile_mode = navi10_set_power_profile_mode, .set_watermarks_table = navi10_set_watermarks_table, @@ -2457,7 +2471,6 @@ static const struct pptable_funcs navi10_ppt_funcs = { .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, - .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, .register_irq_handler = smu_v11_0_register_irq_handler, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index d68d3dfee51d..af73e1430af5 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -261,6 +261,11 @@ sienna_cichlid_get_allowed_feature_mask(struct smu_context *smu, *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFX_GPO_BIT); } + if ((adev->pm.pp_feature & PP_GFX_DCS_MASK) && + (adev->asic_type > CHIP_SIENNA_CICHLID) && + !(adev->flags & AMD_IS_APU)) + *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_DCS_BIT); + if (adev->pm.pp_feature & PP_MCLK_DPM_MASK) *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_UCLK_BIT) | FEATURE_MASK(FEATURE_MEM_VDDCI_SCALING_BIT) @@ -294,6 +299,12 @@ sienna_cichlid_get_allowed_feature_mask(struct smu_context *smu, smu->adev->pg_flags & AMD_PG_SUPPORT_JPEG) *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_MM_DPM_PG_BIT); + if (smu->dc_controlled_by_gpio) + *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_ACDC_BIT); + + if (amdgpu_aspm == 1) + *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_LCLK_BIT); + return 0; } @@ -314,6 +325,12 @@ static int sienna_cichlid_check_powerplay_table(struct smu_context *smu) table_context->thermal_controller_type = powerplay_table->thermal_controller_type; + /* + * Instead of having its own buffer space and get overdrive_table copied, + * smu->od_settings just points to the actual overdrive_table + */ + smu->od_settings = &powerplay_table->overdrive_table; + return 0; } @@ -907,6 +924,22 @@ static bool sienna_cichlid_is_support_fine_grained_dpm(struct smu_context *smu, return dpm_desc->SnapToDiscrete == 0 ? true : false; } +static bool sienna_cichlid_is_od_feature_supported(struct smu_11_0_7_overdrive_table *od_table, + enum SMU_11_0_7_ODFEATURE_CAP cap) +{ + return od_table->cap[cap]; +} + +static void sienna_cichlid_get_od_setting_range(struct smu_11_0_7_overdrive_table *od_table, + enum SMU_11_0_7_ODSETTING_ID setting, + uint32_t *min, uint32_t *max) +{ + if (min) + *min = od_table->min[setting]; + if (max) + *max = od_table->max[setting]; +} + static int sienna_cichlid_print_clk_levels(struct smu_context *smu, enum smu_clk_type clk_type, char *buf) { @@ -915,11 +948,16 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, struct smu_dpm_context *smu_dpm = &smu->smu_dpm; struct smu_11_0_dpm_context *dpm_context = smu_dpm->dpm_context; PPTable_t *pptable = (PPTable_t *)table_context->driver_pptable; + struct smu_11_0_7_overdrive_table *od_settings = smu->od_settings; + OverDriveTable_t *od_table = + (OverDriveTable_t *)table_context->overdrive_table; int i, size = 0, ret = 0; uint32_t cur_value = 0, value = 0, count = 0; uint32_t freq_values[3] = {0}; uint32_t mark_index = 0; uint32_t gen_speed, lane_width; + uint32_t min_value, max_value; + uint32_t smu_version; switch (clk_type) { case SMU_GFXCLK: @@ -995,6 +1033,70 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, (lane_width == dpm_context->dpm_tables.pcie_table.pcie_lane[i]) ? "*" : ""); break; + case SMU_OD_SCLK: + if (!smu->od_enabled || !od_table || !od_settings) + break; + + if (!sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_GFXCLK_LIMITS)) + break; + + size += sprintf(buf + size, "OD_SCLK:\n"); + size += sprintf(buf + size, "0: %uMhz\n1: %uMhz\n", od_table->GfxclkFmin, od_table->GfxclkFmax); + break; + + case SMU_OD_MCLK: + if (!smu->od_enabled || !od_table || !od_settings) + break; + + if (!sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_UCLK_LIMITS)) + break; + + size += sprintf(buf + size, "OD_MCLK:\n"); + size += sprintf(buf + size, "0: %uMhz\n1: %uMHz\n", od_table->UclkFmin, od_table->UclkFmax); + break; + + case SMU_OD_VDDGFX_OFFSET: + if (!smu->od_enabled || !od_table || !od_settings) + break; + + /* + * OD GFX Voltage Offset functionality is supported only by 58.41.0 + * and onwards SMU firmwares. + */ + smu_cmn_get_smc_version(smu, NULL, &smu_version); + if ((adev->asic_type == CHIP_SIENNA_CICHLID) && + (smu_version < 0x003a2900)) + break; + + size += sprintf(buf + size, "OD_VDDGFX_OFFSET:\n"); + size += sprintf(buf + size, "%dmV\n", od_table->VddGfxOffset); + break; + + case SMU_OD_RANGE: + if (!smu->od_enabled || !od_table || !od_settings) + break; + + size = sprintf(buf, "%s:\n", "OD_RANGE"); + + if (sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_GFXCLK_LIMITS)) { + sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_GFXCLKFMIN, + &min_value, NULL); + sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_GFXCLKFMAX, + NULL, &max_value); + size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", + min_value, max_value); + } + + if (sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_UCLK_LIMITS)) { + sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_UCLKFMIN, + &min_value, NULL); + sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_UCLKFMAX, + NULL, &max_value); + size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n", + min_value, max_value); + } + break; + default: break; } @@ -1146,15 +1248,27 @@ static bool sienna_cichlid_is_dpm_running(struct smu_context *smu) return !!(feature_enabled & SMC_DPM_FEATURE); } -static int sienna_cichlid_get_fan_speed_rpm(struct smu_context *smu, - uint32_t *speed) +static int sienna_cichlid_get_fan_speed_percent(struct smu_context *smu, + uint32_t *speed) { + int ret; + u32 rpm; + if (!speed) return -EINVAL; - return sienna_cichlid_get_smu_metrics_data(smu, - METRICS_CURR_FANSPEED, - speed); + switch (smu_v11_0_get_fan_control_mode(smu)) { + case AMD_FAN_CTRL_AUTO: + ret = sienna_cichlid_get_smu_metrics_data(smu, + METRICS_CURR_FANSPEED, + &rpm); + if (!ret && smu->fan_max_rpm) + *speed = rpm * 100 / smu->fan_max_rpm; + return ret; + default: + *speed = smu->user_dpm_profile.fan_speed_percent; + return 0; + } } static int sienna_cichlid_get_fan_parameters(struct smu_context *smu) @@ -1694,6 +1808,243 @@ static int sienna_cichlid_get_dpm_ultimate_freq(struct smu_context *smu, return ret; } +static void sienna_cichlid_dump_od_table(struct smu_context *smu, + OverDriveTable_t *od_table) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t smu_version; + + dev_dbg(smu->adev->dev, "OD: Gfxclk: (%d, %d)\n", od_table->GfxclkFmin, + od_table->GfxclkFmax); + dev_dbg(smu->adev->dev, "OD: Uclk: (%d, %d)\n", od_table->UclkFmin, + od_table->UclkFmax); + + smu_cmn_get_smc_version(smu, NULL, &smu_version); + if (!((adev->asic_type == CHIP_SIENNA_CICHLID) && + (smu_version < 0x003a2900))) + dev_dbg(smu->adev->dev, "OD: VddGfxOffset: %d\n", od_table->VddGfxOffset); +} + +static int sienna_cichlid_set_default_od_settings(struct smu_context *smu) +{ + OverDriveTable_t *od_table = + (OverDriveTable_t *)smu->smu_table.overdrive_table; + OverDriveTable_t *boot_od_table = + (OverDriveTable_t *)smu->smu_table.boot_overdrive_table; + int ret = 0; + + ret = smu_cmn_update_table(smu, SMU_TABLE_OVERDRIVE, + 0, (void *)od_table, false); + if (ret) { + dev_err(smu->adev->dev, "Failed to get overdrive table!\n"); + return ret; + } + + memcpy(boot_od_table, od_table, sizeof(OverDriveTable_t)); + + sienna_cichlid_dump_od_table(smu, od_table); + + return 0; +} + +static int sienna_cichlid_od_setting_check_range(struct smu_context *smu, + struct smu_11_0_7_overdrive_table *od_table, + enum SMU_11_0_7_ODSETTING_ID setting, + uint32_t value) +{ + if (value < od_table->min[setting]) { + dev_warn(smu->adev->dev, "OD setting (%d, %d) is less than the minimum allowed (%d)\n", + setting, value, od_table->min[setting]); + return -EINVAL; + } + if (value > od_table->max[setting]) { + dev_warn(smu->adev->dev, "OD setting (%d, %d) is greater than the maximum allowed (%d)\n", + setting, value, od_table->max[setting]); + return -EINVAL; + } + + return 0; +} + +static int sienna_cichlid_od_edit_dpm_table(struct smu_context *smu, + enum PP_OD_DPM_TABLE_COMMAND type, + long input[], uint32_t size) +{ + struct smu_table_context *table_context = &smu->smu_table; + OverDriveTable_t *od_table = + (OverDriveTable_t *)table_context->overdrive_table; + struct smu_11_0_7_overdrive_table *od_settings = + (struct smu_11_0_7_overdrive_table *)smu->od_settings; + struct amdgpu_device *adev = smu->adev; + enum SMU_11_0_7_ODSETTING_ID freq_setting; + uint16_t *freq_ptr; + int i, ret = 0; + uint32_t smu_version; + + if (!smu->od_enabled) { + dev_warn(smu->adev->dev, "OverDrive is not enabled!\n"); + return -EINVAL; + } + + if (!smu->od_settings) { + dev_err(smu->adev->dev, "OD board limits are not set!\n"); + return -ENOENT; + } + + if (!(table_context->overdrive_table && table_context->boot_overdrive_table)) { + dev_err(smu->adev->dev, "Overdrive table was not initialized!\n"); + return -EINVAL; + } + + switch (type) { + case PP_OD_EDIT_SCLK_VDDC_TABLE: + if (!sienna_cichlid_is_od_feature_supported(od_settings, + SMU_11_0_7_ODCAP_GFXCLK_LIMITS)) { + dev_warn(smu->adev->dev, "GFXCLK_LIMITS not supported!\n"); + return -ENOTSUPP; + } + + for (i = 0; i < size; i += 2) { + if (i + 2 > size) { + dev_info(smu->adev->dev, "invalid number of input parameters %d\n", size); + return -EINVAL; + } + + switch (input[i]) { + case 0: + if (input[i + 1] > od_table->GfxclkFmax) { + dev_info(smu->adev->dev, "GfxclkFmin (%ld) must be <= GfxclkFmax (%u)!\n", + input[i + 1], od_table->GfxclkFmax); + return -EINVAL; + } + + freq_setting = SMU_11_0_7_ODSETTING_GFXCLKFMIN; + freq_ptr = &od_table->GfxclkFmin; + break; + + case 1: + if (input[i + 1] < od_table->GfxclkFmin) { + dev_info(smu->adev->dev, "GfxclkFmax (%ld) must be >= GfxclkFmin (%u)!\n", + input[i + 1], od_table->GfxclkFmin); + return -EINVAL; + } + + freq_setting = SMU_11_0_7_ODSETTING_GFXCLKFMAX; + freq_ptr = &od_table->GfxclkFmax; + break; + + default: + dev_info(smu->adev->dev, "Invalid SCLK_VDDC_TABLE index: %ld\n", input[i]); + dev_info(smu->adev->dev, "Supported indices: [0:min,1:max]\n"); + return -EINVAL; + } + + ret = sienna_cichlid_od_setting_check_range(smu, od_settings, + freq_setting, input[i + 1]); + if (ret) + return ret; + + *freq_ptr = (uint16_t)input[i + 1]; + } + break; + + case PP_OD_EDIT_MCLK_VDDC_TABLE: + if (!sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_UCLK_LIMITS)) { + dev_warn(smu->adev->dev, "UCLK_LIMITS not supported!\n"); + return -ENOTSUPP; + } + + for (i = 0; i < size; i += 2) { + if (i + 2 > size) { + dev_info(smu->adev->dev, "invalid number of input parameters %d\n", size); + return -EINVAL; + } + + switch (input[i]) { + case 0: + if (input[i + 1] > od_table->UclkFmax) { + dev_info(smu->adev->dev, "UclkFmin (%ld) must be <= UclkFmax (%u)!\n", + input[i + 1], od_table->UclkFmax); + return -EINVAL; + } + + freq_setting = SMU_11_0_7_ODSETTING_UCLKFMIN; + freq_ptr = &od_table->UclkFmin; + break; + + case 1: + if (input[i + 1] < od_table->UclkFmin) { + dev_info(smu->adev->dev, "UclkFmax (%ld) must be >= UclkFmin (%u)!\n", + input[i + 1], od_table->UclkFmin); + return -EINVAL; + } + + freq_setting = SMU_11_0_7_ODSETTING_UCLKFMAX; + freq_ptr = &od_table->UclkFmax; + break; + + default: + dev_info(smu->adev->dev, "Invalid MCLK_VDDC_TABLE index: %ld\n", input[i]); + dev_info(smu->adev->dev, "Supported indices: [0:min,1:max]\n"); + return -EINVAL; + } + + ret = sienna_cichlid_od_setting_check_range(smu, od_settings, + freq_setting, input[i + 1]); + if (ret) + return ret; + + *freq_ptr = (uint16_t)input[i + 1]; + } + break; + + case PP_OD_RESTORE_DEFAULT_TABLE: + memcpy(table_context->overdrive_table, + table_context->boot_overdrive_table, + sizeof(OverDriveTable_t)); + fallthrough; + + case PP_OD_COMMIT_DPM_TABLE: + sienna_cichlid_dump_od_table(smu, od_table); + + ret = smu_cmn_update_table(smu, SMU_TABLE_OVERDRIVE, + 0, (void *)od_table, true); + if (ret) { + dev_err(smu->adev->dev, "Failed to import overdrive table!\n"); + return ret; + } + break; + + case PP_OD_EDIT_VDDGFX_OFFSET: + if (size != 1) { + dev_info(smu->adev->dev, "invalid number of parameters: %d\n", size); + return -EINVAL; + } + + /* + * OD GFX Voltage Offset functionality is supported only by 58.41.0 + * and onwards SMU firmwares. + */ + smu_cmn_get_smc_version(smu, NULL, &smu_version); + if ((adev->asic_type == CHIP_SIENNA_CICHLID) && + (smu_version < 0x003a2900)) { + dev_err(smu->adev->dev, "OD GFX Voltage offset functionality is supported " + "only by 58.41.0 and onwards SMU firmwares!\n"); + return -EOPNOTSUPP; + } + + od_table->VddGfxOffset = (int16_t)input[0]; + + sienna_cichlid_dump_od_table(smu, od_table); + break; + + default: + return -ENOSYS; + } + + return ret; +} + static int sienna_cichlid_run_btc(struct smu_context *smu) { return smu_cmn_send_smc_msg(smu, SMU_MSG_RunDcBtc, NULL); @@ -2610,7 +2961,7 @@ static ssize_t sienna_cichlid_get_gpu_metrics(struct smu_context *smu, if (ret) return ret; - smu_v11_0_init_gpu_metrics_v1_0(gpu_metrics); + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 0); gpu_metrics->temperature_edge = metrics->TemperatureEdge; gpu_metrics->temperature_hotspot = metrics->TemperatureHotspot; @@ -2653,6 +3004,8 @@ static ssize_t sienna_cichlid_get_gpu_metrics(struct smu_context *smu, gpu_metrics->pcie_link_speed = smu_v11_0_get_current_pcie_link_speed(smu); + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + *table = (void *)gpu_metrics; return sizeof(struct gpu_metrics_v1_0); @@ -2759,7 +3112,7 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .display_config_changed = sienna_cichlid_display_config_changed, .notify_smc_display_config = sienna_cichlid_notify_smc_display_config, .is_dpm_running = sienna_cichlid_is_dpm_running, - .get_fan_speed_rpm = sienna_cichlid_get_fan_speed_rpm, + .get_fan_speed_percent = sienna_cichlid_get_fan_speed_percent, .get_power_profile_mode = sienna_cichlid_get_power_profile_mode, .set_power_profile_mode = sienna_cichlid_set_power_profile_mode, .set_watermarks_table = sienna_cichlid_set_watermarks_table, @@ -2803,7 +3156,6 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, - .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, .register_irq_handler = smu_v11_0_register_irq_handler, @@ -2818,6 +3170,8 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .mode1_reset = smu_v11_0_mode1_reset, .get_dpm_ultimate_freq = sienna_cichlid_get_dpm_ultimate_freq, .set_soft_freq_limited_range = smu_v11_0_set_soft_freq_limited_range, + .set_default_od_settings = sienna_cichlid_set_default_od_settings, + .od_edit_dpm_table = sienna_cichlid_od_edit_dpm_table, .run_btc = sienna_cichlid_run_btc, .set_power_source = smu_v11_0_set_power_source, .get_pp_feature_mask = smu_cmn_get_pp_feature_mask, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c index 5aeb5f5a0447..90585461a56e 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c @@ -474,12 +474,14 @@ int smu_v11_0_fini_smc_tables(struct smu_context *smu) int smu_v11_0_init_power(struct smu_context *smu) { struct smu_power_context *smu_power = &smu->smu_power; + size_t size = smu->adev->asic_type == CHIP_VANGOGH ? + sizeof(struct smu_11_5_power_context) : + sizeof(struct smu_11_0_power_context); - smu_power->power_context = kzalloc(sizeof(struct smu_11_0_power_context), - GFP_KERNEL); + smu_power->power_context = kzalloc(size, GFP_KERNEL); if (!smu_power->power_context) return -ENOMEM; - smu_power->power_context_size = sizeof(struct smu_11_0_power_context); + smu_power->power_context_size = size; return 0; } @@ -1119,6 +1121,7 @@ int smu_v11_0_gfx_off_control(struct smu_context *smu, bool enable) case CHIP_SIENNA_CICHLID: case CHIP_NAVY_FLOUNDER: case CHIP_DIMGREY_CAVEFISH: + case CHIP_VANGOGH: if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) return 0; if (enable) @@ -1136,10 +1139,10 @@ int smu_v11_0_gfx_off_control(struct smu_context *smu, bool enable) uint32_t smu_v11_0_get_fan_control_mode(struct smu_context *smu) { - if (!smu_cmn_feature_is_enabled(smu, SMU_FEATURE_FAN_CONTROL_BIT)) - return AMD_FAN_CTRL_MANUAL; - else + if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_FAN_CONTROL_BIT)) return AMD_FAN_CTRL_AUTO; + else + return smu->user_dpm_profile.fan_mode; } static int @@ -1230,58 +1233,6 @@ smu_v11_0_set_fan_control_mode(struct smu_context *smu, return ret; } -int smu_v11_0_set_fan_speed_rpm(struct smu_context *smu, - uint32_t speed) -{ - struct amdgpu_device *adev = smu->adev; - int ret; - uint32_t tach_period, crystal_clock_freq; - - if (!speed) - return -EINVAL; - - ret = smu_v11_0_auto_fan_control(smu, 0); - if (ret) - return ret; - - /* - * crystal_clock_freq div by 4 is required since the fan control - * module refers to 25MHz - */ - - crystal_clock_freq = amdgpu_asic_get_xclk(adev) / 4; - tach_period = 60 * crystal_clock_freq * 10000 / (8 * speed); - WREG32_SOC15(THM, 0, mmCG_TACH_CTRL, - REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_CTRL), - CG_TACH_CTRL, TARGET_PERIOD, - tach_period)); - - ret = smu_v11_0_set_fan_static_mode(smu, FDO_PWM_MODE_STATIC_RPM); - - return ret; -} - -int smu_v11_0_get_fan_speed_rpm(struct smu_context *smu, - uint32_t *speed) -{ - struct amdgpu_device *adev = smu->adev; - uint32_t tach_period, crystal_clock_freq; - uint64_t tmp64; - - tach_period = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_CTRL), - CG_TACH_CTRL, TARGET_PERIOD); - if (!tach_period) - return -EINVAL; - - crystal_clock_freq = amdgpu_asic_get_xclk(adev); - - tmp64 = (uint64_t)crystal_clock_freq * 60 * 10000; - do_div(tmp64, (tach_period * 8)); - *speed = (uint32_t)tmp64; - - return 0; -} - int smu_v11_0_set_xgmi_pstate(struct smu_context *smu, uint32_t pstate) { @@ -2072,30 +2023,6 @@ int smu_v11_0_get_current_pcie_link_speed(struct smu_context *smu) return link_speed[speed_level]; } -void smu_v11_0_init_gpu_metrics_v1_0(struct gpu_metrics_v1_0 *gpu_metrics) -{ - memset(gpu_metrics, 0xFF, sizeof(struct gpu_metrics_v1_0)); - - gpu_metrics->common_header.structure_size = - sizeof(struct gpu_metrics_v1_0); - gpu_metrics->common_header.format_revision = 1; - gpu_metrics->common_header.content_revision = 0; - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); -} - -void smu_v11_0_init_gpu_metrics_v2_0(struct gpu_metrics_v2_0 *gpu_metrics) -{ - memset(gpu_metrics, 0xFF, sizeof(struct gpu_metrics_v2_0)); - - gpu_metrics->common_header.structure_size = - sizeof(struct gpu_metrics_v2_0); - gpu_metrics->common_header.format_revision = 2; - gpu_metrics->common_header.content_revision = 0; - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); -} - int smu_v11_0_gfx_ulv_control(struct smu_context *smu, bool enablement) { diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c index 92ad2cdbae10..093b01159408 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c @@ -31,6 +31,10 @@ #include "smu_v11_5_ppsmc.h" #include "smu_v11_5_pmfw.h" #include "smu_cmn.h" +#include "soc15_common.h" +#include "asic_reg/gc/gc_10_3_0_offset.h" +#include "asic_reg/gc/gc_10_3_0_sh_mask.h" +#include /* * DO NOT use these for err/warn/info/debug messages. @@ -59,7 +63,8 @@ static struct cmn2asic_msg_mapping vangogh_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(GetSmuVersion, PPSMC_MSG_GetSmuVersion, 0), MSG_MAP(GetDriverIfVersion, PPSMC_MSG_GetDriverIfVersion, 0), MSG_MAP(EnableGfxOff, PPSMC_MSG_EnableGfxOff, 0), - MSG_MAP(DisallowGfxOff, PPSMC_MSG_DisableGfxOff, 0), + MSG_MAP(AllowGfxOff, PPSMC_MSG_AllowGfxOff, 0), + MSG_MAP(DisallowGfxOff, PPSMC_MSG_DisallowGfxOff, 0), MSG_MAP(PowerDownIspByTile, PPSMC_MSG_PowerDownIspByTile, 0), MSG_MAP(PowerUpIspByTile, PPSMC_MSG_PowerUpIspByTile, 0), MSG_MAP(PowerDownVcn, PPSMC_MSG_PowerDownVcn, 0), @@ -76,7 +81,6 @@ static struct cmn2asic_msg_mapping vangogh_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(TransferTableDram2Smu, PPSMC_MSG_TransferTableDram2Smu, 0), MSG_MAP(GfxDeviceDriverReset, PPSMC_MSG_GfxDeviceDriverReset, 0), MSG_MAP(GetEnabledSmuFeatures, PPSMC_MSG_GetEnabledSmuFeatures, 0), - MSG_MAP(Spare1, PPSMC_MSG_spare1, 0), MSG_MAP(SetHardMinSocclkByFreq, PPSMC_MSG_SetHardMinSocclkByFreq, 0), MSG_MAP(SetSoftMinFclk, PPSMC_MSG_SetSoftMinFclk, 0), MSG_MAP(SetSoftMinVcn, PPSMC_MSG_SetSoftMinVcn, 0), @@ -88,7 +92,6 @@ static struct cmn2asic_msg_mapping vangogh_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(SetSoftMaxSocclkByFreq, PPSMC_MSG_SetSoftMaxSocclkByFreq, 0), MSG_MAP(SetSoftMaxFclkByFreq, PPSMC_MSG_SetSoftMaxFclkByFreq, 0), MSG_MAP(SetSoftMaxVcn, PPSMC_MSG_SetSoftMaxVcn, 0), - MSG_MAP(Spare2, PPSMC_MSG_spare2, 0), MSG_MAP(SetPowerLimitPercentage, PPSMC_MSG_SetPowerLimitPercentage, 0), MSG_MAP(PowerDownJpeg, PPSMC_MSG_PowerDownJpeg, 0), MSG_MAP(PowerUpJpeg, PPSMC_MSG_PowerUpJpeg, 0), @@ -118,6 +121,11 @@ static struct cmn2asic_msg_mapping vangogh_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(StopDramLogging, PPSMC_MSG_StopDramLogging, 0), MSG_MAP(SetSoftMinCclk, PPSMC_MSG_SetSoftMinCclk, 0), MSG_MAP(SetSoftMaxCclk, PPSMC_MSG_SetSoftMaxCclk, 0), + MSG_MAP(RequestActiveWgp, PPSMC_MSG_RequestActiveWgp, 0), + MSG_MAP(SetFastPPTLimit, PPSMC_MSG_SetFastPPTLimit, 0), + MSG_MAP(SetSlowPPTLimit, PPSMC_MSG_SetSlowPPTLimit, 0), + MSG_MAP(GetFastPPTLimit, PPSMC_MSG_GetFastPPTLimit, 0), + MSG_MAP(GetSlowPPTLimit, PPSMC_MSG_GetSlowPPTLimit, 0), }; static struct cmn2asic_mapping vangogh_feature_mask_map[SMU_FEATURE_COUNT] = { @@ -162,6 +170,9 @@ static struct cmn2asic_mapping vangogh_feature_mask_map[SMU_FEATURE_COUNT] = { FEA_MAP(A55_DPM), FEA_MAP(CVIP_DSP_DPM), FEA_MAP(MSMU_LOW_POWER), + FEA_MAP_REVERSE(SOCCLK), + FEA_MAP_REVERSE(FCLK), + FEA_MAP_HALF_REVERSE(GFX), }; static struct cmn2asic_mapping vangogh_table_map[SMU_TABLE_COUNT] = { @@ -171,6 +182,14 @@ static struct cmn2asic_mapping vangogh_table_map[SMU_TABLE_COUNT] = { TAB_MAP_VALID(DPMCLOCKS), }; +static struct cmn2asic_mapping vangogh_workload_map[PP_SMC_POWER_PROFILE_COUNT] = { + WORKLOAD_MAP(PP_SMC_POWER_PROFILE_FULLSCREEN3D, WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT), + WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VIDEO, WORKLOAD_PPLIB_VIDEO_BIT), + WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VR, WORKLOAD_PPLIB_VR_BIT), + WORKLOAD_MAP(PP_SMC_POWER_PROFILE_COMPUTE, WORKLOAD_PPLIB_COMPUTE_BIT), + WORKLOAD_MAP(PP_SMC_POWER_PROFILE_CUSTOM, WORKLOAD_PPLIB_CUSTOM_BIT), +}; + static int vangogh_tables_init(struct smu_context *smu) { struct smu_table_context *smu_table = &smu->smu_table; @@ -242,6 +261,12 @@ static int vangogh_get_smu_metrics_data(struct smu_context *smu, case METRICS_AVERAGE_SOCCLK: *value = metrics->SocclkFrequency; break; + case METRICS_AVERAGE_VCLK: + *value = metrics->VclkFrequency; + break; + case METRICS_AVERAGE_DCLK: + *value = metrics->DclkFrequency; + break; case METRICS_AVERAGE_UCLK: *value = metrics->MemclkFrequency; break; @@ -272,6 +297,10 @@ static int vangogh_get_smu_metrics_data(struct smu_context *smu, case METRICS_VOLTAGE_VDDSOC: *value = metrics->Voltage[1]; break; + case METRICS_AVERAGE_CPUCLK: + memcpy(value, &metrics->CoreFrequency[0], + smu->cpu_core_num * sizeof(uint16_t)); + break; default: *value = UINT_MAX; break; @@ -308,6 +337,13 @@ static int vangogh_init_smc_tables(struct smu_context *smu) if (ret) return ret; +#ifdef CONFIG_X86 + /* AMD x86 APU only */ + smu->cpu_core_num = boot_cpu_data.x86_max_cores; +#else + smu->cpu_core_num = 4; +#endif + return smu_v11_0_init_smc_tables(smu); } @@ -317,17 +353,13 @@ static int vangogh_dpm_set_vcn_enable(struct smu_context *smu, bool enable) if (enable) { /* vcn dpm on is a prerequisite for vcn power gate messages */ - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_PG_BIT)) { - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerUpVcn, 0, NULL); - if (ret) - return ret; - } + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerUpVcn, 0, NULL); + if (ret) + return ret; } else { - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_PG_BIT)) { - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerDownVcn, 0, NULL); - if (ret) - return ret; - } + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerDownVcn, 0, NULL); + if (ret) + return ret; } return ret; @@ -338,54 +370,18 @@ static int vangogh_dpm_set_jpeg_enable(struct smu_context *smu, bool enable) int ret = 0; if (enable) { - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_JPEG_PG_BIT)) { - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerUpJpeg, 0, NULL); - if (ret) - return ret; - } + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerUpJpeg, 0, NULL); + if (ret) + return ret; } else { - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_JPEG_PG_BIT)) { - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerDownJpeg, 0, NULL); - if (ret) - return ret; - } + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerDownJpeg, 0, NULL); + if (ret) + return ret; } return ret; } -static int vangogh_get_allowed_feature_mask(struct smu_context *smu, - uint32_t *feature_mask, - uint32_t num) -{ - struct amdgpu_device *adev = smu->adev; - - if (num > 2) - return -EINVAL; - - memset(feature_mask, 0, sizeof(uint32_t) * num); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_DPM_BIT) - | FEATURE_MASK(FEATURE_MP0CLK_DPM_BIT) - | FEATURE_MASK(FEATURE_DS_SOCCLK_BIT) - | FEATURE_MASK(FEATURE_PPT_BIT) - | FEATURE_MASK(FEATURE_TDC_BIT) - | FEATURE_MASK(FEATURE_FAN_CONTROLLER_BIT) - | FEATURE_MASK(FEATURE_DS_LCLK_BIT) - | FEATURE_MASK(FEATURE_DS_DCFCLK_BIT); - - if (adev->pm.pp_feature & PP_SOCCLK_DPM_MASK) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_SOCCLK_DPM_BIT); - - if (adev->pm.pp_feature & PP_DCEFCLK_DPM_MASK) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DCFCLK_DPM_BIT); - - if (smu->adev->pg_flags & AMD_PG_SUPPORT_ATHUB) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_ATHUB_PG_BIT); - - return 0; -} - static bool vangogh_is_dpm_running(struct smu_context *smu) { int ret = 0; @@ -403,14 +399,68 @@ static bool vangogh_is_dpm_running(struct smu_context *smu) return !!(feature_enabled & SMC_DPM_FEATURE); } +static int vangogh_get_dpm_clk_limited(struct smu_context *smu, enum smu_clk_type clk_type, + uint32_t dpm_level, uint32_t *freq) +{ + DpmClocks_t *clk_table = smu->smu_table.clocks_table; + + if (!clk_table || clk_type >= SMU_CLK_COUNT) + return -EINVAL; + + switch (clk_type) { + case SMU_SOCCLK: + if (dpm_level >= clk_table->NumSocClkLevelsEnabled) + return -EINVAL; + *freq = clk_table->SocClocks[dpm_level]; + break; + case SMU_VCLK: + if (dpm_level >= clk_table->VcnClkLevelsEnabled) + return -EINVAL; + *freq = clk_table->VcnClocks[dpm_level].vclk; + break; + case SMU_DCLK: + if (dpm_level >= clk_table->VcnClkLevelsEnabled) + return -EINVAL; + *freq = clk_table->VcnClocks[dpm_level].dclk; + break; + case SMU_UCLK: + case SMU_MCLK: + if (dpm_level >= clk_table->NumDfPstatesEnabled) + return -EINVAL; + *freq = clk_table->DfPstateTable[dpm_level].memclk; + + break; + case SMU_FCLK: + if (dpm_level >= clk_table->NumDfPstatesEnabled) + return -EINVAL; + *freq = clk_table->DfPstateTable[dpm_level].fclk; + break; + default: + return -EINVAL; + } + + return 0; +} + static int vangogh_print_fine_grain_clk(struct smu_context *smu, enum smu_clk_type clk_type, char *buf) { - int size = 0; + DpmClocks_t *clk_table = smu->smu_table.clocks_table; + SmuMetrics_t metrics; + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); + int i, size = 0, ret = 0; + uint32_t cur_value = 0, value = 0, count = 0; + bool cur_value_match_level = false; + + memset(&metrics, 0, sizeof(metrics)); + + ret = smu_cmn_get_metrics_table(smu, &metrics, false); + if (ret) + return ret; switch (clk_type) { case SMU_OD_SCLK: - if (smu->od_enabled) { + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { size = sprintf(buf, "%s:\n", "OD_SCLK"); size += sprintf(buf + size, "0: %10uMhz\n", (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq); @@ -418,13 +468,72 @@ static int vangogh_print_fine_grain_clk(struct smu_context *smu, (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq); } break; + case SMU_OD_CCLK: + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { + size = sprintf(buf, "CCLK_RANGE in Core%d:\n", smu->cpu_core_id_select); + size += sprintf(buf + size, "0: %10uMhz\n", + (smu->cpu_actual_soft_min_freq > 0) ? smu->cpu_actual_soft_min_freq : smu->cpu_default_soft_min_freq); + size += sprintf(buf + size, "1: %10uMhz\n", + (smu->cpu_actual_soft_max_freq > 0) ? smu->cpu_actual_soft_max_freq : smu->cpu_default_soft_max_freq); + } + break; case SMU_OD_RANGE: - if (smu->od_enabled) { + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { size = sprintf(buf, "%s:\n", "OD_RANGE"); size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", smu->gfx_default_hard_min_freq, smu->gfx_default_soft_max_freq); + size += sprintf(buf + size, "CCLK: %7uMhz %10uMhz\n", + smu->cpu_default_soft_min_freq, smu->cpu_default_soft_max_freq); } break; + case SMU_SOCCLK: + /* the level 3 ~ 6 of socclk use the same frequency for vangogh */ + count = clk_table->NumSocClkLevelsEnabled; + cur_value = metrics.SocclkFrequency; + break; + case SMU_VCLK: + count = clk_table->VcnClkLevelsEnabled; + cur_value = metrics.VclkFrequency; + break; + case SMU_DCLK: + count = clk_table->VcnClkLevelsEnabled; + cur_value = metrics.DclkFrequency; + break; + case SMU_MCLK: + count = clk_table->NumDfPstatesEnabled; + cur_value = metrics.MemclkFrequency; + break; + case SMU_FCLK: + count = clk_table->NumDfPstatesEnabled; + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_GetFclkFrequency, 0, &cur_value); + if (ret) + return ret; + break; + default: + break; + } + + switch (clk_type) { + case SMU_SOCCLK: + case SMU_VCLK: + case SMU_DCLK: + case SMU_MCLK: + case SMU_FCLK: + for (i = 0; i < count; i++) { + ret = vangogh_get_dpm_clk_limited(smu, clk_type, i, &value); + if (ret) + return ret; + if (!value) + continue; + size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, + cur_value == value ? "*" : ""); + if (cur_value == value) + cur_value_match_level = true; + } + + if (!cur_value_match_level) + size += sprintf(buf + size, " %uMhz *\n", cur_value); + break; default: break; } @@ -432,6 +541,726 @@ static int vangogh_print_fine_grain_clk(struct smu_context *smu, return size; } +static int vangogh_get_profiling_clk_mask(struct smu_context *smu, + enum amd_dpm_forced_level level, + uint32_t *vclk_mask, + uint32_t *dclk_mask, + uint32_t *mclk_mask, + uint32_t *fclk_mask, + uint32_t *soc_mask) +{ + DpmClocks_t *clk_table = smu->smu_table.clocks_table; + + if (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK) { + if (mclk_mask) + *mclk_mask = clk_table->NumDfPstatesEnabled - 1; + + if (fclk_mask) + *fclk_mask = clk_table->NumDfPstatesEnabled - 1; + + if (soc_mask) + *soc_mask = 0; + } else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) { + if (mclk_mask) + *mclk_mask = 0; + + if (fclk_mask) + *fclk_mask = 0; + + if (soc_mask) + *soc_mask = 1; + + if (vclk_mask) + *vclk_mask = 1; + + if (dclk_mask) + *dclk_mask = 1; + } else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD) { + if (mclk_mask) + *mclk_mask = 0; + + if (fclk_mask) + *fclk_mask = 0; + + if (soc_mask) + *soc_mask = 1; + + if (vclk_mask) + *vclk_mask = 1; + + if (dclk_mask) + *dclk_mask = 1; + } + + return 0; +} + +static bool vangogh_clk_dpm_is_enabled(struct smu_context *smu, + enum smu_clk_type clk_type) +{ + enum smu_feature_mask feature_id = 0; + + switch (clk_type) { + case SMU_MCLK: + case SMU_UCLK: + case SMU_FCLK: + feature_id = SMU_FEATURE_DPM_FCLK_BIT; + break; + case SMU_GFXCLK: + case SMU_SCLK: + feature_id = SMU_FEATURE_DPM_GFXCLK_BIT; + break; + case SMU_SOCCLK: + feature_id = SMU_FEATURE_DPM_SOCCLK_BIT; + break; + case SMU_VCLK: + case SMU_DCLK: + feature_id = SMU_FEATURE_VCN_DPM_BIT; + break; + default: + return true; + } + + if (!smu_cmn_feature_is_enabled(smu, feature_id)) + return false; + + return true; +} + +static int vangogh_get_dpm_ultimate_freq(struct smu_context *smu, + enum smu_clk_type clk_type, + uint32_t *min, + uint32_t *max) +{ + int ret = 0; + uint32_t soc_mask; + uint32_t vclk_mask; + uint32_t dclk_mask; + uint32_t mclk_mask; + uint32_t fclk_mask; + uint32_t clock_limit; + + if (!vangogh_clk_dpm_is_enabled(smu, clk_type)) { + switch (clk_type) { + case SMU_MCLK: + case SMU_UCLK: + clock_limit = smu->smu_table.boot_values.uclk; + break; + case SMU_FCLK: + clock_limit = smu->smu_table.boot_values.fclk; + break; + case SMU_GFXCLK: + case SMU_SCLK: + clock_limit = smu->smu_table.boot_values.gfxclk; + break; + case SMU_SOCCLK: + clock_limit = smu->smu_table.boot_values.socclk; + break; + case SMU_VCLK: + clock_limit = smu->smu_table.boot_values.vclk; + break; + case SMU_DCLK: + clock_limit = smu->smu_table.boot_values.dclk; + break; + default: + clock_limit = 0; + break; + } + + /* clock in Mhz unit */ + if (min) + *min = clock_limit / 100; + if (max) + *max = clock_limit / 100; + + return 0; + } + if (max) { + ret = vangogh_get_profiling_clk_mask(smu, + AMD_DPM_FORCED_LEVEL_PROFILE_PEAK, + &vclk_mask, + &dclk_mask, + &mclk_mask, + &fclk_mask, + &soc_mask); + if (ret) + goto failed; + + switch (clk_type) { + case SMU_UCLK: + case SMU_MCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, mclk_mask, max); + if (ret) + goto failed; + break; + case SMU_SOCCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, soc_mask, max); + if (ret) + goto failed; + break; + case SMU_FCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, fclk_mask, max); + if (ret) + goto failed; + break; + case SMU_VCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, vclk_mask, max); + if (ret) + goto failed; + break; + case SMU_DCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, dclk_mask, max); + if (ret) + goto failed; + break; + default: + ret = -EINVAL; + goto failed; + } + } + if (min) { + switch (clk_type) { + case SMU_UCLK: + case SMU_MCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, mclk_mask, min); + if (ret) + goto failed; + break; + case SMU_SOCCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, soc_mask, min); + if (ret) + goto failed; + break; + case SMU_FCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, fclk_mask, min); + if (ret) + goto failed; + break; + case SMU_VCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, vclk_mask, min); + if (ret) + goto failed; + break; + case SMU_DCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, dclk_mask, min); + if (ret) + goto failed; + break; + default: + ret = -EINVAL; + goto failed; + } + } +failed: + return ret; +} + +static int vangogh_get_power_profile_mode(struct smu_context *smu, + char *buf) +{ + static const char *profile_name[] = { + "BOOTUP_DEFAULT", + "3D_FULL_SCREEN", + "POWER_SAVING", + "VIDEO", + "VR", + "COMPUTE", + "CUSTOM"}; + uint32_t i, size = 0; + int16_t workload_type = 0; + + if (!buf) + return -EINVAL; + + for (i = 0; i <= PP_SMC_POWER_PROFILE_CUSTOM; i++) { + /* + * Conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT + * Not all profile modes are supported on vangogh. + */ + workload_type = smu_cmn_to_asic_specific_index(smu, + CMN2ASIC_MAPPING_WORKLOAD, + i); + + if (workload_type < 0) + continue; + + size += sprintf(buf + size, "%2d %14s%s\n", + i, profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); + } + + return size; +} + +static int vangogh_set_power_profile_mode(struct smu_context *smu, long *input, uint32_t size) +{ + int workload_type, ret; + uint32_t profile_mode = input[size]; + + if (profile_mode > PP_SMC_POWER_PROFILE_CUSTOM) { + dev_err(smu->adev->dev, "Invalid power profile mode %d\n", profile_mode); + return -EINVAL; + } + + if (profile_mode == PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT || + profile_mode == PP_SMC_POWER_PROFILE_POWERSAVING) + return 0; + + /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ + workload_type = smu_cmn_to_asic_specific_index(smu, + CMN2ASIC_MAPPING_WORKLOAD, + profile_mode); + if (workload_type < 0) { + dev_err_once(smu->adev->dev, "Unsupported power profile mode %d on VANGOGH\n", + profile_mode); + return -EINVAL; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_ActiveProcessNotify, + 1 << workload_type, + NULL); + if (ret) { + dev_err_once(smu->adev->dev, "Fail to set workload type %d\n", + workload_type); + return ret; + } + + smu->power_profile_mode = profile_mode; + + return 0; +} + +static int vangogh_set_soft_freq_limited_range(struct smu_context *smu, + enum smu_clk_type clk_type, + uint32_t min, + uint32_t max) +{ + int ret = 0; + + if (!vangogh_clk_dpm_is_enabled(smu, clk_type)) + return 0; + + switch (clk_type) { + case SMU_GFXCLK: + case SMU_SCLK: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinGfxClk, + min, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxGfxClk, + max, NULL); + if (ret) + return ret; + break; + case SMU_FCLK: + case SMU_MCLK: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinFclkByFreq, + min, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxFclkByFreq, + max, NULL); + if (ret) + return ret; + break; + case SMU_SOCCLK: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinSocclkByFreq, + min, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxSocclkByFreq, + max, NULL); + if (ret) + return ret; + break; + case SMU_VCLK: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinVcn, + min << 16, NULL); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxVcn, + max << 16, NULL); + if (ret) + return ret; + break; + case SMU_DCLK: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinVcn, + min, NULL); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxVcn, + max, NULL); + if (ret) + return ret; + break; + default: + return -EINVAL; + } + + return ret; +} + +static int vangogh_force_clk_levels(struct smu_context *smu, + enum smu_clk_type clk_type, uint32_t mask) +{ + uint32_t soft_min_level = 0, soft_max_level = 0; + uint32_t min_freq = 0, max_freq = 0; + int ret = 0 ; + + soft_min_level = mask ? (ffs(mask) - 1) : 0; + soft_max_level = mask ? (fls(mask) - 1) : 0; + + switch (clk_type) { + case SMU_SOCCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, + soft_min_level, &min_freq); + if (ret) + return ret; + ret = vangogh_get_dpm_clk_limited(smu, clk_type, + soft_max_level, &max_freq); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxSocclkByFreq, + max_freq, NULL); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinSocclkByFreq, + min_freq, NULL); + if (ret) + return ret; + break; + case SMU_MCLK: + case SMU_FCLK: + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_min_level, &min_freq); + if (ret) + return ret; + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_max_level, &max_freq); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxFclkByFreq, + max_freq, NULL); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinFclkByFreq, + min_freq, NULL); + if (ret) + return ret; + break; + case SMU_VCLK: + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_min_level, &min_freq); + if (ret) + return ret; + + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_max_level, &max_freq); + if (ret) + return ret; + + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinVcn, + min_freq << 16, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxVcn, + max_freq << 16, NULL); + if (ret) + return ret; + + break; + case SMU_DCLK: + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_min_level, &min_freq); + if (ret) + return ret; + + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_max_level, &max_freq); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinVcn, + min_freq, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxVcn, + max_freq, NULL); + if (ret) + return ret; + + break; + default: + break; + } + + return ret; +} + +static int vangogh_force_dpm_limit_value(struct smu_context *smu, bool highest) +{ + int ret = 0, i = 0; + uint32_t min_freq, max_freq, force_freq; + enum smu_clk_type clk_type; + + enum smu_clk_type clks[] = { + SMU_SOCCLK, + SMU_VCLK, + SMU_DCLK, + SMU_MCLK, + SMU_FCLK, + }; + + for (i = 0; i < ARRAY_SIZE(clks); i++) { + clk_type = clks[i]; + ret = vangogh_get_dpm_ultimate_freq(smu, clk_type, &min_freq, &max_freq); + if (ret) + return ret; + + force_freq = highest ? max_freq : min_freq; + ret = vangogh_set_soft_freq_limited_range(smu, clk_type, force_freq, force_freq); + if (ret) + return ret; + } + + return ret; +} + +static int vangogh_unforce_dpm_levels(struct smu_context *smu) +{ + int ret = 0, i = 0; + uint32_t min_freq, max_freq; + enum smu_clk_type clk_type; + + struct clk_feature_map { + enum smu_clk_type clk_type; + uint32_t feature; + } clk_feature_map[] = { + {SMU_MCLK, SMU_FEATURE_DPM_FCLK_BIT}, + {SMU_FCLK, SMU_FEATURE_DPM_FCLK_BIT}, + {SMU_SOCCLK, SMU_FEATURE_DPM_SOCCLK_BIT}, + {SMU_VCLK, SMU_FEATURE_VCN_DPM_BIT}, + {SMU_DCLK, SMU_FEATURE_VCN_DPM_BIT}, + }; + + for (i = 0; i < ARRAY_SIZE(clk_feature_map); i++) { + + if (!smu_cmn_feature_is_enabled(smu, clk_feature_map[i].feature)) + continue; + + clk_type = clk_feature_map[i].clk_type; + + ret = vangogh_get_dpm_ultimate_freq(smu, clk_type, &min_freq, &max_freq); + + if (ret) + return ret; + + ret = vangogh_set_soft_freq_limited_range(smu, clk_type, min_freq, max_freq); + + if (ret) + return ret; + } + + return ret; +} + +static int vangogh_set_peak_clock_by_device(struct smu_context *smu) +{ + int ret = 0; + uint32_t socclk_freq = 0, fclk_freq = 0; + uint32_t vclk_freq = 0, dclk_freq = 0; + + ret = vangogh_get_dpm_ultimate_freq(smu, SMU_FCLK, NULL, &fclk_freq); + if (ret) + return ret; + + ret = vangogh_set_soft_freq_limited_range(smu, SMU_FCLK, fclk_freq, fclk_freq); + if (ret) + return ret; + + ret = vangogh_get_dpm_ultimate_freq(smu, SMU_SOCCLK, NULL, &socclk_freq); + if (ret) + return ret; + + ret = vangogh_set_soft_freq_limited_range(smu, SMU_SOCCLK, socclk_freq, socclk_freq); + if (ret) + return ret; + + ret = vangogh_get_dpm_ultimate_freq(smu, SMU_VCLK, NULL, &vclk_freq); + if (ret) + return ret; + + ret = vangogh_set_soft_freq_limited_range(smu, SMU_VCLK, vclk_freq, vclk_freq); + if (ret) + return ret; + + ret = vangogh_get_dpm_ultimate_freq(smu, SMU_DCLK, NULL, &dclk_freq); + if (ret) + return ret; + + ret = vangogh_set_soft_freq_limited_range(smu, SMU_DCLK, dclk_freq, dclk_freq); + if (ret) + return ret; + + return ret; +} + +static int vangogh_set_performance_level(struct smu_context *smu, + enum amd_dpm_forced_level level) +{ + int ret = 0; + uint32_t soc_mask, mclk_mask, fclk_mask; + uint32_t vclk_mask = 0, dclk_mask = 0; + + switch (level) { + case AMD_DPM_FORCED_LEVEL_HIGH: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = vangogh_force_dpm_limit_value(smu, true); + break; + case AMD_DPM_FORCED_LEVEL_LOW: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = vangogh_force_dpm_limit_value(smu, false); + break; + case AMD_DPM_FORCED_LEVEL_AUTO: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = vangogh_unforce_dpm_levels(smu); + break; + case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinGfxClk, + VANGOGH_UMD_PSTATE_STANDARD_GFXCLK, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxGfxClk, + VANGOGH_UMD_PSTATE_STANDARD_GFXCLK, NULL); + if (ret) + return ret; + + ret = vangogh_get_profiling_clk_mask(smu, level, + &vclk_mask, + &dclk_mask, + &mclk_mask, + &fclk_mask, + &soc_mask); + if (ret) + return ret; + + vangogh_force_clk_levels(smu, SMU_MCLK, 1 << mclk_mask); + vangogh_force_clk_levels(smu, SMU_FCLK, 1 << fclk_mask); + vangogh_force_clk_levels(smu, SMU_SOCCLK, 1 << soc_mask); + vangogh_force_clk_levels(smu, SMU_VCLK, 1 << vclk_mask); + vangogh_force_clk_levels(smu, SMU_DCLK, 1 << dclk_mask); + + break; + case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinVcn, + VANGOGH_UMD_PSTATE_PEAK_DCLK, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxVcn, + VANGOGH_UMD_PSTATE_PEAK_DCLK, NULL); + if (ret) + return ret; + break; + case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = vangogh_get_profiling_clk_mask(smu, level, + NULL, + NULL, + &mclk_mask, + &fclk_mask, + NULL); + if (ret) + return ret; + + vangogh_force_clk_levels(smu, SMU_MCLK, 1 << mclk_mask); + vangogh_force_clk_levels(smu, SMU_FCLK, 1 << fclk_mask); + break; + case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinGfxClk, + VANGOGH_UMD_PSTATE_PEAK_GFXCLK, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxGfxClk, + VANGOGH_UMD_PSTATE_PEAK_GFXCLK, NULL); + if (ret) + return ret; + + ret = vangogh_set_peak_clock_by_device(smu); + break; + case AMD_DPM_FORCED_LEVEL_MANUAL: + case AMD_DPM_FORCED_LEVEL_PROFILE_EXIT: + default: + break; + } + return ret; +} + static int vangogh_read_sensor(struct smu_context *smu, enum amd_pp_sensors sensor, void *data, uint32_t *size) @@ -493,6 +1322,12 @@ static int vangogh_read_sensor(struct smu_context *smu, (uint32_t *)data); *size = 4; break; + case AMDGPU_PP_SENSOR_CPU_CLK: + ret = vangogh_get_smu_metrics_data(smu, + METRICS_AVERAGE_CPUCLK, + (uint32_t *)data); + *size = smu->cpu_core_num * sizeof(uint16_t); + break; default: ret = -EOPNOTSUPP; break; @@ -514,7 +1349,7 @@ static int vangogh_set_watermarks_table(struct smu_context *smu, if (clock_ranges) { if (clock_ranges->num_reader_wm_sets > NUM_WM_RANGES || - clock_ranges->num_writer_wm_sets > NUM_WM_RANGES) + clock_ranges->num_writer_wm_sets > NUM_WM_RANGES) return -EINVAL; for (i = 0; i < clock_ranges->num_reader_wm_sets; i++) { @@ -575,7 +1410,7 @@ static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu, if (ret) return ret; - smu_v11_0_init_gpu_metrics_v2_0(gpu_metrics); + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 0); gpu_metrics->temperature_gfx = metrics.GfxTemperature; gpu_metrics->temperature_soc = metrics.SocTemperature; @@ -611,22 +1446,55 @@ static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu, gpu_metrics->throttle_status = metrics.ThrottlerStatus; + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + *table = (void *)gpu_metrics; return sizeof(struct gpu_metrics_v2_0); } static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TABLE_COMMAND type, - long input[], uint32_t size) + long input[], uint32_t size) { int ret = 0; + int i; + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); - if (!smu->od_enabled) { - dev_warn(smu->adev->dev, "Fine grain is not enabled!\n"); + if (!(smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL)) { + dev_warn(smu->adev->dev, + "pp_od_clk_voltage is not accessible if power_dpm_force_perfomance_level is not in manual mode!\n"); return -EINVAL; } switch (type) { + case PP_OD_EDIT_CCLK_VDDC_TABLE: + if (size != 3) { + dev_err(smu->adev->dev, "Input parameter number not correct (should be 4 for processor)\n"); + return -EINVAL; + } + if (input[0] >= smu->cpu_core_num) { + dev_err(smu->adev->dev, "core index is overflow, should be less than %d\n", + smu->cpu_core_num); + } + smu->cpu_core_id_select = input[0]; + if (input[1] == 0) { + if (input[2] < smu->cpu_default_soft_min_freq) { + dev_warn(smu->adev->dev, "Fine grain setting minimum cclk (%ld) MHz is less than the minimum allowed (%d) MHz\n", + input[2], smu->cpu_default_soft_min_freq); + return -EINVAL; + } + smu->cpu_actual_soft_min_freq = input[2]; + } else if (input[1] == 1) { + if (input[2] > smu->cpu_default_soft_max_freq) { + dev_warn(smu->adev->dev, "Fine grain setting maximum cclk (%ld) MHz is greater than the maximum allowed (%d) MHz\n", + input[2], smu->cpu_default_soft_max_freq); + return -EINVAL; + } + smu->cpu_actual_soft_max_freq = input[2]; + } else { + return -EINVAL; + } + break; case PP_OD_EDIT_SCLK_VDDC_TABLE: if (size != 2) { dev_err(smu->adev->dev, "Input parameter number not correct\n"); @@ -635,14 +1503,16 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB if (input[0] == 0) { if (input[1] < smu->gfx_default_hard_min_freq) { - dev_warn(smu->adev->dev, "Fine grain setting minimum sclk (%ld) MHz is less than the minimum allowed (%d) MHz\n", + dev_warn(smu->adev->dev, + "Fine grain setting minimum sclk (%ld) MHz is less than the minimum allowed (%d) MHz\n", input[1], smu->gfx_default_hard_min_freq); return -EINVAL; } smu->gfx_actual_hard_min_freq = input[1]; } else if (input[0] == 1) { if (input[1] > smu->gfx_default_soft_max_freq) { - dev_warn(smu->adev->dev, "Fine grain setting maximum sclk (%ld) MHz is greater than the maximum allowed (%d) MHz\n", + dev_warn(smu->adev->dev, + "Fine grain setting maximum sclk (%ld) MHz is greater than the maximum allowed (%d) MHz\n", input[1], smu->gfx_default_soft_max_freq); return -EINVAL; } @@ -658,6 +1528,8 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB } else { smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinGfxClk, smu->gfx_actual_hard_min_freq, NULL); @@ -672,6 +1544,29 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB dev_err(smu->adev->dev, "Restore the default soft max sclk failed!"); return ret; } + + if (smu->adev->pm.fw_version < 0x43f1b00) { + dev_warn(smu->adev->dev, "CPUSoftMax/CPUSoftMin are not supported, please update SBIOS!\n"); + break; + } + + for (i = 0; i < smu->cpu_core_num; i++) { + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMinCclk, + (i << 20) | smu->cpu_actual_soft_min_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set hard min cclk failed!"); + return ret; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxCclk, + (i << 20) | smu->cpu_actual_soft_max_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set soft max cclk failed!"); + return ret; + } + } } break; case PP_OD_COMMIT_DPM_TABLE: @@ -680,8 +1575,10 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB return -EINVAL; } else { if (smu->gfx_actual_hard_min_freq > smu->gfx_actual_soft_max_freq) { - dev_err(smu->adev->dev, "The setting minimun sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", - smu->gfx_actual_hard_min_freq, smu->gfx_actual_soft_max_freq); + dev_err(smu->adev->dev, + "The setting minimun sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", + smu->gfx_actual_hard_min_freq, + smu->gfx_actual_soft_max_freq); return -EINVAL; } @@ -698,6 +1595,29 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB dev_err(smu->adev->dev, "Set soft max sclk failed!"); return ret; } + + if (smu->adev->pm.fw_version < 0x43f1b00) { + dev_warn(smu->adev->dev, "CPUSoftMax/CPUSoftMin are not supported, please update SBIOS!\n"); + break; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMinCclk, + ((smu->cpu_core_id_select << 20) + | smu->cpu_actual_soft_min_freq), + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set hard min cclk failed!"); + return ret; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxCclk, + ((smu->cpu_core_id_select << 20) + | smu->cpu_actual_soft_max_freq), + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set soft max cclk failed!"); + return ret; + } } break; default: @@ -723,18 +1643,245 @@ static int vangogh_set_fine_grain_gfx_freq_parameters(struct smu_context *smu) smu->gfx_actual_hard_min_freq = 0; smu->gfx_actual_soft_max_freq = 0; + smu->cpu_default_soft_min_freq = 1400; + smu->cpu_default_soft_max_freq = 3500; + smu->cpu_actual_soft_min_freq = 0; + smu->cpu_actual_soft_max_freq = 0; + return 0; } +static int vangogh_get_dpm_clock_table(struct smu_context *smu, struct dpm_clocks *clock_table) +{ + DpmClocks_t *table = smu->smu_table.clocks_table; + int i; + + if (!clock_table || !table) + return -EINVAL; + + for (i = 0; i < NUM_SOCCLK_DPM_LEVELS; i++) { + clock_table->SocClocks[i].Freq = table->SocClocks[i]; + clock_table->SocClocks[i].Vol = table->SocVoltage[i]; + } + + for (i = 0; i < NUM_FCLK_DPM_LEVELS; i++) { + clock_table->FClocks[i].Freq = table->DfPstateTable[i].fclk; + clock_table->FClocks[i].Vol = table->DfPstateTable[i].voltage; + } + + for (i = 0; i < NUM_FCLK_DPM_LEVELS; i++) { + clock_table->MemClocks[i].Freq = table->DfPstateTable[i].memclk; + clock_table->MemClocks[i].Vol = table->DfPstateTable[i].voltage; + } + + return 0; +} + + static int vangogh_system_features_control(struct smu_context *smu, bool en) { struct amdgpu_device *adev = smu->adev; + struct smu_feature *feature = &smu->smu_feature; + uint32_t feature_mask[2]; + int ret = 0; if (adev->pm.fw_version >= 0x43f1700) - return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_RlcPowerNotify, - en ? RLC_STATUS_NORMAL : RLC_STATUS_OFF, NULL); - else + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_RlcPowerNotify, + en ? RLC_STATUS_NORMAL : RLC_STATUS_OFF, NULL); + + bitmap_zero(feature->enabled, feature->feature_num); + bitmap_zero(feature->supported, feature->feature_num); + + if (!en) + return ret; + + ret = smu_cmn_get_enabled_32_bits_mask(smu, feature_mask, 2); + if (ret) + return ret; + + bitmap_copy(feature->enabled, (unsigned long *)&feature_mask, + feature->feature_num); + bitmap_copy(feature->supported, (unsigned long *)&feature_mask, + feature->feature_num); + + return 0; +} + +static int vangogh_post_smu_init(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t tmp; + int ret = 0; + uint8_t aon_bits = 0; + /* Two CUs in one WGP */ + uint32_t req_active_wgps = adev->gfx.cu_info.number/2; + uint32_t total_cu = adev->gfx.config.max_cu_per_sh * + adev->gfx.config.max_sh_per_se * adev->gfx.config.max_shader_engines; + + /* allow message will be sent after enable message on Vangogh*/ + if (smu_cmn_feature_is_supported(smu, SMU_FEATURE_DPM_GFXCLK_BIT) && + (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) { + ret = smu_cmn_send_smc_msg(smu, SMU_MSG_EnableGfxOff, NULL); + if (ret) { + dev_err(adev->dev, "Failed to Enable GfxOff!\n"); + return ret; + } + } else { + adev->pm.pp_feature &= ~PP_GFXOFF_MASK; + dev_info(adev->dev, "If GFX DPM or power gate disabled, disable GFXOFF\n"); + } + + /* if all CUs are active, no need to power off any WGPs */ + if (total_cu == adev->gfx.cu_info.number) return 0; + + /* + * Calculate the total bits number of always on WGPs for all SA/SEs in + * RLC_PG_ALWAYS_ON_WGP_MASK. + */ + tmp = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_ALWAYS_ON_WGP_MASK)); + tmp &= RLC_PG_ALWAYS_ON_WGP_MASK__AON_WGP_MASK_MASK; + + aon_bits = hweight32(tmp) * adev->gfx.config.max_sh_per_se * adev->gfx.config.max_shader_engines; + + /* Do not request any WGPs less than set in the AON_WGP_MASK */ + if (aon_bits > req_active_wgps) { + dev_info(adev->dev, "Number of always on WGPs greater than active WGPs: WGP power save not requested.\n"); + return 0; + } else { + return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_RequestActiveWgp, req_active_wgps, NULL); + } +} + +static int vangogh_mode_reset(struct smu_context *smu, int type) +{ + int ret = 0, index = 0; + + index = smu_cmn_to_asic_specific_index(smu, CMN2ASIC_MAPPING_MSG, + SMU_MSG_GfxDeviceDriverReset); + if (index < 0) + return index == -EACCES ? 0 : index; + + mutex_lock(&smu->message_lock); + + ret = smu_cmn_send_msg_without_waiting(smu, (uint16_t)index, type); + + mutex_unlock(&smu->message_lock); + + mdelay(10); + + return ret; +} + +static int vangogh_mode2_reset(struct smu_context *smu) +{ + return vangogh_mode_reset(smu, SMU_RESET_MODE_2); +} + +static int vangogh_get_power_limit(struct smu_context *smu) +{ + struct smu_11_5_power_context *power_context = + smu->smu_power.power_context; + uint32_t ppt_limit; + int ret = 0; + + if (smu->adev->pm.fw_version < 0x43f1e00) + return ret; + + ret = smu_cmn_send_smc_msg(smu, SMU_MSG_GetSlowPPTLimit, &ppt_limit); + if (ret) { + dev_err(smu->adev->dev, "Get slow PPT limit failed!\n"); + return ret; + } + /* convert from milliwatt to watt */ + smu->current_power_limit = ppt_limit / 1000; + smu->max_power_limit = 29; + + ret = smu_cmn_send_smc_msg(smu, SMU_MSG_GetFastPPTLimit, &ppt_limit); + if (ret) { + dev_err(smu->adev->dev, "Get fast PPT limit failed!\n"); + return ret; + } + /* convert from milliwatt to watt */ + power_context->current_fast_ppt_limit = ppt_limit / 1000; + power_context->max_fast_ppt_limit = 30; + + return ret; +} + +static int vangogh_get_ppt_limit(struct smu_context *smu, + uint32_t *ppt_limit, + enum smu_ppt_limit_type type, + enum smu_ppt_limit_level level) +{ + struct smu_11_5_power_context *power_context = + smu->smu_power.power_context; + + if (!power_context) + return -EOPNOTSUPP; + + if (type == SMU_FAST_PPT_LIMIT) { + switch (level) { + case SMU_PPT_LIMIT_MAX: + *ppt_limit = power_context->max_fast_ppt_limit; + break; + case SMU_PPT_LIMIT_CURRENT: + *ppt_limit = power_context->current_fast_ppt_limit; + break; + default: + break; + } + } + + return 0; +} + +static int vangogh_set_power_limit(struct smu_context *smu, uint32_t ppt_limit) +{ + struct smu_11_5_power_context *power_context = + smu->smu_power.power_context; + uint32_t limit_type = ppt_limit >> 24; + int ret = 0; + + if (!smu_cmn_feature_is_enabled(smu, SMU_FEATURE_PPT_BIT)) { + dev_err(smu->adev->dev, "Setting new power limit is not supported!\n"); + return -EOPNOTSUPP; + } + + switch (limit_type) { + case SMU_DEFAULT_PPT_LIMIT: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSlowPPTLimit, + ppt_limit * 1000, /* convert from watt to milliwatt */ + NULL); + if (ret) + return ret; + + smu->current_power_limit = ppt_limit; + break; + case SMU_FAST_PPT_LIMIT: + ppt_limit &= ~(SMU_FAST_PPT_LIMIT << 24); + if (ppt_limit > power_context->max_fast_ppt_limit) { + dev_err(smu->adev->dev, + "New power limit (%d) is over the max allowed %d\n", + ppt_limit, power_context->max_fast_ppt_limit); + return ret; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetFastPPTLimit, + ppt_limit * 1000, /* convert from watt to milliwatt */ + NULL); + if (ret) + return ret; + + power_context->current_fast_ppt_limit = ppt_limit; + break; + default: + return -EINVAL; + } + + return ret; } static const struct pptable_funcs vangogh_ppt_funcs = { @@ -746,7 +1893,6 @@ static const struct pptable_funcs vangogh_ppt_funcs = { .init_power = smu_v11_0_init_power, .fini_power = smu_v11_0_fini_power, .register_irq_handler = smu_v11_0_register_irq_handler, - .get_allowed_feature_mask = vangogh_get_allowed_feature_mask, .notify_memory_pool_location = smu_v11_0_notify_memory_pool_location, .send_smc_msg_with_param = smu_cmn_send_smc_msg_with_param, .send_smc_msg = smu_cmn_send_smc_msg, @@ -765,6 +1911,18 @@ static const struct pptable_funcs vangogh_ppt_funcs = { .set_default_dpm_table = vangogh_set_default_dpm_tables, .set_fine_grain_gfx_freq_parameters = vangogh_set_fine_grain_gfx_freq_parameters, .system_features_control = vangogh_system_features_control, + .feature_is_enabled = smu_cmn_feature_is_enabled, + .set_power_profile_mode = vangogh_set_power_profile_mode, + .get_power_profile_mode = vangogh_get_power_profile_mode, + .get_dpm_clock_table = vangogh_get_dpm_clock_table, + .force_clk_levels = vangogh_force_clk_levels, + .set_performance_level = vangogh_set_performance_level, + .post_init = vangogh_post_smu_init, + .mode2_reset = vangogh_mode2_reset, + .gfx_off_control = smu_v11_0_gfx_off_control, + .get_ppt_limit = vangogh_get_ppt_limit, + .get_power_limit = vangogh_get_power_limit, + .set_power_limit = vangogh_set_power_limit, }; void vangogh_set_ppt_funcs(struct smu_context *smu) @@ -773,5 +1931,6 @@ void vangogh_set_ppt_funcs(struct smu_context *smu) smu->message_map = vangogh_message_map; smu->feature_map = vangogh_feature_mask_map; smu->table_map = vangogh_table_map; + smu->workload_map = vangogh_workload_map; smu->is_apu = true; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.h index eab455493076..c56d4583dc72 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.h @@ -28,9 +28,29 @@ extern void vangogh_set_ppt_funcs(struct smu_context *smu); /* UMD PState Vangogh Msg Parameters in MHz */ -#define VANGOGH_UMD_PSTATE_GFXCLK 700 -#define VANGOGH_UMD_PSTATE_SOCCLK 678 -#define VANGOGH_UMD_PSTATE_FCLK 800 +#define VANGOGH_UMD_PSTATE_STANDARD_GFXCLK 1100 +#define VANGOGH_UMD_PSTATE_STANDARD_SOCCLK 600 +#define VANGOGH_UMD_PSTATE_STANDARD_FCLK 800 +#define VANGOGH_UMD_PSTATE_STANDARD_VCLK 705 +#define VANGOGH_UMD_PSTATE_STANDARD_DCLK 600 + +#define VANGOGH_UMD_PSTATE_PEAK_GFXCLK 1300 +#define VANGOGH_UMD_PSTATE_PEAK_SOCCLK 600 +#define VANGOGH_UMD_PSTATE_PEAK_FCLK 800 +#define VANGOGH_UMD_PSTATE_PEAK_VCLK 705 +#define VANGOGH_UMD_PSTATE_PEAK_DCLK 600 + +#define VANGOGH_UMD_PSTATE_MIN_SCLK_GFXCLK 400 +#define VANGOGH_UMD_PSTATE_MIN_SCLK_SOCCLK 1000 +#define VANGOGH_UMD_PSTATE_MIN_SCLK_FCLK 800 +#define VANGOGH_UMD_PSTATE_MIN_SCLK_VCLK 1000 +#define VANGOGH_UMD_PSTATE_MIN_SCLK_DCLK 800 + +#define VANGOGH_UMD_PSTATE_MIN_MCLK_GFXCLK 1100 +#define VANGOGH_UMD_PSTATE_MIN_MCLK_SOCCLK 1000 +#define VANGOGH_UMD_PSTATE_MIN_MCLK_FCLK 400 +#define VANGOGH_UMD_PSTATE_MIN_MCLK_VCLK 1000 +#define VANGOGH_UMD_PSTATE_MIN_MCLK_DCLK 800 /* RLC Power Status */ #define RLC_STATUS_OFF 0 diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c index 9a9697038016..5faa509f0dba 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c @@ -56,8 +56,6 @@ static struct cmn2asic_msg_mapping renoir_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(PowerUpSdma, PPSMC_MSG_PowerUpSdma, 1), MSG_MAP(SetHardMinIspclkByFreq, PPSMC_MSG_SetHardMinIspclkByFreq, 1), MSG_MAP(SetHardMinVcn, PPSMC_MSG_SetHardMinVcn, 1), - MSG_MAP(Spare1, PPSMC_MSG_spare1, 1), - MSG_MAP(Spare2, PPSMC_MSG_spare2, 1), MSG_MAP(SetAllowFclkSwitch, PPSMC_MSG_SetAllowFclkSwitch, 1), MSG_MAP(SetMinVideoGfxclkFreq, PPSMC_MSG_SetMinVideoGfxclkFreq, 1), MSG_MAP(ActiveProcessNotify, PPSMC_MSG_ActiveProcessNotify, 1), @@ -344,12 +342,142 @@ failed: return ret; } +static int renoir_od_edit_dpm_table(struct smu_context *smu, + enum PP_OD_DPM_TABLE_COMMAND type, + long input[], uint32_t size) +{ + int ret = 0; + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); + + if (!(smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL)) { + dev_warn(smu->adev->dev, + "pp_od_clk_voltage is not accessible if power_dpm_force_perfomance_level is not in manual mode!\n"); + return -EINVAL; + } + + switch (type) { + case PP_OD_EDIT_SCLK_VDDC_TABLE: + if (size != 2) { + dev_err(smu->adev->dev, "Input parameter number not correct\n"); + return -EINVAL; + } + + if (input[0] == 0) { + if (input[1] < smu->gfx_default_hard_min_freq) { + dev_warn(smu->adev->dev, + "Fine grain setting minimum sclk (%ld) MHz is less than the minimum allowed (%d) MHz\n", + input[1], smu->gfx_default_hard_min_freq); + return -EINVAL; + } + smu->gfx_actual_hard_min_freq = input[1]; + } else if (input[0] == 1) { + if (input[1] > smu->gfx_default_soft_max_freq) { + dev_warn(smu->adev->dev, + "Fine grain setting maximum sclk (%ld) MHz is greater than the maximum allowed (%d) MHz\n", + input[1], smu->gfx_default_soft_max_freq); + return -EINVAL; + } + smu->gfx_actual_soft_max_freq = input[1]; + } else { + return -EINVAL; + } + break; + case PP_OD_RESTORE_DEFAULT_TABLE: + if (size != 0) { + dev_err(smu->adev->dev, "Input parameter number not correct\n"); + return -EINVAL; + } + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinGfxClk, + smu->gfx_actual_hard_min_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Restore the default hard min sclk failed!"); + return ret; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxGfxClk, + smu->gfx_actual_soft_max_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Restore the default soft max sclk failed!"); + return ret; + } + break; + case PP_OD_COMMIT_DPM_TABLE: + if (size != 0) { + dev_err(smu->adev->dev, "Input parameter number not correct\n"); + return -EINVAL; + } else { + if (smu->gfx_actual_hard_min_freq > smu->gfx_actual_soft_max_freq) { + dev_err(smu->adev->dev, + "The setting minimun sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", + smu->gfx_actual_hard_min_freq, + smu->gfx_actual_soft_max_freq); + return -EINVAL; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinGfxClk, + smu->gfx_actual_hard_min_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set hard min sclk failed!"); + return ret; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxGfxClk, + smu->gfx_actual_soft_max_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set soft max sclk failed!"); + return ret; + } + } + break; + default: + return -ENOSYS; + } + + return ret; +} + +static int renoir_set_fine_grain_gfx_freq_parameters(struct smu_context *smu) +{ + uint32_t min = 0, max = 0; + uint32_t ret = 0; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_GetMinGfxclkFrequency, + 0, &min); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_GetMaxGfxclkFrequency, + 0, &max); + if (ret) + return ret; + + smu->gfx_default_hard_min_freq = min; + smu->gfx_default_soft_max_freq = max; + smu->gfx_actual_hard_min_freq = 0; + smu->gfx_actual_soft_max_freq = 0; + + return 0; +} + static int renoir_print_clk_levels(struct smu_context *smu, enum smu_clk_type clk_type, char *buf) { int i, size = 0, ret = 0; uint32_t cur_value = 0, value = 0, count = 0, min = 0, max = 0; SmuMetrics_t metrics; + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); bool cur_value_match_level = false; memset(&metrics, 0, sizeof(metrics)); @@ -359,6 +487,30 @@ static int renoir_print_clk_levels(struct smu_context *smu, return ret; switch (clk_type) { + case SMU_OD_RANGE: + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_GetMinGfxclkFrequency, + 0, &min); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_GetMaxGfxclkFrequency, + 0, &max); + if (ret) + return ret; + size += sprintf(buf + size, "OD_RANGE\nSCLK: %10uMhz %10uMhz\n", min, max); + } + break; + case SMU_OD_SCLK: + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { + min = (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq; + max = (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq; + size += sprintf(buf + size, "OD_SCLK\n"); + size += sprintf(buf + size, "0:%10uMhz\n", min); + size += sprintf(buf + size, "1:%10uMhz\n", max); + } + break; case SMU_GFXCLK: case SMU_SCLK: /* retirve table returned paramters unit is MHz */ @@ -399,23 +551,35 @@ static int renoir_print_clk_levels(struct smu_context *smu, cur_value = metrics.ClockFrequency[CLOCK_FCLK]; break; default: - return -EINVAL; + break; } - for (i = 0; i < count; i++) { - ret = renoir_get_dpm_clk_limited(smu, clk_type, i, &value); - if (ret) - return ret; - if (!value) - continue; - size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, - cur_value == value ? "*" : ""); - if (cur_value == value) - cur_value_match_level = true; - } + switch (clk_type) { + case SMU_GFXCLK: + case SMU_SCLK: + case SMU_SOCCLK: + case SMU_MCLK: + case SMU_DCEFCLK: + case SMU_FCLK: + for (i = 0; i < count; i++) { + ret = renoir_get_dpm_clk_limited(smu, clk_type, i, &value); + if (ret) + return ret; + if (!value) + continue; + size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, + cur_value == value ? "*" : ""); + if (cur_value == value) + cur_value_match_level = true; + } - if (!cur_value_match_level) - size += sprintf(buf + size, " %uMhz *\n", cur_value); + if (!cur_value_match_level) + size += sprintf(buf + size, " %uMhz *\n", cur_value); + + break; + default: + break; + } return size; } @@ -667,6 +831,10 @@ static int renoir_set_power_profile_mode(struct smu_context *smu, long *input, u return -EINVAL; } + if (profile_mode == PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT || + profile_mode == PP_SMC_POWER_PROFILE_POWERSAVING) + return 0; + /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ workload_type = smu_cmn_to_asic_specific_index(smu, CMN2ASIC_MAPPING_WORKLOAD, @@ -725,15 +893,27 @@ static int renoir_set_performance_level(struct smu_context *smu, switch (level) { case AMD_DPM_FORCED_LEVEL_HIGH: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = renoir_force_dpm_limit_value(smu, true); break; case AMD_DPM_FORCED_LEVEL_LOW: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = renoir_force_dpm_limit_value(smu, false); break; case AMD_DPM_FORCED_LEVEL_AUTO: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = renoir_unforce_dpm_levels(smu); break; case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinGfxClk, RENOIR_UMD_PSTATE_GFXCLK, @@ -786,6 +966,9 @@ static int renoir_set_performance_level(struct smu_context *smu, break; case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK: case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = renoir_get_profiling_clk_mask(smu, level, &sclk_mask, &mclk_mask, @@ -797,6 +980,9 @@ static int renoir_set_performance_level(struct smu_context *smu, renoir_force_clk_levels(smu, SMU_SOCCLK, 1 << soc_mask); break; case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = renoir_set_peak_clock_by_device(smu); break; case AMD_DPM_FORCED_LEVEL_MANUAL: @@ -944,7 +1130,7 @@ static int renoir_get_smu_metrics_data(struct smu_context *smu, *value = metrics->AverageUvdActivity / 100; break; case METRICS_AVERAGE_SOCKETPOWER: - *value = metrics->CurrentSocketPower << 8; + *value = (metrics->CurrentSocketPower << 8) / 1000; break; case METRICS_TEMPERATURE_EDGE: *value = (metrics->GfxTemperature / 100) * @@ -1072,7 +1258,7 @@ static ssize_t renoir_get_gpu_metrics(struct smu_context *smu, if (ret) return ret; - smu_v12_0_init_gpu_metrics_v2_0(gpu_metrics); + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 0); gpu_metrics->temperature_gfx = metrics.GfxTemperature; gpu_metrics->temperature_soc = metrics.SocTemperature; @@ -1113,6 +1299,8 @@ static ssize_t renoir_get_gpu_metrics(struct smu_context *smu, gpu_metrics->fan_pwm = metrics.FanPwm; + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + *table = (void *)gpu_metrics; return sizeof(struct gpu_metrics_v2_0); @@ -1160,6 +1348,8 @@ static const struct pptable_funcs renoir_ppt_funcs = { .set_pp_feature_mask = smu_cmn_set_pp_feature_mask, .get_gpu_metrics = renoir_get_gpu_metrics, .gfx_state_change_set = renoir_gfx_state_change_set, + .set_fine_grain_gfx_freq_parameters = renoir_set_fine_grain_gfx_freq_parameters, + .od_edit_dpm_table = renoir_od_edit_dpm_table, }; void renoir_set_ppt_funcs(struct smu_context *smu) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c index 06abf2a7ce9e..6cc4855c8a37 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c @@ -278,15 +278,3 @@ int smu_v12_0_set_driver_table_location(struct smu_context *smu) return ret; } - -void smu_v12_0_init_gpu_metrics_v2_0(struct gpu_metrics_v2_0 *gpu_metrics) -{ - memset(gpu_metrics, 0xFF, sizeof(struct gpu_metrics_v2_0)); - - gpu_metrics->common_header.structure_size = - sizeof(struct gpu_metrics_v2_0); - gpu_metrics->common_header.format_revision = 2; - gpu_metrics->common_header.content_revision = 0; - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); -} diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index f8260769061c..bcedd4d92e35 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -68,14 +68,6 @@ static const char *smu_get_message_name(struct smu_context *smu, return __smu_message_names[type]; } -static void smu_cmn_send_msg_without_waiting(struct smu_context *smu, - uint16_t msg) -{ - struct amdgpu_device *adev = smu->adev; - - WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_66, msg); -} - static void smu_cmn_read_arg(struct smu_context *smu, uint32_t *arg) { @@ -92,7 +84,7 @@ static int smu_cmn_wait_for_response(struct smu_context *smu) for (i = 0; i < timeout; i++) { cur_value = RREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_90); if ((cur_value & MP1_C2PMSG_90__CONTENT_MASK) != 0) - return cur_value == 0x1 ? 0 : -EIO; + return cur_value; udelay(1); } @@ -101,7 +93,29 @@ static int smu_cmn_wait_for_response(struct smu_context *smu) if (i == timeout) return -ETIME; - return RREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_90) == 0x1 ? 0 : -EIO; + return RREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_90); +} + +int smu_cmn_send_msg_without_waiting(struct smu_context *smu, + uint16_t msg, uint32_t param) +{ + struct amdgpu_device *adev = smu->adev; + int ret; + + ret = smu_cmn_wait_for_response(smu); + if (ret != 0x1) { + dev_err(adev->dev, "Msg issuing pre-check failed and " + "SMU may be not in the right state!\n"); + if (ret != -ETIME) + ret = -EIO; + return ret; + } + + WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_90, 0); + WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_82, param); + WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_66, msg); + + return 0; } int smu_cmn_send_smc_msg_with_param(struct smu_context *smu, @@ -122,29 +136,28 @@ int smu_cmn_send_smc_msg_with_param(struct smu_context *smu, return index == -EACCES ? 0 : index; mutex_lock(&smu->message_lock); - ret = smu_cmn_wait_for_response(smu); - if (ret) { - dev_err(adev->dev, "Msg issuing pre-check failed and " - "SMU may be not in the right state!\n"); + ret = smu_cmn_send_msg_without_waiting(smu, (uint16_t)index, param); + if (ret) goto out; - } - - WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_90, 0); - - WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_82, param); - - smu_cmn_send_msg_without_waiting(smu, (uint16_t)index); ret = smu_cmn_wait_for_response(smu); - if (ret) { - dev_err(adev->dev, "failed send message: %10s (%d) \tparam: 0x%08x response %#x\n", - smu_get_message_name(smu, msg), index, param, ret); + if (ret != 0x1) { + if (ret == -ETIME) { + dev_err(adev->dev, "message: %15s (%d) \tparam: 0x%08x is timeout (no response)\n", + smu_get_message_name(smu, msg), index, param); + } else { + dev_err(adev->dev, "failed send message: %15s (%d) \tparam: 0x%08x response %#x\n", + smu_get_message_name(smu, msg), index, param, + ret); + ret = -EIO; + } goto out; } if (read_arg) smu_cmn_read_arg(smu, read_arg); + ret = 0; /* 0 as driver return value */ out: mutex_unlock(&smu->message_lock); return ret; @@ -269,11 +282,13 @@ int smu_cmn_feature_is_enabled(struct smu_context *smu, enum smu_feature_mask mask) { struct smu_feature *feature = &smu->smu_feature; + struct amdgpu_device *adev = smu->adev; int feature_id; int ret = 0; - if (smu->is_apu) + if (smu->is_apu && adev->family < AMDGPU_FAMILY_VGH) return 1; + feature_id = smu_cmn_to_asic_specific_index(smu, CMN2ASIC_MAPPING_FEATURE, mask); @@ -731,3 +746,31 @@ int smu_cmn_get_metrics_table(struct smu_context *smu, return ret; } + +void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev) +{ + struct metrics_table_header *header = (struct metrics_table_header *)table; + uint16_t structure_size; + +#define METRICS_VERSION(a, b) ((a << 16) | b ) + + switch (METRICS_VERSION(frev, crev)) { + case METRICS_VERSION(1, 0): + structure_size = sizeof(struct gpu_metrics_v1_0); + break; + case METRICS_VERSION(2, 0): + structure_size = sizeof(struct gpu_metrics_v2_0); + break; + default: + return; + } + +#undef METRICS_VERSION + + memset(header, 0xFF, structure_size); + + header->format_revision = frev; + header->content_revision = crev; + header->structure_size = structure_size; + +} diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h index 01e825d83d8d..c69250185575 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h @@ -26,6 +26,8 @@ #include "amdgpu_smu.h" #if defined(SWSMU_CODE_LAYER_L2) || defined(SWSMU_CODE_LAYER_L3) || defined(SWSMU_CODE_LAYER_L4) +int smu_cmn_send_msg_without_waiting(struct smu_context *smu, + uint16_t msg, uint32_t param); int smu_cmn_send_smc_msg_with_param(struct smu_context *smu, enum smu_message_type msg, uint32_t param, @@ -95,5 +97,7 @@ int smu_cmn_get_metrics_table(struct smu_context *smu, void *metrics_table, bool bypass_cache); +void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev); + #endif #endif diff --git a/drivers/gpu/drm/arc/arcpgu_crtc.c b/drivers/gpu/drm/arc/arcpgu_crtc.c index 042d7b54a6de..895cdd991af6 100644 --- a/drivers/gpu/drm/arc/arcpgu_crtc.c +++ b/drivers/gpu/drm/arc/arcpgu_crtc.c @@ -162,15 +162,10 @@ static const struct drm_plane_helper_funcs arc_pgu_plane_helper_funcs = { .atomic_update = arc_pgu_plane_atomic_update, }; -static void arc_pgu_plane_destroy(struct drm_plane *plane) -{ - drm_plane_cleanup(plane); -} - static const struct drm_plane_funcs arc_pgu_plane_funcs = { .update_plane = drm_atomic_helper_update_plane, .disable_plane = drm_atomic_helper_disable_plane, - .destroy = arc_pgu_plane_destroy, + .destroy = drm_plane_cleanup, .reset = drm_atomic_helper_plane_reset, .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state, .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, @@ -213,7 +208,7 @@ int arc_pgu_setup_crtc(struct drm_device *drm) ret = drm_crtc_init_with_planes(drm, &arcpgu->crtc, primary, NULL, &arc_pgu_crtc_funcs, NULL); if (ret) { - arc_pgu_plane_destroy(primary); + drm_plane_cleanup(primary); return ret; } diff --git a/drivers/gpu/drm/arc/arcpgu_drv.c b/drivers/gpu/drm/arc/arcpgu_drv.c index f164818ec477..077d006b1fbf 100644 --- a/drivers/gpu/drm/arc/arcpgu_drv.c +++ b/drivers/gpu/drm/arc/arcpgu_drv.c @@ -145,7 +145,7 @@ static void arcpgu_debugfs_init(struct drm_minor *minor) } #endif -static struct drm_driver arcpgu_drm_driver = { +static const struct drm_driver arcpgu_drm_driver = { .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC, .name = "arcpgu", .desc = "ARC PGU Controller", diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c index 4b485eb512e2..59172acb9738 100644 --- a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c +++ b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c @@ -550,7 +550,6 @@ static void komeda_crtc_vblank_disable(struct drm_crtc *crtc) } static const struct drm_crtc_funcs komeda_crtc_funcs = { - .gamma_set = drm_atomic_helper_legacy_gamma_set, .destroy = drm_crtc_cleanup, .set_config = drm_atomic_helper_set_config, .page_flip = drm_atomic_helper_page_flip, diff --git a/drivers/gpu/drm/arm/malidp_crtc.c b/drivers/gpu/drm/arm/malidp_crtc.c index 108e7a31bd26..494075ddbef6 100644 --- a/drivers/gpu/drm/arm/malidp_crtc.c +++ b/drivers/gpu/drm/arm/malidp_crtc.c @@ -510,7 +510,6 @@ static void malidp_crtc_disable_vblank(struct drm_crtc *crtc) } static const struct drm_crtc_funcs malidp_crtc_funcs = { - .gamma_set = drm_atomic_helper_legacy_gamma_set, .destroy = drm_crtc_cleanup, .set_config = drm_atomic_helper_set_config, .page_flip = drm_atomic_helper_page_flip, diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c index 3ebcf5a52c8b..b7bb90ae787f 100644 --- a/drivers/gpu/drm/armada/armada_crtc.c +++ b/drivers/gpu/drm/armada/armada_crtc.c @@ -820,7 +820,6 @@ static const struct drm_crtc_funcs armada_crtc_funcs = { .cursor_set = armada_drm_crtc_cursor_set, .cursor_move = armada_drm_crtc_cursor_move, .destroy = armada_drm_crtc_destroy, - .gamma_set = drm_atomic_helper_legacy_gamma_set, .set_config = drm_atomic_helper_set_config, .page_flip = drm_atomic_helper_page_flip, .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, diff --git a/drivers/gpu/drm/ast/ast_cursor.c b/drivers/gpu/drm/ast/ast_cursor.c index 742d43a7edf4..fac1ee79c372 100644 --- a/drivers/gpu/drm/ast/ast_cursor.c +++ b/drivers/gpu/drm/ast/ast_cursor.c @@ -39,7 +39,6 @@ static void ast_cursor_fini(struct ast_private *ast) for (i = 0; i < ARRAY_SIZE(ast->cursor.gbo); ++i) { gbo = ast->cursor.gbo[i]; - drm_gem_vram_vunmap(gbo, &ast->cursor.map[i]); drm_gem_vram_unpin(gbo); drm_gem_vram_put(gbo); } @@ -53,14 +52,13 @@ static void ast_cursor_release(struct drm_device *dev, void *ptr) } /* - * Allocate cursor BOs and pins them at the end of VRAM. + * Allocate cursor BOs and pin them at the end of VRAM. */ int ast_cursor_init(struct ast_private *ast) { struct drm_device *dev = &ast->base; size_t size, i; struct drm_gem_vram_object *gbo; - struct dma_buf_map map; int ret; size = roundup(AST_HWC_SIZE + AST_HWC_SIGNATURE_SIZE, PAGE_SIZE); @@ -77,15 +75,7 @@ int ast_cursor_init(struct ast_private *ast) drm_gem_vram_put(gbo); goto err_drm_gem_vram_put; } - ret = drm_gem_vram_vmap(gbo, &map); - if (ret) { - drm_gem_vram_unpin(gbo); - drm_gem_vram_put(gbo); - goto err_drm_gem_vram_put; - } - ast->cursor.gbo[i] = gbo; - ast->cursor.map[i] = map; } return drmm_add_action_or_reset(dev, ast_cursor_release, NULL); @@ -94,7 +84,6 @@ err_drm_gem_vram_put: while (i) { --i; gbo = ast->cursor.gbo[i]; - drm_gem_vram_vunmap(gbo, &ast->cursor.map[i]); drm_gem_vram_unpin(gbo); drm_gem_vram_put(gbo); } @@ -168,38 +157,37 @@ static void update_cursor_image(u8 __iomem *dst, const u8 *src, int width, int h int ast_cursor_blit(struct ast_private *ast, struct drm_framebuffer *fb) { struct drm_device *dev = &ast->base; - struct drm_gem_vram_object *gbo; - struct dma_buf_map map; - int ret; - void *src; + struct drm_gem_vram_object *dst_gbo = ast->cursor.gbo[ast->cursor.next_index]; + struct drm_gem_vram_object *src_gbo = drm_gem_vram_of_gem(fb->obj[0]); + struct dma_buf_map src_map, dst_map; void __iomem *dst; + void *src; + int ret; if (drm_WARN_ON_ONCE(dev, fb->width > AST_MAX_HWC_WIDTH) || drm_WARN_ON_ONCE(dev, fb->height > AST_MAX_HWC_HEIGHT)) return -EINVAL; - gbo = drm_gem_vram_of_gem(fb->obj[0]); - - ret = drm_gem_vram_pin(gbo, 0); + ret = drm_gem_vram_vmap(src_gbo, &src_map); if (ret) return ret; - ret = drm_gem_vram_vmap(gbo, &map); - if (ret) - goto err_drm_gem_vram_unpin; - src = map.vaddr; /* TODO: Use mapping abstraction properly */ + src = src_map.vaddr; /* TODO: Use mapping abstraction properly */ - dst = ast->cursor.map[ast->cursor.next_index].vaddr_iomem; + ret = drm_gem_vram_vmap(dst_gbo, &dst_map); + if (ret) + goto err_drm_gem_vram_vunmap; + dst = dst_map.vaddr_iomem; /* TODO: Use mapping abstraction properly */ /* do data transfer to cursor BO */ update_cursor_image(dst, src, fb->width, fb->height); - drm_gem_vram_vunmap(gbo, &map); - drm_gem_vram_unpin(gbo); + drm_gem_vram_vunmap(dst_gbo, &dst_map); + drm_gem_vram_vunmap(src_gbo, &src_map); return 0; -err_drm_gem_vram_unpin: - drm_gem_vram_unpin(gbo); +err_drm_gem_vram_vunmap: + drm_gem_vram_vunmap(src_gbo, &src_map); return ret; } @@ -251,17 +239,26 @@ static void ast_cursor_set_location(struct ast_private *ast, u16 x, u16 y, void ast_cursor_show(struct ast_private *ast, int x, int y, unsigned int offset_x, unsigned int offset_y) { + struct drm_device *dev = &ast->base; + struct drm_gem_vram_object *gbo = ast->cursor.gbo[ast->cursor.next_index]; + struct dma_buf_map map; u8 x_offset, y_offset; u8 __iomem *dst; u8 __iomem *sig; u8 jreg; + int ret; - dst = ast->cursor.map[ast->cursor.next_index].vaddr; + ret = drm_gem_vram_vmap(gbo, &map); + if (drm_WARN_ONCE(dev, ret, "drm_gem_vram_vmap() failed, ret=%d\n", ret)) + return; + dst = map.vaddr_iomem; /* TODO: Use mapping abstraction properly */ sig = dst + AST_HWC_SIZE; writel(x, sig + AST_HWC_SIGNATURE_X); writel(y, sig + AST_HWC_SIGNATURE_Y); + drm_gem_vram_vunmap(gbo, &map); + if (x < 0) { x_offset = (-x) + offset_x; x = 0; diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c index 667b450606ef..ea8164e7a6dc 100644 --- a/drivers/gpu/drm/ast/ast_drv.c +++ b/drivers/gpu/drm/ast/ast_drv.c @@ -147,7 +147,7 @@ static int ast_drm_freeze(struct drm_device *dev) error = drm_mode_config_helper_suspend(dev); if (error) return error; - pci_save_state(dev->pdev); + pci_save_state(to_pci_dev(dev->dev)); return 0; } @@ -162,7 +162,7 @@ static int ast_drm_resume(struct drm_device *dev) { int ret; - if (pci_enable_device(dev->pdev)) + if (pci_enable_device(to_pci_dev(dev->dev))) return -EIO; ret = ast_drm_thaw(dev); diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h index ccaff81924ee..f871fc36c2f7 100644 --- a/drivers/gpu/drm/ast/ast_drv.h +++ b/drivers/gpu/drm/ast/ast_drv.h @@ -28,7 +28,6 @@ #ifndef __AST_DRV_H__ #define __AST_DRV_H__ -#include #include #include #include @@ -133,7 +132,6 @@ struct ast_private { struct { struct drm_gem_vram_object *gbo[AST_DEFAULT_HWC_NUM]; - struct dma_buf_map map[AST_DEFAULT_HWC_NUM]; unsigned int next_index; } cursor; diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c index 1b13199858cb..0ac3c2039c4b 100644 --- a/drivers/gpu/drm/ast/ast_main.c +++ b/drivers/gpu/drm/ast/ast_main.c @@ -67,8 +67,9 @@ uint8_t ast_get_index_reg_mask(struct ast_private *ast, static void ast_detect_config_mode(struct drm_device *dev, u32 *scu_rev) { - struct device_node *np = dev->pdev->dev.of_node; + struct device_node *np = dev->dev->of_node; struct ast_private *ast = to_ast_private(dev); + struct pci_dev *pdev = to_pci_dev(dev->dev); uint32_t data, jregd0, jregd1; /* Defaults */ @@ -85,7 +86,7 @@ static void ast_detect_config_mode(struct drm_device *dev, u32 *scu_rev) } /* Not all families have a P2A bridge */ - if (dev->pdev->device != PCI_CHIP_AST2000) + if (pdev->device != PCI_CHIP_AST2000) return; /* @@ -119,6 +120,7 @@ static void ast_detect_config_mode(struct drm_device *dev, u32 *scu_rev) static int ast_detect_chip(struct drm_device *dev, bool *need_post) { struct ast_private *ast = to_ast_private(dev); + struct pci_dev *pdev = to_pci_dev(dev->dev); uint32_t jreg, scu_rev; /* @@ -143,19 +145,19 @@ static int ast_detect_chip(struct drm_device *dev, bool *need_post) ast_detect_config_mode(dev, &scu_rev); /* Identify chipset */ - if (dev->pdev->revision >= 0x50) { + if (pdev->revision >= 0x50) { ast->chip = AST2600; drm_info(dev, "AST 2600 detected\n"); - } else if (dev->pdev->revision >= 0x40) { + } else if (pdev->revision >= 0x40) { ast->chip = AST2500; drm_info(dev, "AST 2500 detected\n"); - } else if (dev->pdev->revision >= 0x30) { + } else if (pdev->revision >= 0x30) { ast->chip = AST2400; drm_info(dev, "AST 2400 detected\n"); - } else if (dev->pdev->revision >= 0x20) { + } else if (pdev->revision >= 0x20) { ast->chip = AST2300; drm_info(dev, "AST 2300 detected\n"); - } else if (dev->pdev->revision >= 0x10) { + } else if (pdev->revision >= 0x10) { switch (scu_rev & 0x0300) { case 0x0200: ast->chip = AST1100; @@ -265,7 +267,7 @@ static int ast_detect_chip(struct drm_device *dev, bool *need_post) static int ast_get_dram_info(struct drm_device *dev) { - struct device_node *np = dev->pdev->dev.of_node; + struct device_node *np = dev->dev->of_node; struct ast_private *ast = to_ast_private(dev); uint32_t mcr_cfg, mcr_scu_mpll, mcr_scu_strap; uint32_t denum, num, div, ref_pll, dsel; @@ -409,10 +411,9 @@ struct ast_private *ast_device_create(const struct drm_driver *drv, return ast; dev = &ast->base; - dev->pdev = pdev; pci_set_drvdata(pdev, dev); - ast->regs = pci_iomap(dev->pdev, 1, 0); + ast->regs = pci_iomap(pdev, 1, 0); if (!ast->regs) return ERR_PTR(-EIO); @@ -421,14 +422,14 @@ struct ast_private *ast_device_create(const struct drm_driver *drv, * assume the chip has MMIO enabled by default (rev 0x20 * and higher). */ - if (!(pci_resource_flags(dev->pdev, 2) & IORESOURCE_IO)) { + if (!(pci_resource_flags(pdev, 2) & IORESOURCE_IO)) { drm_info(dev, "platform has no IO space, trying MMIO\n"); ast->ioregs = ast->regs + AST_IO_MM_OFFSET; } /* "map" IO regs if the above hasn't done so already */ if (!ast->ioregs) { - ast->ioregs = pci_iomap(dev->pdev, 2, 0); + ast->ioregs = pci_iomap(pdev, 2, 0); if (!ast->ioregs) return ERR_PTR(-EIO); } diff --git a/drivers/gpu/drm/ast/ast_mm.c b/drivers/gpu/drm/ast/ast_mm.c index 8392ebde504b..7592f1b9e1f1 100644 --- a/drivers/gpu/drm/ast/ast_mm.c +++ b/drivers/gpu/drm/ast/ast_mm.c @@ -77,31 +77,32 @@ static u32 ast_get_vram_size(struct ast_private *ast) static void ast_mm_release(struct drm_device *dev, void *ptr) { struct ast_private *ast = to_ast_private(dev); + struct pci_dev *pdev = to_pci_dev(dev->dev); arch_phys_wc_del(ast->fb_mtrr); - arch_io_free_memtype_wc(pci_resource_start(dev->pdev, 0), - pci_resource_len(dev->pdev, 0)); + arch_io_free_memtype_wc(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); } int ast_mm_init(struct ast_private *ast) { struct drm_device *dev = &ast->base; + struct pci_dev *pdev = to_pci_dev(dev->dev); u32 vram_size; int ret; vram_size = ast_get_vram_size(ast); - ret = drmm_vram_helper_init(dev, pci_resource_start(dev->pdev, 0), - vram_size); + ret = drmm_vram_helper_init(dev, pci_resource_start(pdev, 0), vram_size); if (ret) { drm_err(dev, "Error initializing VRAM MM; %d\n", ret); return ret; } - arch_io_reserve_memtype_wc(pci_resource_start(dev->pdev, 0), - pci_resource_len(dev->pdev, 0)); - ast->fb_mtrr = arch_phys_wc_add(pci_resource_start(dev->pdev, 0), - pci_resource_len(dev->pdev, 0)); + arch_io_reserve_memtype_wc(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); + ast->fb_mtrr = arch_phys_wc_add(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); return drmm_add_action_or_reset(dev, ast_mm_release, NULL); } diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c index 9db371f4054f..988b270fea5e 100644 --- a/drivers/gpu/drm/ast/ast_mode.c +++ b/drivers/gpu/drm/ast/ast_mode.c @@ -903,7 +903,6 @@ static void ast_crtc_atomic_destroy_state(struct drm_crtc *crtc, static const struct drm_crtc_funcs ast_crtc_funcs = { .reset = ast_crtc_reset, - .gamma_set = drm_atomic_helper_legacy_gamma_set, .destroy = drm_crtc_cleanup, .set_config = drm_atomic_helper_set_config, .page_flip = drm_atomic_helper_page_flip, @@ -1107,6 +1106,7 @@ static const struct drm_mode_config_funcs ast_mode_config_funcs = { int ast_mode_config_init(struct ast_private *ast) { struct drm_device *dev = &ast->base; + struct pci_dev *pdev = to_pci_dev(dev->dev); int ret; ret = ast_cursor_init(ast); @@ -1122,7 +1122,7 @@ int ast_mode_config_init(struct ast_private *ast) dev->mode_config.min_height = 0; dev->mode_config.preferred_depth = 24; dev->mode_config.prefer_shadow = 1; - dev->mode_config.fb_base = pci_resource_start(dev->pdev, 0); + dev->mode_config.fb_base = pci_resource_start(pdev, 0); if (ast->chip == AST2100 || ast->chip == AST2200 || @@ -1259,7 +1259,7 @@ static struct ast_i2c_chan *ast_i2c_create(struct drm_device *dev) i2c->adapter.owner = THIS_MODULE; i2c->adapter.class = I2C_CLASS_DDC; - i2c->adapter.dev.parent = &dev->pdev->dev; + i2c->adapter.dev.parent = dev->dev; i2c->dev = dev; i2c_set_adapdata(&i2c->adapter, i2c); snprintf(i2c->adapter.name, sizeof(i2c->adapter.name), diff --git a/drivers/gpu/drm/ast/ast_post.c b/drivers/gpu/drm/ast/ast_post.c index 8902c2f84bf9..0607658dde51 100644 --- a/drivers/gpu/drm/ast/ast_post.c +++ b/drivers/gpu/drm/ast/ast_post.c @@ -71,6 +71,7 @@ static void ast_set_def_ext_reg(struct drm_device *dev) { struct ast_private *ast = to_ast_private(dev); + struct pci_dev *pdev = to_pci_dev(dev->dev); u8 i, index, reg; const u8 *ext_reg_info; @@ -80,7 +81,7 @@ ast_set_def_ext_reg(struct drm_device *dev) if (ast->chip == AST2300 || ast->chip == AST2400 || ast->chip == AST2500) { - if (dev->pdev->revision >= 0x20) + if (pdev->revision >= 0x20) ext_reg_info = extreginfo_ast2300; else ext_reg_info = extreginfo_ast2300a0; @@ -366,11 +367,12 @@ static void ast_init_dram_reg(struct drm_device *dev) void ast_post_gpu(struct drm_device *dev) { struct ast_private *ast = to_ast_private(dev); + struct pci_dev *pdev = to_pci_dev(dev->dev); u32 reg; - pci_read_config_dword(dev->pdev, 0x04, ®); + pci_read_config_dword(pdev, 0x04, ®); reg |= 0x3; - pci_write_config_dword(dev->pdev, 0x04, reg); + pci_write_config_dword(pdev, 0x04, reg); ast_enable_vga(dev); ast_open_key(ast); diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c index c58fa00b4848..05ad75d155e8 100644 --- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c +++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c @@ -473,7 +473,6 @@ static const struct drm_crtc_funcs atmel_hlcdc_crtc_funcs = { .atomic_destroy_state = atmel_hlcdc_crtc_destroy_state, .enable_vblank = atmel_hlcdc_crtc_enable_vblank, .disable_vblank = atmel_hlcdc_crtc_disable_vblank, - .gamma_set = drm_atomic_helper_legacy_gamma_set, }; int atmel_hlcdc_crtc_create(struct drm_device *dev) diff --git a/drivers/gpu/drm/bochs/bochs_drv.c b/drivers/gpu/drm/bochs/bochs_drv.c index fd454225fd19..b469624fe40d 100644 --- a/drivers/gpu/drm/bochs/bochs_drv.c +++ b/drivers/gpu/drm/bochs/bochs_drv.c @@ -121,7 +121,6 @@ static int bochs_pci_probe(struct pci_dev *pdev, if (ret) goto err_free_dev; - dev->pdev = pdev; pci_set_drvdata(pdev, dev); ret = bochs_load(dev); diff --git a/drivers/gpu/drm/bochs/bochs_hw.c b/drivers/gpu/drm/bochs/bochs_hw.c index dce4672e3fc8..2d7380a9890e 100644 --- a/drivers/gpu/drm/bochs/bochs_hw.c +++ b/drivers/gpu/drm/bochs/bochs_hw.c @@ -110,7 +110,7 @@ int bochs_hw_load_edid(struct bochs_device *bochs) int bochs_hw_init(struct drm_device *dev) { struct bochs_device *bochs = dev->dev_private; - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = to_pci_dev(dev->dev); unsigned long addr, size, mem, ioaddr, iosize; u16 id; @@ -201,7 +201,7 @@ void bochs_hw_fini(struct drm_device *dev) release_region(VBE_DISPI_IOPORT_INDEX, 2); if (bochs->fb_map) iounmap(bochs->fb_map); - pci_release_regions(dev->pdev); + pci_release_regions(to_pci_dev(dev->dev)); kfree(bochs->edid); } diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c index a0d392c338da..76555ae64e9c 100644 --- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c +++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c @@ -1292,8 +1292,7 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id) err_unregister_cec: i2c_unregister_device(adv7511->i2c_cec); - if (adv7511->cec_clk) - clk_disable_unprepare(adv7511->cec_clk); + clk_disable_unprepare(adv7511->cec_clk); err_i2c_unregister_packet: i2c_unregister_device(adv7511->i2c_packet); err_i2c_unregister_edid: @@ -1311,8 +1310,7 @@ static int adv7511_remove(struct i2c_client *i2c) if (adv7511->type == ADV7533 || adv7511->type == ADV7535) adv7533_detach_dsi(adv7511); i2c_unregister_device(adv7511->i2c_cec); - if (adv7511->cec_clk) - clk_disable_unprepare(adv7511->cec_clk); + clk_disable_unprepare(adv7511->cec_clk); adv7511_uninit_regulators(adv7511); diff --git a/drivers/gpu/drm/bridge/display-connector.c b/drivers/gpu/drm/bridge/display-connector.c index 4d278573cdb9..05eb759da6fc 100644 --- a/drivers/gpu/drm/bridge/display-connector.c +++ b/drivers/gpu/drm/bridge/display-connector.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -20,6 +21,8 @@ struct display_connector { struct gpio_desc *hpd_gpio; int hpd_irq; + + struct regulator *dp_pwr; }; static inline struct display_connector * @@ -172,11 +175,12 @@ static int display_connector_probe(struct platform_device *pdev) of_property_read_string(pdev->dev.of_node, "label", &label); /* - * Get the HPD GPIO for DVI and HDMI connectors. If the GPIO can provide + * Get the HPD GPIO for DVI, HDMI and DP connectors. If the GPIO can provide * edge interrupts, register an interrupt handler. */ if (type == DRM_MODE_CONNECTOR_DVII || - type == DRM_MODE_CONNECTOR_HDMIA) { + type == DRM_MODE_CONNECTOR_HDMIA || + type == DRM_MODE_CONNECTOR_DisplayPort) { conn->hpd_gpio = devm_gpiod_get_optional(&pdev->dev, "hpd", GPIOD_IN); if (IS_ERR(conn->hpd_gpio)) { @@ -223,6 +227,38 @@ static int display_connector_probe(struct platform_device *pdev) } } + /* Get the DP PWR for DP connector. */ + if (type == DRM_MODE_CONNECTOR_DisplayPort) { + int ret; + + conn->dp_pwr = devm_regulator_get_optional(&pdev->dev, "dp-pwr"); + + if (IS_ERR(conn->dp_pwr)) { + ret = PTR_ERR(conn->dp_pwr); + + switch (ret) { + case -ENODEV: + conn->dp_pwr = NULL; + break; + + case -EPROBE_DEFER: + return -EPROBE_DEFER; + + default: + dev_err(&pdev->dev, "failed to get DP PWR regulator: %d\n", ret); + return ret; + } + } + + if (conn->dp_pwr) { + ret = regulator_enable(conn->dp_pwr); + if (ret) { + dev_err(&pdev->dev, "failed to enable DP PWR regulator: %d\n", ret); + return ret; + } + } + } + conn->bridge.funcs = &display_connector_bridge_funcs; conn->bridge.of_node = pdev->dev.of_node; @@ -251,6 +287,9 @@ static int display_connector_remove(struct platform_device *pdev) { struct display_connector *conn = platform_get_drvdata(pdev); + if (conn->dp_pwr) + regulator_disable(conn->dp_pwr); + drm_bridge_remove(&conn->bridge); if (!IS_ERR(conn->bridge.ddc)) @@ -275,6 +314,9 @@ static const struct of_device_id display_connector_match[] = { }, { .compatible = "vga-connector", .data = (void *)DRM_MODE_CONNECTOR_VGA, + }, { + .compatible = "dp-connector", + .data = (void *)DRM_MODE_CONNECTOR_DisplayPort, }, {}, }; diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c index 0c79a9ba48bb..dda4fa9a1a08 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c @@ -3440,8 +3440,7 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, err_iahb: clk_disable_unprepare(hdmi->iahb_clk); - if (hdmi->cec_clk) - clk_disable_unprepare(hdmi->cec_clk); + clk_disable_unprepare(hdmi->cec_clk); err_isfr: clk_disable_unprepare(hdmi->isfr_clk); err_res: @@ -3465,8 +3464,7 @@ void dw_hdmi_remove(struct dw_hdmi *hdmi) clk_disable_unprepare(hdmi->iahb_clk); clk_disable_unprepare(hdmi->isfr_clk); - if (hdmi->cec_clk) - clk_disable_unprepare(hdmi->cec_clk); + clk_disable_unprepare(hdmi->cec_clk); if (hdmi->i2c) i2c_del_adapter(&hdmi->i2c->adap); diff --git a/drivers/gpu/drm/bridge/thc63lvd1024.c b/drivers/gpu/drm/bridge/thc63lvd1024.c index 86b06975bfdd..e21078b2f8b5 100644 --- a/drivers/gpu/drm/bridge/thc63lvd1024.c +++ b/drivers/gpu/drm/bridge/thc63lvd1024.c @@ -202,7 +202,7 @@ static int thc63_probe(struct platform_device *pdev) thc63->dev = &pdev->dev; platform_set_drvdata(pdev, thc63); - thc63->vcc = devm_regulator_get_optional(thc63->dev, "vcc"); + thc63->vcc = devm_regulator_get(thc63->dev, "vcc"); if (IS_ERR(thc63->vcc)) { if (PTR_ERR(thc63->vcc) == -EPROBE_DEFER) return -EPROBE_DEFER; diff --git a/drivers/gpu/drm/drm_agpsupport.c b/drivers/gpu/drm/drm_agpsupport.c index 4c7ad46fdd21..5311d03d49cc 100644 --- a/drivers/gpu/drm/drm_agpsupport.c +++ b/drivers/gpu/drm/drm_agpsupport.c @@ -45,13 +45,9 @@ #include "drm_legacy.h" -/** +/* * Get AGP information. * - * \param inode device inode. - * \param file_priv DRM file private. - * \param cmd command. - * \param arg pointer to a (output) drm_agp_info structure. * \return zero on success or a negative number on failure. * * Verifies the AGP device has been initialized and acquired and fills in the @@ -92,7 +88,7 @@ int drm_agp_info_ioctl(struct drm_device *dev, void *data, return 0; } -/** +/* * Acquire the AGP device. * * \param dev DRM device that is to acquire AGP. @@ -103,11 +99,13 @@ int drm_agp_info_ioctl(struct drm_device *dev, void *data, */ int drm_agp_acquire(struct drm_device *dev) { + struct pci_dev *pdev = to_pci_dev(dev->dev); + if (!dev->agp) return -ENODEV; if (dev->agp->acquired) return -EBUSY; - dev->agp->bridge = agp_backend_acquire(dev->pdev); + dev->agp->bridge = agp_backend_acquire(pdev); if (!dev->agp->bridge) return -ENODEV; dev->agp->acquired = 1; @@ -115,13 +113,9 @@ int drm_agp_acquire(struct drm_device *dev) } EXPORT_SYMBOL(drm_agp_acquire); -/** +/* * Acquire the AGP device (ioctl). * - * \param inode device inode. - * \param file_priv DRM file private. - * \param cmd command. - * \param arg user argument. * \return zero on success or a negative number on failure. * * Verifies the AGP device hasn't been acquired before and calls @@ -133,7 +127,7 @@ int drm_agp_acquire_ioctl(struct drm_device *dev, void *data, return drm_agp_acquire((struct drm_device *) file_priv->minor->dev); } -/** +/* * Release the AGP device. * * \param dev DRM device that is to release AGP. @@ -157,7 +151,7 @@ int drm_agp_release_ioctl(struct drm_device *dev, void *data, return drm_agp_release(dev); } -/** +/* * Enable the AGP bus. * * \param dev DRM device that has previously acquired AGP. @@ -187,13 +181,9 @@ int drm_agp_enable_ioctl(struct drm_device *dev, void *data, return drm_agp_enable(dev, *mode); } -/** +/* * Allocate AGP memory. * - * \param inode device inode. - * \param file_priv file private pointer. - * \param cmd command. - * \param arg pointer to a drm_agp_buffer structure. * \return zero on success or a negative number on failure. * * Verifies the AGP device is present and has been acquired, allocates the @@ -242,7 +232,7 @@ int drm_agp_alloc_ioctl(struct drm_device *dev, void *data, return drm_agp_alloc(dev, request); } -/** +/* * Search for the AGP memory entry associated with a handle. * * \param dev DRM device structure. @@ -263,13 +253,9 @@ static struct drm_agp_mem *drm_agp_lookup_entry(struct drm_device *dev, return NULL; } -/** +/* * Unbind AGP memory from the GATT (ioctl). * - * \param inode device inode. - * \param file_priv DRM file private. - * \param cmd command. - * \param arg pointer to a drm_agp_binding structure. * \return zero on success or a negative number on failure. * * Verifies the AGP device is present and acquired, looks-up the AGP memory @@ -285,7 +271,7 @@ int drm_agp_unbind(struct drm_device *dev, struct drm_agp_binding *request) entry = drm_agp_lookup_entry(dev, request->handle); if (!entry || !entry->bound) return -EINVAL; - ret = drm_unbind_agp(entry->memory); + ret = agp_unbind_memory(entry->memory); if (ret == 0) entry->bound = 0; return ret; @@ -301,13 +287,9 @@ int drm_agp_unbind_ioctl(struct drm_device *dev, void *data, return drm_agp_unbind(dev, request); } -/** +/* * Bind AGP memory into the GATT (ioctl) * - * \param inode device inode. - * \param file_priv DRM file private. - * \param cmd command. - * \param arg pointer to a drm_agp_binding structure. * \return zero on success or a negative number on failure. * * Verifies the AGP device is present and has been acquired and that no memory @@ -326,7 +308,7 @@ int drm_agp_bind(struct drm_device *dev, struct drm_agp_binding *request) if (!entry || entry->bound) return -EINVAL; page = DIV_ROUND_UP(request->offset, PAGE_SIZE); - retcode = drm_bind_agp(entry->memory, page); + retcode = agp_bind_memory(entry->memory, page); if (retcode) return retcode; entry->bound = dev->agp->base + (page << PAGE_SHIFT); @@ -345,13 +327,9 @@ int drm_agp_bind_ioctl(struct drm_device *dev, void *data, return drm_agp_bind(dev, request); } -/** +/* * Free AGP memory (ioctl). * - * \param inode device inode. - * \param file_priv DRM file private. - * \param cmd command. - * \param arg pointer to a drm_agp_buffer structure. * \return zero on success or a negative number on failure. * * Verifies the AGP device is present and has been acquired and looks up the @@ -369,11 +347,11 @@ int drm_agp_free(struct drm_device *dev, struct drm_agp_buffer *request) if (!entry) return -EINVAL; if (entry->bound) - drm_unbind_agp(entry->memory); + agp_unbind_memory(entry->memory); list_del(&entry->head); - drm_free_agp(entry->memory, entry->pages); + agp_free_memory(entry->memory); kfree(entry); return 0; } @@ -388,7 +366,7 @@ int drm_agp_free_ioctl(struct drm_device *dev, void *data, return drm_agp_free(dev, request); } -/** +/* * Initialize the AGP resources. * * \return pointer to a drm_agp_head structure. @@ -402,14 +380,15 @@ int drm_agp_free_ioctl(struct drm_device *dev, void *data, */ struct drm_agp_head *drm_agp_init(struct drm_device *dev) { + struct pci_dev *pdev = to_pci_dev(dev->dev); struct drm_agp_head *head = NULL; head = kzalloc(sizeof(*head), GFP_KERNEL); if (!head) return NULL; - head->bridge = agp_find_bridge(dev->pdev); + head->bridge = agp_find_bridge(pdev); if (!head->bridge) { - head->bridge = agp_backend_acquire(dev->pdev); + head->bridge = agp_backend_acquire(pdev); if (!head->bridge) { kfree(head); return NULL; @@ -453,8 +432,8 @@ void drm_legacy_agp_clear(struct drm_device *dev) list_for_each_entry_safe(entry, tempe, &dev->agp->memory, head) { if (entry->bound) - drm_unbind_agp(entry->memory); - drm_free_agp(entry->memory, entry->pages); + agp_unbind_memory(entry->memory); + agp_free_memory(entry->memory); kfree(entry); } INIT_LIST_HEAD(&dev->agp->memory); diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 4a8cbec832bc..560aaecba31b 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -2040,6 +2040,9 @@ crtc_or_fake_commit(struct drm_atomic_state *state, struct drm_crtc *crtc) * should always call this function from their * &drm_mode_config_funcs.atomic_commit hook. * + * Drivers that need to extend the commit setup to private objects can use the + * &drm_mode_config_helper_funcs.atomic_commit_setup hook. + * * To be able to use this support drivers need to use a few more helper * functions. drm_atomic_helper_wait_for_dependencies() must be called before * actually committing the hardware state, and for nonblocking commits this call @@ -2083,8 +2086,11 @@ int drm_atomic_helper_setup_commit(struct drm_atomic_state *state, struct drm_plane *plane; struct drm_plane_state *old_plane_state, *new_plane_state; struct drm_crtc_commit *commit; + const struct drm_mode_config_helper_funcs *funcs; int i, ret; + funcs = state->dev->mode_config.helper_private; + for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { commit = kzalloc(sizeof(*commit), GFP_KERNEL); if (!commit) @@ -2169,6 +2175,9 @@ int drm_atomic_helper_setup_commit(struct drm_atomic_state *state, new_plane_state->commit = drm_crtc_commit_get(commit); } + if (funcs && funcs->atomic_commit_setup) + return funcs->atomic_commit_setup(state); + return 0; } EXPORT_SYMBOL(drm_atomic_helper_setup_commit); @@ -3499,76 +3508,6 @@ fail: } EXPORT_SYMBOL(drm_atomic_helper_page_flip_target); -/** - * drm_atomic_helper_legacy_gamma_set - set the legacy gamma correction table - * @crtc: CRTC object - * @red: red correction table - * @green: green correction table - * @blue: green correction table - * @size: size of the tables - * @ctx: lock acquire context - * - * Implements support for legacy gamma correction table for drivers - * that support color management through the DEGAMMA_LUT/GAMMA_LUT - * properties. See drm_crtc_enable_color_mgmt() and the containing chapter for - * how the atomic color management and gamma tables work. - */ -int drm_atomic_helper_legacy_gamma_set(struct drm_crtc *crtc, - u16 *red, u16 *green, u16 *blue, - uint32_t size, - struct drm_modeset_acquire_ctx *ctx) -{ - struct drm_device *dev = crtc->dev; - struct drm_atomic_state *state; - struct drm_crtc_state *crtc_state; - struct drm_property_blob *blob = NULL; - struct drm_color_lut *blob_data; - int i, ret = 0; - bool replaced; - - state = drm_atomic_state_alloc(crtc->dev); - if (!state) - return -ENOMEM; - - blob = drm_property_create_blob(dev, - sizeof(struct drm_color_lut) * size, - NULL); - if (IS_ERR(blob)) { - ret = PTR_ERR(blob); - blob = NULL; - goto fail; - } - - /* Prepare GAMMA_LUT with the legacy values. */ - blob_data = blob->data; - for (i = 0; i < size; i++) { - blob_data[i].red = red[i]; - blob_data[i].green = green[i]; - blob_data[i].blue = blue[i]; - } - - state->acquire_ctx = ctx; - crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(crtc_state)) { - ret = PTR_ERR(crtc_state); - goto fail; - } - - /* Reset DEGAMMA_LUT and CTM properties. */ - replaced = drm_property_replace_blob(&crtc_state->degamma_lut, NULL); - replaced |= drm_property_replace_blob(&crtc_state->ctm, NULL); - replaced |= drm_property_replace_blob(&crtc_state->gamma_lut, blob); - crtc_state->color_mgmt_changed |= replaced; - - ret = drm_atomic_commit(state); - -fail: - drm_atomic_state_put(state); - drm_property_blob_put(blob); - return ret; -} -EXPORT_SYMBOL(drm_atomic_helper_legacy_gamma_set); - /** * drm_atomic_helper_bridge_propagate_bus_fmt() - Propagate output format to * the input end of a bridge diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c index 5c2141e9a9f4..26e2f2ffd255 100644 --- a/drivers/gpu/drm/drm_blend.c +++ b/drivers/gpu/drm/drm_blend.c @@ -185,12 +185,6 @@ * plane does not expose the "alpha" property, then this is * assumed to be 1.0 * - * IN_FORMATS: - * Blob property which contains the set of buffer format and modifier - * pairs supported by this plane. The blob is a drm_format_modifier_blob - * struct. Without this property the plane doesn't support buffers with - * modifiers. Userspace cannot change this property. - * * Note that all the property extensions described here apply either to the * plane or the CRTC (e.g. for the background color, which currently is not * exposed and assumed to be black). diff --git a/drivers/gpu/drm/drm_bufs.c b/drivers/gpu/drm/drm_bufs.c index aeb1327e3077..e3d77dfefb0a 100644 --- a/drivers/gpu/drm/drm_bufs.c +++ b/drivers/gpu/drm/drm_bufs.c @@ -326,7 +326,7 @@ static int drm_addmap_core(struct drm_device *dev, resource_size_t offset, * As we're limiting the address to 2^32-1 (or less), * casting it down to 32 bits is no problem, but we * need to point to a 64bit variable first. */ - map->handle = dma_alloc_coherent(&dev->pdev->dev, + map->handle = dma_alloc_coherent(dev->dev, map->size, &map->offset, GFP_KERNEL); @@ -556,7 +556,7 @@ int drm_legacy_rmmap_locked(struct drm_device *dev, struct drm_local_map *map) case _DRM_SCATTER_GATHER: break; case _DRM_CONSISTENT: - dma_free_coherent(&dev->pdev->dev, + dma_free_coherent(dev->dev, map->size, map->handle, map->offset); diff --git a/drivers/gpu/drm/drm_cache.c b/drivers/gpu/drm/drm_cache.c index 0fe3c496002a..79a50ef1250f 100644 --- a/drivers/gpu/drm/drm_cache.c +++ b/drivers/gpu/drm/drm_cache.c @@ -30,6 +30,8 @@ #include #include +#include +#include #include @@ -176,3 +178,34 @@ drm_clflush_virt_range(void *addr, unsigned long length) #endif } EXPORT_SYMBOL(drm_clflush_virt_range); + +bool drm_need_swiotlb(int dma_bits) +{ + struct resource *tmp; + resource_size_t max_iomem = 0; + + /* + * Xen paravirtual hosts require swiotlb regardless of requested dma + * transfer size. + * + * NOTE: Really, what it requires is use of the dma_alloc_coherent + * allocator used in ttm_dma_populate() instead of + * ttm_populate_and_map_pages(), which bounce buffers so much in + * Xen it leads to swiotlb buffer exhaustion. + */ + if (xen_pv_domain()) + return true; + + /* + * Enforce dma_alloc_coherent when memory encryption is active as well + * for the same reasons as for Xen paravirtual hosts. + */ + if (mem_encrypt_active()) + return true; + + for (tmp = iomem_resource.child; tmp; tmp = tmp->sibling) + max_iomem = max(max_iomem, tmp->end); + + return max_iomem > ((u64)1 << dma_bits); +} +EXPORT_SYMBOL(drm_need_swiotlb); diff --git a/drivers/gpu/drm/drm_client_modeset.c b/drivers/gpu/drm/drm_client_modeset.c index b7e9e1c2564c..ced09c7c06f9 100644 --- a/drivers/gpu/drm/drm_client_modeset.c +++ b/drivers/gpu/drm/drm_client_modeset.c @@ -7,6 +7,7 @@ * Copyright (c) 2007 Dave Airlie */ +#include "drm/drm_modeset_lock.h" #include #include #include @@ -1181,9 +1182,11 @@ static void drm_client_modeset_dpms_legacy(struct drm_client_dev *client, int dp struct drm_device *dev = client->dev; struct drm_connector *connector; struct drm_mode_set *modeset; + struct drm_modeset_acquire_ctx ctx; int j; + int ret; - drm_modeset_lock_all(dev); + DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); drm_client_for_each_modeset(modeset, client) { if (!modeset->crtc->enabled) continue; @@ -1195,7 +1198,7 @@ static void drm_client_modeset_dpms_legacy(struct drm_client_dev *client, int dp dev->mode_config.dpms_property, dpms_mode); } } - drm_modeset_unlock_all(dev); + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); } /** diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c index 3bcabc2f6e0e..bb14f488c8f6 100644 --- a/drivers/gpu/drm/drm_color_mgmt.c +++ b/drivers/gpu/drm/drm_color_mgmt.c @@ -22,6 +22,7 @@ #include +#include #include #include #include @@ -89,9 +90,8 @@ * modes) appropriately. * * There is also support for a legacy gamma table, which is set up by calling - * drm_mode_crtc_set_gamma_size(). Drivers which support both should use - * drm_atomic_helper_legacy_gamma_set() to alias the legacy gamma ramp with the - * "GAMMA_LUT" property above. + * drm_mode_crtc_set_gamma_size(). The DRM core will then alias the legacy gamma + * ramp with "GAMMA_LUT" or, if that is unavailable, "DEGAMMA_LUT". * * Support for different non RGB color encodings is controlled through * &drm_plane specific COLOR_ENCODING and COLOR_RANGE properties. They @@ -156,9 +156,6 @@ EXPORT_SYMBOL(drm_color_ctm_s31_32_to_qm_n); * optional. The gamma and degamma properties are only attached if * their size is not 0 and ctm_property is only attached if has_ctm is * true. - * - * Drivers should use drm_atomic_helper_legacy_gamma_set() to implement the - * legacy &drm_crtc_funcs.gamma_set callback. */ void drm_crtc_enable_color_mgmt(struct drm_crtc *crtc, uint degamma_lut_size, @@ -231,6 +228,116 @@ int drm_mode_crtc_set_gamma_size(struct drm_crtc *crtc, } EXPORT_SYMBOL(drm_mode_crtc_set_gamma_size); +/** + * drm_crtc_supports_legacy_gamma - does the crtc support legacy gamma correction table + * @crtc: CRTC object + * + * Returns true/false if the given crtc supports setting the legacy gamma + * correction table. + */ +static bool drm_crtc_supports_legacy_gamma(struct drm_crtc *crtc) +{ + u32 gamma_id = crtc->dev->mode_config.gamma_lut_property->base.id; + u32 degamma_id = crtc->dev->mode_config.degamma_lut_property->base.id; + + if (!crtc->gamma_size) + return false; + + if (crtc->funcs->gamma_set) + return true; + + return !!(drm_mode_obj_find_prop_id(&crtc->base, gamma_id) || + drm_mode_obj_find_prop_id(&crtc->base, degamma_id)); +} + +/** + * drm_crtc_legacy_gamma_set - set the legacy gamma correction table + * @crtc: CRTC object + * @red: red correction table + * @green: green correction table + * @blue: green correction table + * @size: size of the tables + * @ctx: lock acquire context + * + * Implements support for legacy gamma correction table for drivers + * that have set drm_crtc_funcs.gamma_set or that support color management + * through the DEGAMMA_LUT/GAMMA_LUT properties. See + * drm_crtc_enable_color_mgmt() and the containing chapter for + * how the atomic color management and gamma tables work. + * + * This function sets the gamma using drm_crtc_funcs.gamma_set if set, or + * alternatively using crtc color management properties. + */ +static int drm_crtc_legacy_gamma_set(struct drm_crtc *crtc, + u16 *red, u16 *green, u16 *blue, + u32 size, + struct drm_modeset_acquire_ctx *ctx) +{ + struct drm_device *dev = crtc->dev; + struct drm_atomic_state *state; + struct drm_crtc_state *crtc_state; + struct drm_property_blob *blob; + struct drm_color_lut *blob_data; + u32 gamma_id = dev->mode_config.gamma_lut_property->base.id; + u32 degamma_id = dev->mode_config.degamma_lut_property->base.id; + bool use_gamma_lut; + int i, ret = 0; + bool replaced; + + if (crtc->funcs->gamma_set) + return crtc->funcs->gamma_set(crtc, red, green, blue, size, ctx); + + if (drm_mode_obj_find_prop_id(&crtc->base, gamma_id)) + use_gamma_lut = true; + else if (drm_mode_obj_find_prop_id(&crtc->base, degamma_id)) + use_gamma_lut = false; + else + return -ENODEV; + + state = drm_atomic_state_alloc(crtc->dev); + if (!state) + return -ENOMEM; + + blob = drm_property_create_blob(dev, + sizeof(struct drm_color_lut) * size, + NULL); + if (IS_ERR(blob)) { + ret = PTR_ERR(blob); + blob = NULL; + goto fail; + } + + /* Prepare GAMMA_LUT with the legacy values. */ + blob_data = blob->data; + for (i = 0; i < size; i++) { + blob_data[i].red = red[i]; + blob_data[i].green = green[i]; + blob_data[i].blue = blue[i]; + } + + state->acquire_ctx = ctx; + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto fail; + } + + /* Set GAMMA_LUT and reset DEGAMMA_LUT and CTM */ + replaced = drm_property_replace_blob(&crtc_state->degamma_lut, + use_gamma_lut ? NULL : blob); + replaced |= drm_property_replace_blob(&crtc_state->ctm, NULL); + replaced |= drm_property_replace_blob(&crtc_state->gamma_lut, + use_gamma_lut ? blob : NULL); + crtc_state->color_mgmt_changed |= replaced; + + ret = drm_atomic_commit(state); + +fail: + drm_atomic_state_put(state); + drm_property_blob_put(blob); + return ret; +} + /** * drm_mode_gamma_set_ioctl - set the gamma table * @dev: DRM device @@ -262,7 +369,7 @@ int drm_mode_gamma_set_ioctl(struct drm_device *dev, if (!crtc) return -ENOENT; - if (crtc->funcs->gamma_set == NULL) + if (!drm_crtc_supports_legacy_gamma(crtc)) return -ENOSYS; /* memcpy into gamma store */ @@ -290,8 +397,8 @@ int drm_mode_gamma_set_ioctl(struct drm_device *dev, goto out; } - ret = crtc->funcs->gamma_set(crtc, r_base, g_base, b_base, - crtc->gamma_size, &ctx); + ret = drm_crtc_legacy_gamma_set(crtc, r_base, g_base, b_base, + crtc->gamma_size, &ctx); out: DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c index 74090fc3aa55..9c4f9947b194 100644 --- a/drivers/gpu/drm/drm_crtc.c +++ b/drivers/gpu/drm/drm_crtc.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -67,7 +68,7 @@ * &drm_crtc_funcs.page_flip and &drm_crtc_funcs.cursor_set2, and other legacy * operations like &drm_crtc_funcs.gamma_set. For atomic drivers all these * features are controlled through &drm_property and - * &drm_mode_config_funcs.atomic_check and &drm_mode_config_funcs.atomic_check. + * &drm_mode_config_funcs.atomic_check. */ /** @@ -240,30 +241,12 @@ struct dma_fence *drm_crtc_create_fence(struct drm_crtc *crtc) * Nearest Neighbor scaling filter */ -/** - * drm_crtc_init_with_planes - Initialise a new CRTC object with - * specified primary and cursor planes. - * @dev: DRM device - * @crtc: CRTC object to init - * @primary: Primary plane for CRTC - * @cursor: Cursor plane for CRTC - * @funcs: callbacks for the new CRTC - * @name: printf style format string for the CRTC name, or NULL for default name - * - * Inits a new object created as base part of a driver crtc object. Drivers - * should use this function instead of drm_crtc_init(), which is only provided - * for backwards compatibility with drivers which do not yet support universal - * planes). For really simple hardware which has only 1 plane look at - * drm_simple_display_pipe_init() instead. - * - * Returns: - * Zero on success, error code on failure. - */ -int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc, - struct drm_plane *primary, - struct drm_plane *cursor, - const struct drm_crtc_funcs *funcs, - const char *name, ...) +__printf(6, 0) +static int __drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc, + struct drm_plane *primary, + struct drm_plane *cursor, + const struct drm_crtc_funcs *funcs, + const char *name, va_list ap) { struct drm_mode_config *config = &dev->mode_config; int ret; @@ -291,11 +274,7 @@ int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc, return ret; if (name) { - va_list ap; - - va_start(ap, name); crtc->name = kvasprintf(GFP_KERNEL, name, ap); - va_end(ap); } else { crtc->name = kasprintf(GFP_KERNEL, "crtc-%d", drm_num_crtcs(dev)); @@ -339,8 +318,101 @@ int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc, return 0; } + +/** + * drm_crtc_init_with_planes - Initialise a new CRTC object with + * specified primary and cursor planes. + * @dev: DRM device + * @crtc: CRTC object to init + * @primary: Primary plane for CRTC + * @cursor: Cursor plane for CRTC + * @funcs: callbacks for the new CRTC + * @name: printf style format string for the CRTC name, or NULL for default name + * + * Inits a new object created as base part of a driver crtc object. Drivers + * should use this function instead of drm_crtc_init(), which is only provided + * for backwards compatibility with drivers which do not yet support universal + * planes). For really simple hardware which has only 1 plane look at + * drm_simple_display_pipe_init() instead. + * The &drm_crtc_funcs.destroy hook should call drm_crtc_cleanup() and kfree() + * the crtc structure. The crtc structure should not be allocated with + * devm_kzalloc(). + * + * The @primary and @cursor planes are only relevant for legacy uAPI, see + * &drm_crtc.primary and &drm_crtc.cursor. + * + * Note: consider using drmm_crtc_alloc_with_planes() instead of + * drm_crtc_init_with_planes() to let the DRM managed resource infrastructure + * take care of cleanup and deallocation. + * + * Returns: + * Zero on success, error code on failure. + */ +int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc, + struct drm_plane *primary, + struct drm_plane *cursor, + const struct drm_crtc_funcs *funcs, + const char *name, ...) +{ + va_list ap; + int ret; + + WARN_ON(!funcs->destroy); + + va_start(ap, name); + ret = __drm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs, + name, ap); + va_end(ap); + + return ret; +} EXPORT_SYMBOL(drm_crtc_init_with_planes); +static void drmm_crtc_alloc_with_planes_cleanup(struct drm_device *dev, + void *ptr) +{ + struct drm_crtc *crtc = ptr; + + drm_crtc_cleanup(crtc); +} + +void *__drmm_crtc_alloc_with_planes(struct drm_device *dev, + size_t size, size_t offset, + struct drm_plane *primary, + struct drm_plane *cursor, + const struct drm_crtc_funcs *funcs, + const char *name, ...) +{ + void *container; + struct drm_crtc *crtc; + va_list ap; + int ret; + + if (WARN_ON(!funcs || funcs->destroy)) + return ERR_PTR(-EINVAL); + + container = drmm_kzalloc(dev, size, GFP_KERNEL); + if (!container) + return ERR_PTR(-ENOMEM); + + crtc = container + offset; + + va_start(ap, name); + ret = __drm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs, + name, ap); + va_end(ap); + if (ret) + return ERR_PTR(ret); + + ret = drmm_add_action_or_reset(dev, drmm_crtc_alloc_with_planes_cleanup, + crtc); + if (ret) + return ERR_PTR(ret); + + return container; +} +EXPORT_SYMBOL(__drmm_crtc_alloc_with_planes); + /** * drm_crtc_cleanup - Clean up the core crtc usage * @crtc: CRTC to cleanup diff --git a/drivers/gpu/drm/drm_crtc_helper_internal.h b/drivers/gpu/drm/drm_crtc_helper_internal.h index 25ce42e79995..61e09f8a8d0f 100644 --- a/drivers/gpu/drm/drm_crtc_helper_internal.h +++ b/drivers/gpu/drm/drm_crtc_helper_internal.h @@ -32,16 +32,6 @@ #include #include -/* drm_fb_helper.c */ -#ifdef CONFIG_DRM_FBDEV_EMULATION -int drm_fb_helper_modinit(void); -#else -static inline int drm_fb_helper_modinit(void) -{ - return 0; -} -#endif - /* drm_dp_aux_dev.c */ #ifdef CONFIG_DRM_DP_AUX_CHARDEV int drm_dp_aux_dev_init(void); diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c index 5bd0934004e3..eedbb48815b7 100644 --- a/drivers/gpu/drm/drm_dp_helper.c +++ b/drivers/gpu/drm/drm_dp_helper.c @@ -949,6 +949,38 @@ bool drm_dp_downstream_444_to_420_conversion(const u8 dpcd[DP_RECEIVER_CAP_SIZE] } EXPORT_SYMBOL(drm_dp_downstream_444_to_420_conversion); +/** + * drm_dp_downstream_rgb_to_ycbcr_conversion() - determine downstream facing port + * RGB->YCbCr conversion capability + * @dpcd: DisplayPort configuration data + * @port_cap: downstream facing port capabilities + * @color_spc: Colorspace for which conversion cap is sought + * + * Returns: whether the downstream facing port can convert RGB->YCbCr for a given + * colorspace. + */ +bool drm_dp_downstream_rgb_to_ycbcr_conversion(const u8 dpcd[DP_RECEIVER_CAP_SIZE], + const u8 port_cap[4], + u8 color_spc) +{ + if (!drm_dp_is_branch(dpcd)) + return false; + + if (dpcd[DP_DPCD_REV] < 0x13) + return false; + + switch (port_cap[0] & DP_DS_PORT_TYPE_MASK) { + case DP_DS_PORT_TYPE_HDMI: + if ((dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DETAILED_CAP_INFO_AVAILABLE) == 0) + return false; + + return port_cap[3] & color_spc; + default: + return false; + } +} +EXPORT_SYMBOL(drm_dp_downstream_rgb_to_ycbcr_conversion); + /** * drm_dp_downstream_mode() - return a mode for downstream facing port * @dev: DRM device @@ -1204,7 +1236,7 @@ bool drm_dp_read_sink_count_cap(struct drm_connector *connector, return connector->connector_type != DRM_MODE_CONNECTOR_eDP && dpcd[DP_DPCD_REV] >= DP_DPCD_REV_11 && dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT && - !drm_dp_has_quirk(desc, 0, DP_DPCD_QUIRK_NO_SINK_COUNT); + !drm_dp_has_quirk(desc, DP_DPCD_QUIRK_NO_SINK_COUNT); } EXPORT_SYMBOL(drm_dp_read_sink_count_cap); @@ -1925,87 +1957,6 @@ drm_dp_get_quirks(const struct drm_dp_dpcd_ident *ident, bool is_branch) #undef DEVICE_ID_ANY #undef DEVICE_ID -struct edid_quirk { - u8 mfg_id[2]; - u8 prod_id[2]; - u32 quirks; -}; - -#define MFG(first, second) { (first), (second) } -#define PROD_ID(first, second) { (first), (second) } - -/* - * Some devices have unreliable OUIDs where they don't set the device ID - * correctly, and as a result we need to use the EDID for finding additional - * DP quirks in such cases. - */ -static const struct edid_quirk edid_quirk_list[] = { - /* Optional 4K AMOLED panel in the ThinkPad X1 Extreme 2nd Generation - * only supports DPCD backlight controls - */ - { MFG(0x4c, 0x83), PROD_ID(0x41, 0x41), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - /* - * Some Dell CML 2020 systems have panels support both AUX and PWM - * backlight control, and some only support AUX backlight control. All - * said panels start up in AUX mode by default, and we don't have any - * support for disabling HDR mode on these panels which would be - * required to switch to PWM backlight control mode (plus, I'm not - * even sure we want PWM backlight controls over DPCD backlight - * controls anyway...). Until we have a better way of detecting these, - * force DPCD backlight mode on all of them. - */ - { MFG(0x06, 0xaf), PROD_ID(0x9b, 0x32), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - { MFG(0x06, 0xaf), PROD_ID(0xeb, 0x41), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - { MFG(0x4d, 0x10), PROD_ID(0xc7, 0x14), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - { MFG(0x4d, 0x10), PROD_ID(0xe6, 0x14), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - { MFG(0x4c, 0x83), PROD_ID(0x47, 0x41), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - { MFG(0x09, 0xe5), PROD_ID(0xde, 0x08), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, -}; - -#undef MFG -#undef PROD_ID - -/** - * drm_dp_get_edid_quirks() - Check the EDID of a DP device to find additional - * DP-specific quirks - * @edid: The EDID to check - * - * While OUIDs are meant to be used to recognize a DisplayPort device, a lot - * of manufacturers don't seem to like following standards and neglect to fill - * the dev-ID in, making it impossible to only use OUIDs for determining - * quirks in some cases. This function can be used to check the EDID and look - * up any additional DP quirks. The bits returned by this function correspond - * to the quirk bits in &drm_dp_quirk. - * - * Returns: a bitmask of quirks, if any. The driver can check this using - * drm_dp_has_quirk(). - */ -u32 drm_dp_get_edid_quirks(const struct edid *edid) -{ - const struct edid_quirk *quirk; - u32 quirks = 0; - int i; - - if (!edid) - return 0; - - for (i = 0; i < ARRAY_SIZE(edid_quirk_list); i++) { - quirk = &edid_quirk_list[i]; - if (memcmp(quirk->mfg_id, edid->mfg_id, - sizeof(edid->mfg_id)) == 0 && - memcmp(quirk->prod_id, edid->prod_code, - sizeof(edid->prod_code)) == 0) - quirks |= quirk->quirks; - } - - DRM_DEBUG_KMS("DP sink: EDID mfg %*phD prod-ID %*phD quirks: 0x%04x\n", - (int)sizeof(edid->mfg_id), edid->mfg_id, - (int)sizeof(edid->prod_code), edid->prod_code, quirks); - - return quirks; -} -EXPORT_SYMBOL(drm_dp_get_edid_quirks); - /** * drm_dp_read_desc - read sink/branch descriptor from DPCD * @aux: DisplayPort AUX channel @@ -2596,3 +2547,538 @@ void drm_dp_vsc_sdp_log(const char *level, struct device *dev, #undef DP_SDP_LOG } EXPORT_SYMBOL(drm_dp_vsc_sdp_log); + +/** + * drm_dp_get_pcon_max_frl_bw() - maximum frl supported by PCON + * @dpcd: DisplayPort configuration data + * @port_cap: port capabilities + * + * Returns maximum frl bandwidth supported by PCON in GBPS, + * returns 0 if not supported. + */ +int drm_dp_get_pcon_max_frl_bw(const u8 dpcd[DP_RECEIVER_CAP_SIZE], + const u8 port_cap[4]) +{ + int bw; + u8 buf; + + buf = port_cap[2]; + bw = buf & DP_PCON_MAX_FRL_BW; + + switch (bw) { + case DP_PCON_MAX_9GBPS: + return 9; + case DP_PCON_MAX_18GBPS: + return 18; + case DP_PCON_MAX_24GBPS: + return 24; + case DP_PCON_MAX_32GBPS: + return 32; + case DP_PCON_MAX_40GBPS: + return 40; + case DP_PCON_MAX_48GBPS: + return 48; + case DP_PCON_MAX_0GBPS: + default: + return 0; + } + + return 0; +} +EXPORT_SYMBOL(drm_dp_get_pcon_max_frl_bw); + +/** + * drm_dp_pcon_frl_prepare() - Prepare PCON for FRL. + * @aux: DisplayPort AUX channel + * @enable_frl_ready_hpd: Configure DP_PCON_ENABLE_HPD_READY. + * + * Returns 0 if success, else returns negative error code. + */ +int drm_dp_pcon_frl_prepare(struct drm_dp_aux *aux, bool enable_frl_ready_hpd) +{ + int ret; + u8 buf = DP_PCON_ENABLE_SOURCE_CTL_MODE | + DP_PCON_ENABLE_LINK_FRL_MODE; + + if (enable_frl_ready_hpd) + buf |= DP_PCON_ENABLE_HPD_READY; + + ret = drm_dp_dpcd_writeb(aux, DP_PCON_HDMI_LINK_CONFIG_1, buf); + + return ret; +} +EXPORT_SYMBOL(drm_dp_pcon_frl_prepare); + +/** + * drm_dp_pcon_is_frl_ready() - Is PCON ready for FRL + * @aux: DisplayPort AUX channel + * + * Returns true if success, else returns false. + */ +bool drm_dp_pcon_is_frl_ready(struct drm_dp_aux *aux) +{ + int ret; + u8 buf; + + ret = drm_dp_dpcd_readb(aux, DP_PCON_HDMI_TX_LINK_STATUS, &buf); + if (ret < 0) + return false; + + if (buf & DP_PCON_FRL_READY) + return true; + + return false; +} +EXPORT_SYMBOL(drm_dp_pcon_is_frl_ready); + +/** + * drm_dp_pcon_frl_configure_1() - Set HDMI LINK Configuration-Step1 + * @aux: DisplayPort AUX channel + * @max_frl_gbps: maximum frl bw to be configured between PCON and HDMI sink + * @concurrent_mode: true if concurrent mode or operation is required, + * false otherwise. + * + * Returns 0 if success, else returns negative error code. + */ + +int drm_dp_pcon_frl_configure_1(struct drm_dp_aux *aux, int max_frl_gbps, + bool concurrent_mode) +{ + int ret; + u8 buf; + + ret = drm_dp_dpcd_readb(aux, DP_PCON_HDMI_LINK_CONFIG_1, &buf); + if (ret < 0) + return ret; + + if (concurrent_mode) + buf |= DP_PCON_ENABLE_CONCURRENT_LINK; + else + buf &= ~DP_PCON_ENABLE_CONCURRENT_LINK; + + switch (max_frl_gbps) { + case 9: + buf |= DP_PCON_ENABLE_MAX_BW_9GBPS; + break; + case 18: + buf |= DP_PCON_ENABLE_MAX_BW_18GBPS; + break; + case 24: + buf |= DP_PCON_ENABLE_MAX_BW_24GBPS; + break; + case 32: + buf |= DP_PCON_ENABLE_MAX_BW_32GBPS; + break; + case 40: + buf |= DP_PCON_ENABLE_MAX_BW_40GBPS; + break; + case 48: + buf |= DP_PCON_ENABLE_MAX_BW_48GBPS; + break; + case 0: + buf |= DP_PCON_ENABLE_MAX_BW_0GBPS; + break; + default: + return -EINVAL; + } + + ret = drm_dp_dpcd_writeb(aux, DP_PCON_HDMI_LINK_CONFIG_1, buf); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_frl_configure_1); + +/** + * drm_dp_pcon_frl_configure_2() - Set HDMI Link configuration Step-2 + * @aux: DisplayPort AUX channel + * @max_frl_mask : Max FRL BW to be tried by the PCON with HDMI Sink + * @extended_train_mode : true for Extended Mode, false for Normal Mode. + * In Normal mode, the PCON tries each frl bw from the max_frl_mask starting + * from min, and stops when link training is successful. In Extended mode, all + * frl bw selected in the mask are trained by the PCON. + * + * Returns 0 if success, else returns negative error code. + */ +int drm_dp_pcon_frl_configure_2(struct drm_dp_aux *aux, int max_frl_mask, + bool extended_train_mode) +{ + int ret; + u8 buf = max_frl_mask; + + if (extended_train_mode) + buf |= DP_PCON_FRL_LINK_TRAIN_EXTENDED; + + ret = drm_dp_dpcd_writeb(aux, DP_PCON_HDMI_LINK_CONFIG_2, buf); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_frl_configure_2); + +/** + * drm_dp_pcon_reset_frl_config() - Re-Set HDMI Link configuration. + * @aux: DisplayPort AUX channel + * + * Returns 0 if success, else returns negative error code. + */ +int drm_dp_pcon_reset_frl_config(struct drm_dp_aux *aux) +{ + int ret; + + ret = drm_dp_dpcd_writeb(aux, DP_PCON_HDMI_LINK_CONFIG_1, 0x0); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_reset_frl_config); + +/** + * drm_dp_pcon_frl_enable() - Enable HDMI link through FRL + * @aux: DisplayPort AUX channel + * + * Returns 0 if success, else returns negative error code. + */ +int drm_dp_pcon_frl_enable(struct drm_dp_aux *aux) +{ + int ret; + u8 buf = 0; + + ret = drm_dp_dpcd_readb(aux, DP_PCON_HDMI_LINK_CONFIG_1, &buf); + if (ret < 0) + return ret; + if (!(buf & DP_PCON_ENABLE_SOURCE_CTL_MODE)) { + DRM_DEBUG_KMS("PCON in Autonomous mode, can't enable FRL\n"); + return -EINVAL; + } + buf |= DP_PCON_ENABLE_HDMI_LINK; + ret = drm_dp_dpcd_writeb(aux, DP_PCON_HDMI_LINK_CONFIG_1, buf); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_frl_enable); + +/** + * drm_dp_pcon_hdmi_link_active() - check if the PCON HDMI LINK status is active. + * @aux: DisplayPort AUX channel + * + * Returns true if link is active else returns false. + */ +bool drm_dp_pcon_hdmi_link_active(struct drm_dp_aux *aux) +{ + u8 buf; + int ret; + + ret = drm_dp_dpcd_readb(aux, DP_PCON_HDMI_TX_LINK_STATUS, &buf); + if (ret < 0) + return false; + + return buf & DP_PCON_HDMI_TX_LINK_ACTIVE; +} +EXPORT_SYMBOL(drm_dp_pcon_hdmi_link_active); + +/** + * drm_dp_pcon_hdmi_link_mode() - get the PCON HDMI LINK MODE + * @aux: DisplayPort AUX channel + * @frl_trained_mask: pointer to store bitmask of the trained bw configuration. + * Valid only if the MODE returned is FRL. For Normal Link training mode + * only 1 of the bits will be set, but in case of Extended mode, more than + * one bits can be set. + * + * Returns the link mode : TMDS or FRL on success, else returns negative error + * code. + */ +int drm_dp_pcon_hdmi_link_mode(struct drm_dp_aux *aux, u8 *frl_trained_mask) +{ + u8 buf; + int mode; + int ret; + + ret = drm_dp_dpcd_readb(aux, DP_PCON_HDMI_POST_FRL_STATUS, &buf); + if (ret < 0) + return ret; + + mode = buf & DP_PCON_HDMI_LINK_MODE; + + if (frl_trained_mask && DP_PCON_HDMI_MODE_FRL == mode) + *frl_trained_mask = (buf & DP_PCON_HDMI_FRL_TRAINED_BW) >> 1; + + return mode; +} +EXPORT_SYMBOL(drm_dp_pcon_hdmi_link_mode); + +/** + * drm_dp_pcon_hdmi_frl_link_error_count() - print the error count per lane + * during link failure between PCON and HDMI sink + * @aux: DisplayPort AUX channel + * @connector: DRM connector + * code. + **/ + +void drm_dp_pcon_hdmi_frl_link_error_count(struct drm_dp_aux *aux, + struct drm_connector *connector) +{ + u8 buf, error_count; + int i, num_error; + struct drm_hdmi_info *hdmi = &connector->display_info.hdmi; + + for (i = 0; i < hdmi->max_lanes; i++) { + if (drm_dp_dpcd_readb(aux, DP_PCON_HDMI_ERROR_STATUS_LN0 + i, &buf) < 0) + return; + + error_count = buf & DP_PCON_HDMI_ERROR_COUNT_MASK; + switch (error_count) { + case DP_PCON_HDMI_ERROR_COUNT_HUNDRED_PLUS: + num_error = 100; + break; + case DP_PCON_HDMI_ERROR_COUNT_TEN_PLUS: + num_error = 10; + break; + case DP_PCON_HDMI_ERROR_COUNT_THREE_PLUS: + num_error = 3; + break; + default: + num_error = 0; + } + + DRM_ERROR("More than %d errors since the last read for lane %d", num_error, i); + } +} +EXPORT_SYMBOL(drm_dp_pcon_hdmi_frl_link_error_count); + +/* + * drm_dp_pcon_enc_is_dsc_1_2 - Does PCON Encoder supports DSC 1.2 + * @pcon_dsc_dpcd: DSC capabilities of the PCON DSC Encoder + * + * Returns true is PCON encoder is DSC 1.2 else returns false. + */ +bool drm_dp_pcon_enc_is_dsc_1_2(const u8 pcon_dsc_dpcd[DP_PCON_DSC_ENCODER_CAP_SIZE]) +{ + u8 buf; + u8 major_v, minor_v; + + buf = pcon_dsc_dpcd[DP_PCON_DSC_VERSION - DP_PCON_DSC_ENCODER]; + major_v = (buf & DP_PCON_DSC_MAJOR_MASK) >> DP_PCON_DSC_MAJOR_SHIFT; + minor_v = (buf & DP_PCON_DSC_MINOR_MASK) >> DP_PCON_DSC_MINOR_SHIFT; + + if (major_v == 1 && minor_v == 2) + return true; + + return false; +} +EXPORT_SYMBOL(drm_dp_pcon_enc_is_dsc_1_2); + +/* + * drm_dp_pcon_dsc_max_slices - Get max slices supported by PCON DSC Encoder + * @pcon_dsc_dpcd: DSC capabilities of the PCON DSC Encoder + * + * Returns maximum no. of slices supported by the PCON DSC Encoder. + */ +int drm_dp_pcon_dsc_max_slices(const u8 pcon_dsc_dpcd[DP_PCON_DSC_ENCODER_CAP_SIZE]) +{ + u8 slice_cap1, slice_cap2; + + slice_cap1 = pcon_dsc_dpcd[DP_PCON_DSC_SLICE_CAP_1 - DP_PCON_DSC_ENCODER]; + slice_cap2 = pcon_dsc_dpcd[DP_PCON_DSC_SLICE_CAP_2 - DP_PCON_DSC_ENCODER]; + + if (slice_cap2 & DP_PCON_DSC_24_PER_DSC_ENC) + return 24; + if (slice_cap2 & DP_PCON_DSC_20_PER_DSC_ENC) + return 20; + if (slice_cap2 & DP_PCON_DSC_16_PER_DSC_ENC) + return 16; + if (slice_cap1 & DP_PCON_DSC_12_PER_DSC_ENC) + return 12; + if (slice_cap1 & DP_PCON_DSC_10_PER_DSC_ENC) + return 10; + if (slice_cap1 & DP_PCON_DSC_8_PER_DSC_ENC) + return 8; + if (slice_cap1 & DP_PCON_DSC_6_PER_DSC_ENC) + return 6; + if (slice_cap1 & DP_PCON_DSC_4_PER_DSC_ENC) + return 4; + if (slice_cap1 & DP_PCON_DSC_2_PER_DSC_ENC) + return 2; + if (slice_cap1 & DP_PCON_DSC_1_PER_DSC_ENC) + return 1; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_dsc_max_slices); + +/* + * drm_dp_pcon_dsc_max_slice_width() - Get max slice width for Pcon DSC encoder + * @pcon_dsc_dpcd: DSC capabilities of the PCON DSC Encoder + * + * Returns maximum width of the slices in pixel width i.e. no. of pixels x 320. + */ +int drm_dp_pcon_dsc_max_slice_width(const u8 pcon_dsc_dpcd[DP_PCON_DSC_ENCODER_CAP_SIZE]) +{ + u8 buf; + + buf = pcon_dsc_dpcd[DP_PCON_DSC_MAX_SLICE_WIDTH - DP_PCON_DSC_ENCODER]; + + return buf * DP_DSC_SLICE_WIDTH_MULTIPLIER; +} +EXPORT_SYMBOL(drm_dp_pcon_dsc_max_slice_width); + +/* + * drm_dp_pcon_dsc_bpp_incr() - Get bits per pixel increment for PCON DSC encoder + * @pcon_dsc_dpcd: DSC capabilities of the PCON DSC Encoder + * + * Returns the bpp precision supported by the PCON encoder. + */ +int drm_dp_pcon_dsc_bpp_incr(const u8 pcon_dsc_dpcd[DP_PCON_DSC_ENCODER_CAP_SIZE]) +{ + u8 buf; + + buf = pcon_dsc_dpcd[DP_PCON_DSC_BPP_INCR - DP_PCON_DSC_ENCODER]; + + switch (buf & DP_PCON_DSC_BPP_INCR_MASK) { + case DP_PCON_DSC_ONE_16TH_BPP: + return 16; + case DP_PCON_DSC_ONE_8TH_BPP: + return 8; + case DP_PCON_DSC_ONE_4TH_BPP: + return 4; + case DP_PCON_DSC_ONE_HALF_BPP: + return 2; + case DP_PCON_DSC_ONE_BPP: + return 1; + } + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_dsc_bpp_incr); + +static +int drm_dp_pcon_configure_dsc_enc(struct drm_dp_aux *aux, u8 pps_buf_config) +{ + u8 buf; + int ret; + + ret = drm_dp_dpcd_readb(aux, DP_PROTOCOL_CONVERTER_CONTROL_2, &buf); + if (ret < 0) + return ret; + + buf |= DP_PCON_ENABLE_DSC_ENCODER; + + if (pps_buf_config <= DP_PCON_ENC_PPS_OVERRIDE_EN_BUFFER) { + buf &= ~DP_PCON_ENCODER_PPS_OVERRIDE_MASK; + buf |= pps_buf_config << 2; + } + + ret = drm_dp_dpcd_writeb(aux, DP_PROTOCOL_CONVERTER_CONTROL_2, buf); + if (ret < 0) + return ret; + + return 0; +} + +/** + * drm_dp_pcon_pps_default() - Let PCON fill the default pps parameters + * for DSC1.2 between PCON & HDMI2.1 sink + * @aux: DisplayPort AUX channel + * + * Returns 0 on success, else returns negative error code. + */ +int drm_dp_pcon_pps_default(struct drm_dp_aux *aux) +{ + int ret; + + ret = drm_dp_pcon_configure_dsc_enc(aux, DP_PCON_ENC_PPS_OVERRIDE_DISABLED); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_pps_default); + +/** + * drm_dp_pcon_pps_override_buf() - Configure PPS encoder override buffer for + * HDMI sink + * @aux: DisplayPort AUX channel + * @pps_buf: 128 bytes to be written into PPS buffer for HDMI sink by PCON. + * + * Returns 0 on success, else returns negative error code. + */ +int drm_dp_pcon_pps_override_buf(struct drm_dp_aux *aux, u8 pps_buf[128]) +{ + int ret; + + ret = drm_dp_dpcd_write(aux, DP_PCON_HDMI_PPS_OVERRIDE_BASE, &pps_buf, 128); + if (ret < 0) + return ret; + + ret = drm_dp_pcon_configure_dsc_enc(aux, DP_PCON_ENC_PPS_OVERRIDE_EN_BUFFER); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_pps_override_buf); + +/* + * drm_dp_pcon_pps_override_param() - Write PPS parameters to DSC encoder + * override registers + * @aux: DisplayPort AUX channel + * @pps_param: 3 Parameters (2 Bytes each) : Slice Width, Slice Height, + * bits_per_pixel. + * + * Returns 0 on success, else returns negative error code. + */ +int drm_dp_pcon_pps_override_param(struct drm_dp_aux *aux, u8 pps_param[6]) +{ + int ret; + + ret = drm_dp_dpcd_write(aux, DP_PCON_HDMI_PPS_OVRD_SLICE_HEIGHT, &pps_param[0], 2); + if (ret < 0) + return ret; + ret = drm_dp_dpcd_write(aux, DP_PCON_HDMI_PPS_OVRD_SLICE_WIDTH, &pps_param[2], 2); + if (ret < 0) + return ret; + ret = drm_dp_dpcd_write(aux, DP_PCON_HDMI_PPS_OVRD_BPP, &pps_param[4], 2); + if (ret < 0) + return ret; + + ret = drm_dp_pcon_configure_dsc_enc(aux, DP_PCON_ENC_PPS_OVERRIDE_EN_BUFFER); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_pps_override_param); + +/* + * drm_dp_pcon_convert_rgb_to_ycbcr() - Configure the PCon to convert RGB to Ycbcr + * @aux: displayPort AUX channel + * @color_spc: Color-space/s for which conversion is to be enabled, 0 for disable. + * + * Returns 0 on success, else returns negative error code. + */ +int drm_dp_pcon_convert_rgb_to_ycbcr(struct drm_dp_aux *aux, u8 color_spc) +{ + int ret; + u8 buf; + + ret = drm_dp_dpcd_readb(aux, DP_PROTOCOL_CONVERTER_CONTROL_2, &buf); + if (ret < 0) + return ret; + + if (color_spc & DP_CONVERSION_RGB_YCBCR_MASK) + buf |= (color_spc & DP_CONVERSION_RGB_YCBCR_MASK); + else + buf &= ~DP_CONVERSION_RGB_YCBCR_MASK; + + ret = drm_dp_dpcd_writeb(aux, DP_PROTOCOL_CONVERTER_CONTROL_2, buf); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_convert_rgb_to_ycbcr); diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c index b11c0522a441..309afe61afdd 100644 --- a/drivers/gpu/drm/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/drm_dp_mst_topology.c @@ -2302,7 +2302,8 @@ drm_dp_mst_port_add_connector(struct drm_dp_mst_branch *mstb, } if (port->pdt != DP_PEER_DEVICE_NONE && - drm_dp_mst_is_end_device(port->pdt, port->mcs)) { + drm_dp_mst_is_end_device(port->pdt, port->mcs) && + port->port_num >= DP_MST_LOGICAL_PORT_0) { port->cached_edid = drm_get_edid(port->connector, &port->aux.ddc); drm_connector_set_tile_property(port->connector); @@ -2751,7 +2752,7 @@ static void drm_dp_mst_link_probe_work(struct work_struct *work) drm_dp_mst_topology_put_mstb(mstb); mutex_unlock(&mgr->probe_lock); - if (ret) + if (ret > 0) drm_kms_helper_hotplug_event(dev); } @@ -5837,8 +5838,7 @@ struct drm_dp_aux *drm_dp_mst_dsc_aux_for_port(struct drm_dp_mst_port *port) if (drm_dp_read_desc(port->mgr->aux, &desc, true)) return NULL; - if (drm_dp_has_quirk(&desc, 0, - DP_DPCD_QUIRK_DSC_WITHOUT_VIRTUAL_DPCD) && + if (drm_dp_has_quirk(&desc, DP_DPCD_QUIRK_DSC_WITHOUT_VIRTUAL_DPCD) && port->mgr->dpcd[DP_DPCD_REV] >= DP_DPCD_REV_14 && port->parent == port->mgr->mst_primary) { u8 downstreamport; diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c index 734303802bc3..20d22e41d7ce 100644 --- a/drivers/gpu/drm/drm_drv.c +++ b/drivers/gpu/drm/drm_drv.c @@ -469,6 +469,9 @@ void drm_dev_unplug(struct drm_device *dev) synchronize_srcu(&drm_unplug_srcu); drm_dev_unregister(dev); + + /* Clear all CPU mappings pointing to this device */ + unmap_mapping_range(dev->anon_inode->i_mapping, 0, 0, 1); } EXPORT_SYMBOL(drm_dev_unplug); @@ -589,11 +592,7 @@ static int drm_dev_init(struct drm_device *dev, kref_init(&dev->ref); dev->dev = get_device(parent); -#ifdef CONFIG_DRM_LEGACY - dev->driver = (struct drm_driver *)driver; -#else dev->driver = driver; -#endif INIT_LIST_HEAD(&dev->managed.resources); spin_lock_init(&dev->managed.lock); @@ -675,11 +674,8 @@ static int devm_drm_dev_init(struct device *parent, if (ret) return ret; - ret = devm_add_action(parent, devm_drm_dev_init_release, dev); - if (ret) - devm_drm_dev_init_release(dev); - - return ret; + return devm_add_action_or_reset(parent, + devm_drm_dev_init_release, dev); } void *__devm_drm_dev_alloc(struct device *parent, @@ -897,8 +893,6 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags) if (drm_core_check_feature(dev, DRIVER_MODESET)) drm_modeset_register_all(dev); - ret = 0; - DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n", driver->name, driver->major, driver->minor, driver->patchlevel, driver->date, diff --git a/drivers/gpu/drm/drm_dsc.c b/drivers/gpu/drm/drm_dsc.c index 4a475d9696ff..ff602f7ec65b 100644 --- a/drivers/gpu/drm/drm_dsc.c +++ b/drivers/gpu/drm/drm_dsc.c @@ -49,6 +49,33 @@ void drm_dsc_dp_pps_header_init(struct dp_sdp_header *pps_header) } EXPORT_SYMBOL(drm_dsc_dp_pps_header_init); +/** + * drm_dsc_dp_rc_buffer_size - get rc buffer size in bytes + * @rc_buffer_block_size: block size code, according to DPCD offset 62h + * @rc_buffer_size: number of blocks - 1, according to DPCD offset 63h + * + * return: + * buffer size in bytes, or 0 on invalid input + */ +int drm_dsc_dp_rc_buffer_size(u8 rc_buffer_block_size, u8 rc_buffer_size) +{ + int size = 1024 * (rc_buffer_size + 1); + + switch (rc_buffer_block_size) { + case DP_DSC_RC_BUF_BLK_SIZE_1: + return 1 * size; + case DP_DSC_RC_BUF_BLK_SIZE_4: + return 4 * size; + case DP_DSC_RC_BUF_BLK_SIZE_16: + return 16 * size; + case DP_DSC_RC_BUF_BLK_SIZE_64: + return 64 * size; + default: + return 0; + } +} +EXPORT_SYMBOL(drm_dsc_dp_rc_buffer_size); + /** * drm_dsc_pps_payload_pack() - Populates the DSC PPS * @@ -186,8 +213,7 @@ void drm_dsc_pps_payload_pack(struct drm_dsc_picture_parameter_set *pps_payload, pps_payload->flatness_max_qp = dsc_cfg->flatness_max_qp; /* PPS 38, 39 */ - pps_payload->rc_model_size = - cpu_to_be16(DSC_RC_MODEL_SIZE_CONST); + pps_payload->rc_model_size = cpu_to_be16(dsc_cfg->rc_model_size); /* PPS 40 */ pps_payload->rc_edge_factor = DSC_RC_EDGE_FACTOR_CONST; diff --git a/drivers/gpu/drm/drm_dumb_buffers.c b/drivers/gpu/drm/drm_dumb_buffers.c index d18a740fe0f1..ad17fa21cebb 100644 --- a/drivers/gpu/drm/drm_dumb_buffers.c +++ b/drivers/gpu/drm/drm_dumb_buffers.c @@ -29,6 +29,7 @@ #include #include "drm_crtc_internal.h" +#include "drm_internal.h" /** * DOC: overview @@ -46,9 +47,10 @@ * KMS frame buffers. * * To support dumb objects drivers must implement the &drm_driver.dumb_create - * operation. &drm_driver.dumb_destroy defaults to drm_gem_dumb_destroy() if - * not set and &drm_driver.dumb_map_offset defaults to - * drm_gem_dumb_map_offset(). See the callbacks for further details. + * and &drm_driver.dumb_map_offset operations (the latter defaults to + * drm_gem_dumb_map_offset() if not set). Drivers that don't use GEM handles + * additionally need to implement the &drm_driver.dumb_destroy operation. See + * the callbacks for further details. * * Note that dumb objects may not be used for gpu acceleration, as has been * attempted on some ARM embedded platforms. Such drivers really must have diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c index e95cce8e736d..c2bbe7bee7b6 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -2075,9 +2076,13 @@ EXPORT_SYMBOL(drm_get_edid); struct edid *drm_get_edid_switcheroo(struct drm_connector *connector, struct i2c_adapter *adapter) { - struct pci_dev *pdev = connector->dev->pdev; + struct drm_device *dev = connector->dev; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct edid *edid; + if (drm_WARN_ON_ONCE(dev, !dev_is_pci(dev->dev))) + return NULL; + vga_switcheroo_lock_ddc(pdev); edid = drm_get_edid(connector, adapter); vga_switcheroo_unlock_ddc(pdev); @@ -4851,6 +4856,41 @@ static void drm_parse_vcdb(struct drm_connector *connector, const u8 *db) info->rgb_quant_range_selectable = true; } +static +void drm_get_max_frl_rate(int max_frl_rate, u8 *max_lanes, u8 *max_rate_per_lane) +{ + switch (max_frl_rate) { + case 1: + *max_lanes = 3; + *max_rate_per_lane = 3; + break; + case 2: + *max_lanes = 3; + *max_rate_per_lane = 6; + break; + case 3: + *max_lanes = 4; + *max_rate_per_lane = 6; + break; + case 4: + *max_lanes = 4; + *max_rate_per_lane = 8; + break; + case 5: + *max_lanes = 4; + *max_rate_per_lane = 10; + break; + case 6: + *max_lanes = 4; + *max_rate_per_lane = 12; + break; + case 0: + default: + *max_lanes = 0; + *max_rate_per_lane = 0; + } +} + static void drm_parse_ycbcr420_deep_color_info(struct drm_connector *connector, const u8 *db) { @@ -4904,6 +4944,74 @@ static void drm_parse_hdmi_forum_vsdb(struct drm_connector *connector, } } + if (hf_vsdb[7]) { + u8 max_frl_rate; + u8 dsc_max_frl_rate; + u8 dsc_max_slices; + struct drm_hdmi_dsc_cap *hdmi_dsc = &hdmi->dsc_cap; + + DRM_DEBUG_KMS("hdmi_21 sink detected. parsing edid\n"); + max_frl_rate = (hf_vsdb[7] & DRM_EDID_MAX_FRL_RATE_MASK) >> 4; + drm_get_max_frl_rate(max_frl_rate, &hdmi->max_lanes, + &hdmi->max_frl_rate_per_lane); + hdmi_dsc->v_1p2 = hf_vsdb[11] & DRM_EDID_DSC_1P2; + + if (hdmi_dsc->v_1p2) { + hdmi_dsc->native_420 = hf_vsdb[11] & DRM_EDID_DSC_NATIVE_420; + hdmi_dsc->all_bpp = hf_vsdb[11] & DRM_EDID_DSC_ALL_BPP; + + if (hf_vsdb[11] & DRM_EDID_DSC_16BPC) + hdmi_dsc->bpc_supported = 16; + else if (hf_vsdb[11] & DRM_EDID_DSC_12BPC) + hdmi_dsc->bpc_supported = 12; + else if (hf_vsdb[11] & DRM_EDID_DSC_10BPC) + hdmi_dsc->bpc_supported = 10; + else + hdmi_dsc->bpc_supported = 0; + + dsc_max_frl_rate = (hf_vsdb[12] & DRM_EDID_DSC_MAX_FRL_RATE_MASK) >> 4; + drm_get_max_frl_rate(dsc_max_frl_rate, &hdmi_dsc->max_lanes, + &hdmi_dsc->max_frl_rate_per_lane); + hdmi_dsc->total_chunk_kbytes = hf_vsdb[13] & DRM_EDID_DSC_TOTAL_CHUNK_KBYTES; + + dsc_max_slices = hf_vsdb[12] & DRM_EDID_DSC_MAX_SLICES; + switch (dsc_max_slices) { + case 1: + hdmi_dsc->max_slices = 1; + hdmi_dsc->clk_per_slice = 340; + break; + case 2: + hdmi_dsc->max_slices = 2; + hdmi_dsc->clk_per_slice = 340; + break; + case 3: + hdmi_dsc->max_slices = 4; + hdmi_dsc->clk_per_slice = 340; + break; + case 4: + hdmi_dsc->max_slices = 8; + hdmi_dsc->clk_per_slice = 340; + break; + case 5: + hdmi_dsc->max_slices = 8; + hdmi_dsc->clk_per_slice = 400; + break; + case 6: + hdmi_dsc->max_slices = 12; + hdmi_dsc->clk_per_slice = 400; + break; + case 7: + hdmi_dsc->max_slices = 16; + hdmi_dsc->clk_per_slice = 400; + break; + case 0: + default: + hdmi_dsc->max_slices = 0; + hdmi_dsc->clk_per_slice = 0; + } + } + } + drm_parse_ycbcr420_deep_color_info(connector, hf_vsdb); } diff --git a/drivers/gpu/drm/drm_encoder.c b/drivers/gpu/drm/drm_encoder.c index e555281f43d4..72e982323a5e 100644 --- a/drivers/gpu/drm/drm_encoder.c +++ b/drivers/gpu/drm/drm_encoder.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "drm_crtc_internal.h" @@ -72,7 +73,7 @@ int drm_encoder_register_all(struct drm_device *dev) int ret = 0; drm_for_each_encoder(encoder, dev) { - if (encoder->funcs->late_register) + if (encoder->funcs && encoder->funcs->late_register) ret = encoder->funcs->late_register(encoder); if (ret) return ret; @@ -86,30 +87,16 @@ void drm_encoder_unregister_all(struct drm_device *dev) struct drm_encoder *encoder; drm_for_each_encoder(encoder, dev) { - if (encoder->funcs->early_unregister) + if (encoder->funcs && encoder->funcs->early_unregister) encoder->funcs->early_unregister(encoder); } } -/** - * drm_encoder_init - Init a preallocated encoder - * @dev: drm device - * @encoder: the encoder to init - * @funcs: callbacks for this encoder - * @encoder_type: user visible type of the encoder - * @name: printf style format string for the encoder name, or NULL for default name - * - * Initialises a preallocated encoder. Encoder should be subclassed as part of - * driver encoder objects. At driver unload time drm_encoder_cleanup() should be - * called from the driver's &drm_encoder_funcs.destroy hook. - * - * Returns: - * Zero on success, error code on failure. - */ -int drm_encoder_init(struct drm_device *dev, - struct drm_encoder *encoder, - const struct drm_encoder_funcs *funcs, - int encoder_type, const char *name, ...) +__printf(5, 0) +static int __drm_encoder_init(struct drm_device *dev, + struct drm_encoder *encoder, + const struct drm_encoder_funcs *funcs, + int encoder_type, const char *name, va_list ap) { int ret; @@ -125,11 +112,7 @@ int drm_encoder_init(struct drm_device *dev, encoder->encoder_type = encoder_type; encoder->funcs = funcs; if (name) { - va_list ap; - - va_start(ap, name); encoder->name = kvasprintf(GFP_KERNEL, name, ap); - va_end(ap); } else { encoder->name = kasprintf(GFP_KERNEL, "%s-%d", drm_encoder_enum_list[encoder_type].name, @@ -150,6 +133,44 @@ out_put: return ret; } + +/** + * drm_encoder_init - Init a preallocated encoder + * @dev: drm device + * @encoder: the encoder to init + * @funcs: callbacks for this encoder + * @encoder_type: user visible type of the encoder + * @name: printf style format string for the encoder name, or NULL for default name + * + * Initializes a preallocated encoder. Encoder should be subclassed as part of + * driver encoder objects. At driver unload time the driver's + * &drm_encoder_funcs.destroy hook should call drm_encoder_cleanup() and kfree() + * the encoder structure. The encoder structure should not be allocated with + * devm_kzalloc(). + * + * Note: consider using drmm_encoder_alloc() instead of drm_encoder_init() to + * let the DRM managed resource infrastructure take care of cleanup and + * deallocation. + * + * Returns: + * Zero on success, error code on failure. + */ +int drm_encoder_init(struct drm_device *dev, + struct drm_encoder *encoder, + const struct drm_encoder_funcs *funcs, + int encoder_type, const char *name, ...) +{ + va_list ap; + int ret; + + WARN_ON(!funcs->destroy); + + va_start(ap, name); + ret = __drm_encoder_init(dev, encoder, funcs, encoder_type, name, ap); + va_end(ap); + + return ret; +} EXPORT_SYMBOL(drm_encoder_init); /** @@ -181,6 +202,48 @@ void drm_encoder_cleanup(struct drm_encoder *encoder) } EXPORT_SYMBOL(drm_encoder_cleanup); +static void drmm_encoder_alloc_release(struct drm_device *dev, void *ptr) +{ + struct drm_encoder *encoder = ptr; + + if (WARN_ON(!encoder->dev)) + return; + + drm_encoder_cleanup(encoder); +} + +void *__drmm_encoder_alloc(struct drm_device *dev, size_t size, size_t offset, + const struct drm_encoder_funcs *funcs, + int encoder_type, const char *name, ...) +{ + void *container; + struct drm_encoder *encoder; + va_list ap; + int ret; + + if (WARN_ON(funcs && funcs->destroy)) + return ERR_PTR(-EINVAL); + + container = drmm_kzalloc(dev, size, GFP_KERNEL); + if (!container) + return ERR_PTR(-EINVAL); + + encoder = container + offset; + + va_start(ap, name); + ret = __drm_encoder_init(dev, encoder, funcs, encoder_type, name, ap); + va_end(ap); + if (ret) + return ERR_PTR(ret); + + ret = drmm_add_action_or_reset(dev, drmm_encoder_alloc_release, encoder); + if (ret) + return ERR_PTR(ret); + + return container; +} +EXPORT_SYMBOL(__drmm_encoder_alloc); + static struct drm_crtc *drm_encoder_get_crtc(struct drm_encoder *encoder) { struct drm_connector *connector; diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 4b8119510687..b9a616737c0e 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -946,11 +946,15 @@ static int setcmap_legacy(struct fb_cmap *cmap, struct fb_info *info) drm_modeset_lock_all(fb_helper->dev); drm_client_for_each_modeset(modeset, &fb_helper->client) { crtc = modeset->crtc; - if (!crtc->funcs->gamma_set || !crtc->gamma_size) - return -EINVAL; + if (!crtc->funcs->gamma_set || !crtc->gamma_size) { + ret = -EINVAL; + goto out; + } - if (cmap->start + cmap->len > crtc->gamma_size) - return -EINVAL; + if (cmap->start + cmap->len > crtc->gamma_size) { + ret = -EINVAL; + goto out; + } r = crtc->gamma_store; g = r + crtc->gamma_size; @@ -963,8 +967,9 @@ static int setcmap_legacy(struct fb_cmap *cmap, struct fb_info *info) ret = crtc->funcs->gamma_set(crtc, r, g, b, crtc->gamma_size, NULL); if (ret) - return ret; + goto out; } +out: drm_modeset_unlock_all(fb_helper->dev); return ret; @@ -1054,6 +1059,11 @@ retry: goto out_state; } + /* + * FIXME: This always uses gamma_lut. Some HW have only + * degamma_lut, in which case we should reset gamma_lut and set + * degamma_lut. See drm_crtc_legacy_gamma_set(). + */ replaced = drm_property_replace_blob(&crtc_state->degamma_lut, NULL); replaced |= drm_property_replace_blob(&crtc_state->ctm, NULL); @@ -2486,6 +2496,11 @@ void drm_fbdev_generic_setup(struct drm_device *dev, return; } + /* + * FIXME: This mixes up depth with bpp, which results in a glorious + * mess, resulting in some drivers picking wrong fbdev defaults and + * others wrong preferred_depth defaults. + */ if (!preferred_bpp) preferred_bpp = dev->mode_config.preferred_depth; if (!preferred_bpp) @@ -2499,24 +2514,3 @@ void drm_fbdev_generic_setup(struct drm_device *dev, drm_client_register(&fb_helper->client); } EXPORT_SYMBOL(drm_fbdev_generic_setup); - -/* The Kconfig DRM_KMS_HELPER selects FRAMEBUFFER_CONSOLE (if !EXPERT) - * but the module doesn't depend on any fb console symbols. At least - * attempt to load fbcon to avoid leaving the system without a usable console. - */ -int __init drm_fb_helper_modinit(void) -{ -#if defined(CONFIG_FRAMEBUFFER_CONSOLE_MODULE) && !defined(CONFIG_EXPERT) - const char name[] = "fbcon"; - struct module *fbcon; - - mutex_lock(&module_mutex); - fbcon = find_module(name); - mutex_unlock(&module_mutex); - - if (!fbcon) - request_module_nowait(name); -#endif - return 0; -} -EXPORT_SYMBOL(drm_fb_helper_modinit); diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c index b50380fa80ce..7efbccffc2ea 100644 --- a/drivers/gpu/drm/drm_file.c +++ b/drivers/gpu/drm/drm_file.c @@ -113,8 +113,7 @@ bool drm_dev_needs_global_mutex(struct drm_device *dev) * The memory mapping implementation will vary depending on how the driver * manages memory. Legacy drivers will use the deprecated drm_legacy_mmap() * function, modern drivers should use one of the provided memory-manager - * specific implementations. For GEM-based drivers this is drm_gem_mmap(), and - * for drivers which use the CMA GEM helpers it's drm_gem_cma_mmap(). + * specific implementations. For GEM-based drivers this is drm_gem_mmap(). * * No other file operations are supported by the DRM userspace API. Overall the * following is an example &file_operations structure:: @@ -240,9 +239,6 @@ static void drm_events_release(struct drm_file *file_priv) * before calling this. * * If NULL is passed, this is a no-op. - * - * RETURNS: - * 0 on success, or error code on failure. */ void drm_file_free(struct drm_file *file) { @@ -371,6 +367,7 @@ static int drm_open_helper(struct file *filp, struct drm_minor *minor) list_add(&priv->lhead, &dev->filelist); mutex_unlock(&dev->filelist_mutex); +#ifdef CONFIG_DRM_LEGACY #ifdef __alpha__ /* * Default the hose @@ -390,6 +387,7 @@ static int drm_open_helper(struct file *filp, struct drm_minor *minor) dev->hose = b->sysdata; } } +#endif #endif return 0; @@ -776,6 +774,72 @@ void drm_event_cancel_free(struct drm_device *dev, } EXPORT_SYMBOL(drm_event_cancel_free); +/** + * drm_send_event_helper - send DRM event to file descriptor + * @dev: DRM device + * @e: DRM event to deliver + * @timestamp: timestamp to set for the fence event in kernel's CLOCK_MONOTONIC + * time domain + * + * This helper function sends the event @e, initialized with + * drm_event_reserve_init(), to its associated userspace DRM file. + * The timestamp variant of dma_fence_signal is used when the caller + * sends a valid timestamp. + */ +void drm_send_event_helper(struct drm_device *dev, + struct drm_pending_event *e, ktime_t timestamp) +{ + assert_spin_locked(&dev->event_lock); + + if (e->completion) { + complete_all(e->completion); + e->completion_release(e->completion); + e->completion = NULL; + } + + if (e->fence) { + if (timestamp) + dma_fence_signal_timestamp(e->fence, timestamp); + else + dma_fence_signal(e->fence); + dma_fence_put(e->fence); + } + + if (!e->file_priv) { + kfree(e); + return; + } + + list_del(&e->pending_link); + list_add_tail(&e->link, + &e->file_priv->event_list); + wake_up_interruptible_poll(&e->file_priv->event_wait, + EPOLLIN | EPOLLRDNORM); +} + +/** + * drm_send_event_timestamp_locked - send DRM event to file descriptor + * @dev: DRM device + * @e: DRM event to deliver + * @timestamp: timestamp to set for the fence event in kernel's CLOCK_MONOTONIC + * time domain + * + * This function sends the event @e, initialized with drm_event_reserve_init(), + * to its associated userspace DRM file. Callers must already hold + * &drm_device.event_lock. + * + * Note that the core will take care of unlinking and disarming events when the + * corresponding DRM file is closed. Drivers need not worry about whether the + * DRM file for this event still exists and can call this function upon + * completion of the asynchronous work unconditionally. + */ +void drm_send_event_timestamp_locked(struct drm_device *dev, + struct drm_pending_event *e, ktime_t timestamp) +{ + drm_send_event_helper(dev, e, timestamp); +} +EXPORT_SYMBOL(drm_send_event_timestamp_locked); + /** * drm_send_event_locked - send DRM event to file descriptor * @dev: DRM device @@ -792,29 +856,7 @@ EXPORT_SYMBOL(drm_event_cancel_free); */ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) { - assert_spin_locked(&dev->event_lock); - - if (e->completion) { - complete_all(e->completion); - e->completion_release(e->completion); - e->completion = NULL; - } - - if (e->fence) { - dma_fence_signal(e->fence); - dma_fence_put(e->fence); - } - - if (!e->file_priv) { - kfree(e); - return; - } - - list_del(&e->pending_link); - list_add_tail(&e->link, - &e->file_priv->event_list); - wake_up_interruptible_poll(&e->file_priv->event_wait, - EPOLLIN | EPOLLRDNORM); + drm_send_event_helper(dev, e, 0); } EXPORT_SYMBOL(drm_send_event_locked); @@ -838,7 +880,7 @@ void drm_send_event(struct drm_device *dev, struct drm_pending_event *e) unsigned long irqflags; spin_lock_irqsave(&dev->event_lock, irqflags); - drm_send_event_locked(dev, e); + drm_send_event_helper(dev, e, 0); spin_unlock_irqrestore(&dev->event_lock, irqflags); } EXPORT_SYMBOL(drm_send_event); diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 92f89cee213e..c2ce78c4edc3 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -335,22 +335,12 @@ out: } EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset); -/** - * drm_gem_dumb_destroy - dumb fb callback helper for gem based drivers - * @file: drm file-private structure to remove the dumb handle from - * @dev: corresponding drm_device - * @handle: the dumb handle to remove - * - * This implements the &drm_driver.dumb_destroy kms driver callback for drivers - * which use gem to manage their backing storage. - */ int drm_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, - uint32_t handle) + u32 handle) { return drm_gem_handle_delete(file, handle); } -EXPORT_SYMBOL(drm_gem_dumb_destroy); /** * drm_gem_handle_create_tail - internal functions to create a handle @@ -1078,20 +1068,17 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, drm_gem_object_get(obj); vma->vm_private_data = obj; + vma->vm_ops = obj->funcs->vm_ops; if (obj->funcs->mmap) { ret = obj->funcs->mmap(obj, vma); - if (ret) { - drm_gem_object_put(obj); - return ret; - } + if (ret) + goto err_drm_gem_object_put; WARN_ON(!(vma->vm_flags & VM_DONTEXPAND)); } else { - if (obj->funcs->vm_ops) - vma->vm_ops = obj->funcs->vm_ops; - else { - drm_gem_object_put(obj); - return -EINVAL; + if (!vma->vm_ops) { + ret = -EINVAL; + goto err_drm_gem_object_put; } vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; @@ -1100,6 +1087,10 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, } return 0; + +err_drm_gem_object_put: + drm_gem_object_put(obj); + return ret; } EXPORT_SYMBOL(drm_gem_mmap_obj); diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c index 4d5c1d86b022..7942cf05cd93 100644 --- a/drivers/gpu/drm/drm_gem_cma_helper.c +++ b/drivers/gpu/drm/drm_gem_cma_helper.c @@ -36,8 +36,9 @@ static const struct drm_gem_object_funcs drm_gem_cma_default_funcs = { .free = drm_gem_cma_free_object, .print_info = drm_gem_cma_print_info, - .get_sg_table = drm_gem_cma_prime_get_sg_table, - .vmap = drm_gem_cma_prime_vmap, + .get_sg_table = drm_gem_cma_get_sg_table, + .vmap = drm_gem_cma_vmap, + .mmap = drm_gem_cma_mmap, .vm_ops = &drm_gem_cma_vm_ops, }; @@ -277,62 +278,6 @@ const struct vm_operations_struct drm_gem_cma_vm_ops = { }; EXPORT_SYMBOL_GPL(drm_gem_cma_vm_ops); -static int drm_gem_cma_mmap_obj(struct drm_gem_cma_object *cma_obj, - struct vm_area_struct *vma) -{ - int ret; - - /* - * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the - * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map - * the whole buffer. - */ - vma->vm_flags &= ~VM_PFNMAP; - vma->vm_pgoff = 0; - - ret = dma_mmap_wc(cma_obj->base.dev->dev, vma, cma_obj->vaddr, - cma_obj->paddr, vma->vm_end - vma->vm_start); - if (ret) - drm_gem_vm_close(vma); - - return ret; -} - -/** - * drm_gem_cma_mmap - memory-map a CMA GEM object - * @filp: file object - * @vma: VMA for the area to be mapped - * - * This function implements an augmented version of the GEM DRM file mmap - * operation for CMA objects: In addition to the usual GEM VMA setup it - * immediately faults in the entire object instead of using on-demaind - * faulting. Drivers which employ the CMA helpers should use this function - * as their ->mmap() handler in the DRM device file's file_operations - * structure. - * - * Instead of directly referencing this function, drivers should use the - * DEFINE_DRM_GEM_CMA_FOPS().macro. - * - * Returns: - * 0 on success or a negative error code on failure. - */ -int drm_gem_cma_mmap(struct file *filp, struct vm_area_struct *vma) -{ - struct drm_gem_cma_object *cma_obj; - struct drm_gem_object *gem_obj; - int ret; - - ret = drm_gem_mmap(filp, vma); - if (ret) - return ret; - - gem_obj = vma->vm_private_data; - cma_obj = to_drm_gem_cma_obj(gem_obj); - - return drm_gem_cma_mmap_obj(cma_obj, vma); -} -EXPORT_SYMBOL_GPL(drm_gem_cma_mmap); - #ifndef CONFIG_MMU /** * drm_gem_cma_get_unmapped_area - propose address for mapping in noMMU cases @@ -424,18 +369,18 @@ void drm_gem_cma_print_info(struct drm_printer *p, unsigned int indent, EXPORT_SYMBOL(drm_gem_cma_print_info); /** - * drm_gem_cma_prime_get_sg_table - provide a scatter/gather table of pinned + * drm_gem_cma_get_sg_table - provide a scatter/gather table of pinned * pages for a CMA GEM object * @obj: GEM object * - * This function exports a scatter/gather table suitable for PRIME usage by + * This function exports a scatter/gather table by * calling the standard DMA mapping API. Drivers using the CMA helpers should * set this as their &drm_gem_object_funcs.get_sg_table callback. * * Returns: * A pointer to the scatter/gather table of pinned pages or NULL on failure. */ -struct sg_table *drm_gem_cma_prime_get_sg_table(struct drm_gem_object *obj) +struct sg_table *drm_gem_cma_get_sg_table(struct drm_gem_object *obj) { struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj); struct sg_table *sgt; @@ -456,7 +401,7 @@ out: kfree(sgt); return ERR_PTR(ret); } -EXPORT_SYMBOL_GPL(drm_gem_cma_prime_get_sg_table); +EXPORT_SYMBOL_GPL(drm_gem_cma_get_sg_table); /** * drm_gem_cma_prime_import_sg_table - produce a CMA GEM object from another @@ -501,40 +446,13 @@ drm_gem_cma_prime_import_sg_table(struct drm_device *dev, EXPORT_SYMBOL_GPL(drm_gem_cma_prime_import_sg_table); /** - * drm_gem_cma_prime_mmap - memory-map an exported CMA GEM object - * @obj: GEM object - * @vma: VMA for the area to be mapped - * - * This function maps a buffer imported via DRM PRIME into a userspace - * process's address space. Drivers that use the CMA helpers should set this - * as their &drm_driver.gem_prime_mmap callback. - * - * Returns: - * 0 on success or a negative error code on failure. - */ -int drm_gem_cma_prime_mmap(struct drm_gem_object *obj, - struct vm_area_struct *vma) -{ - struct drm_gem_cma_object *cma_obj; - int ret; - - ret = drm_gem_mmap_obj(obj, obj->size, vma); - if (ret < 0) - return ret; - - cma_obj = to_drm_gem_cma_obj(obj); - return drm_gem_cma_mmap_obj(cma_obj, vma); -} -EXPORT_SYMBOL_GPL(drm_gem_cma_prime_mmap); - -/** - * drm_gem_cma_prime_vmap - map a CMA GEM object into the kernel's virtual + * drm_gem_cma_vmap - map a CMA GEM object into the kernel's virtual * address space * @obj: GEM object * @map: Returns the kernel virtual address of the CMA GEM object's backing * store. * - * This function maps a buffer exported via DRM PRIME into the kernel's + * This function maps a buffer into the kernel's * virtual address space. Since the CMA buffers are already mapped into the * kernel virtual address space this simply returns the cached virtual * address. Drivers using the CMA helpers should set this as their DRM @@ -543,7 +461,7 @@ EXPORT_SYMBOL_GPL(drm_gem_cma_prime_mmap); * Returns: * 0 on success, or a negative error code otherwise. */ -int drm_gem_cma_prime_vmap(struct drm_gem_object *obj, struct dma_buf_map *map) +int drm_gem_cma_vmap(struct drm_gem_object *obj, struct dma_buf_map *map) { struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj); @@ -551,7 +469,44 @@ int drm_gem_cma_prime_vmap(struct drm_gem_object *obj, struct dma_buf_map *map) return 0; } -EXPORT_SYMBOL_GPL(drm_gem_cma_prime_vmap); +EXPORT_SYMBOL_GPL(drm_gem_cma_vmap); + +/** + * drm_gem_cma_mmap - memory-map an exported CMA GEM object + * @obj: GEM object + * @vma: VMA for the area to be mapped + * + * This function maps a buffer into a userspace process's address space. + * In addition to the usual GEM VMA setup it immediately faults in the entire + * object instead of using on-demand faulting. Drivers that use the CMA + * helpers should set this as their &drm_gem_object_funcs.mmap callback. + * + * Returns: + * 0 on success or a negative error code on failure. + */ +int drm_gem_cma_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + struct drm_gem_cma_object *cma_obj; + int ret; + + /* + * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the + * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map + * the whole buffer. + */ + vma->vm_pgoff -= drm_vma_node_start(&obj->vma_node); + vma->vm_flags &= ~VM_PFNMAP; + + cma_obj = to_drm_gem_cma_obj(obj); + + ret = dma_mmap_wc(cma_obj->base.dev->dev, vma, cma_obj->vaddr, + cma_obj->paddr, vma->vm_end - vma->vm_start); + if (ret) + drm_gem_vm_close(vma); + + return ret; +} +EXPORT_SYMBOL_GPL(drm_gem_cma_mmap); /** * drm_gem_cma_prime_import_sg_table_vmap - PRIME import another driver's diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h index 81d386b5b92a..fad2249ee67b 100644 --- a/drivers/gpu/drm/drm_internal.h +++ b/drivers/gpu/drm/drm_internal.h @@ -191,6 +191,9 @@ void drm_gem_unpin(struct drm_gem_object *obj); int drm_gem_vmap(struct drm_gem_object *obj, struct dma_buf_map *map); void drm_gem_vunmap(struct drm_gem_object *obj, struct dma_buf_map *map); +int drm_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, + u32 handle); + /* drm_debugfs.c drm_debugfs_crc.c */ #if defined(CONFIG_DEBUG_FS) int drm_debugfs_init(struct drm_minor *minor, int minor_id, diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c index 09d6e9e2e075..c3bd664ea733 100644 --- a/drivers/gpu/drm/drm_irq.c +++ b/drivers/gpu/drm/drm_irq.c @@ -122,7 +122,7 @@ int drm_irq_install(struct drm_device *dev, int irq) dev->driver->irq_preinstall(dev); /* PCI devices require shared interrupts. */ - if (dev->pdev) + if (dev_is_pci(dev->dev)) sh_flags = IRQF_SHARED; ret = request_irq(irq, dev->driver->irq_handler, @@ -140,7 +140,7 @@ int drm_irq_install(struct drm_device *dev, int irq) if (ret < 0) { dev->irq_enabled = false; if (drm_core_check_feature(dev, DRIVER_LEGACY)) - vga_client_register(dev->pdev, NULL, NULL, NULL); + vga_client_register(to_pci_dev(dev->dev), NULL, NULL, NULL); free_irq(irq, dev); } else { dev->irq = irq; @@ -203,7 +203,7 @@ int drm_irq_uninstall(struct drm_device *dev) DRM_DEBUG("irq=%d\n", dev->irq); if (drm_core_check_feature(dev, DRIVER_LEGACY)) - vga_client_register(dev->pdev, NULL, NULL, NULL); + vga_client_register(to_pci_dev(dev->dev), NULL, NULL, NULL); if (dev->driver->irq_uninstall) dev->driver->irq_uninstall(dev); @@ -214,12 +214,45 @@ int drm_irq_uninstall(struct drm_device *dev) } EXPORT_SYMBOL(drm_irq_uninstall); +static void devm_drm_irq_uninstall(void *data) +{ + drm_irq_uninstall(data); +} + +/** + * devm_drm_irq_install - install IRQ handler + * @dev: DRM device + * @irq: IRQ number to install the handler for + * + * devm_drm_irq_install is a help function of drm_irq_install. + * + * if the driver uses devm_drm_irq_install, there is no need + * to call drm_irq_uninstall when the drm module get unloaded, + * as this will done automagically. + * + * Returns: + * Zero on success or a negative error code on failure. + */ +int devm_drm_irq_install(struct drm_device *dev, int irq) +{ + int ret; + + ret = drm_irq_install(dev, irq); + if (ret) + return ret; + + return devm_add_action_or_reset(dev->dev, + devm_drm_irq_uninstall, dev); +} +EXPORT_SYMBOL(devm_drm_irq_install); + #if IS_ENABLED(CONFIG_DRM_LEGACY) int drm_legacy_irq_control(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_control *ctl = data; int ret = 0, irq; + struct pci_dev *pdev; /* if we haven't irq we fallback for compatibility reasons - * this used to be a separate function in drm_dma.h @@ -230,12 +263,13 @@ int drm_legacy_irq_control(struct drm_device *dev, void *data, if (!drm_core_check_feature(dev, DRIVER_LEGACY)) return 0; /* UMS was only ever supported on pci devices. */ - if (WARN_ON(!dev->pdev)) + if (WARN_ON(!dev_is_pci(dev->dev))) return -EINVAL; switch (ctl->func) { case DRM_INST_HANDLER: - irq = dev->pdev->irq; + pdev = to_pci_dev(dev->dev); + irq = pdev->irq; if (dev->if_version < DRM_IF_VERSION(1, 2) && ctl->irq != irq) diff --git a/drivers/gpu/drm/drm_kms_helper_common.c b/drivers/gpu/drm/drm_kms_helper_common.c index 221a8528c993..f933da1656eb 100644 --- a/drivers/gpu/drm/drm_kms_helper_common.c +++ b/drivers/gpu/drm/drm_kms_helper_common.c @@ -64,19 +64,18 @@ MODULE_PARM_DESC(edid_firmware, static int __init drm_kms_helper_init(void) { - int ret; + /* + * The Kconfig DRM_KMS_HELPER selects FRAMEBUFFER_CONSOLE (if !EXPERT) + * but the module doesn't depend on any fb console symbols. At least + * attempt to load fbcon to avoid leaving the system without a usable + * console. + */ + if (IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) && + IS_MODULE(CONFIG_FRAMEBUFFER_CONSOLE) && + !IS_ENABLED(CONFIG_EXPERT)) + request_module_nowait("fbcon"); - /* Call init functions from specific kms helpers here */ - ret = drm_fb_helper_modinit(); - if (ret < 0) - goto out; - - ret = drm_dp_aux_dev_init(); - if (ret < 0) - goto out; - -out: - return ret; + return drm_dp_aux_dev_init(); } static void __exit drm_kms_helper_exit(void) diff --git a/drivers/gpu/drm/drm_legacy.h b/drivers/gpu/drm/drm_legacy.h index 1be3ea320474..f71358f9eac9 100644 --- a/drivers/gpu/drm/drm_legacy.h +++ b/drivers/gpu/drm/drm_legacy.h @@ -127,7 +127,7 @@ static inline void drm_legacy_master_rmmaps(struct drm_device *dev, static inline void drm_legacy_rmmaps(struct drm_device *dev) {} #endif -#if IS_ENABLED(CONFIG_DRM_VM) && IS_ENABLED(CONFIG_DRM_LEGACY) +#if IS_ENABLED(CONFIG_DRM_LEGACY) void drm_legacy_vma_flush(struct drm_device *d); #else static inline void drm_legacy_vma_flush(struct drm_device *d) diff --git a/drivers/gpu/drm/drm_memory.c b/drivers/gpu/drm/drm_memory.c index fbea69d6f909..e4f20a2eb6e7 100644 --- a/drivers/gpu/drm/drm_memory.c +++ b/drivers/gpu/drm/drm_memory.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include @@ -100,24 +99,6 @@ static void *agp_remap(unsigned long offset, unsigned long size, return addr; } -/** Wrapper around agp_free_memory() */ -void drm_free_agp(struct agp_memory *handle, int pages) -{ - agp_free_memory(handle); -} - -/** Wrapper around agp_bind_memory() */ -int drm_bind_agp(struct agp_memory *handle, unsigned int start) -{ - return agp_bind_memory(handle, start); -} - -/** Wrapper around agp_unbind_memory() */ -int drm_unbind_agp(struct agp_memory *handle) -{ - return agp_unbind_memory(handle); -} - #else /* CONFIG_AGP */ static inline void *agp_remap(unsigned long offset, unsigned long size, struct drm_device *dev) @@ -156,35 +137,3 @@ void drm_legacy_ioremapfree(struct drm_local_map *map, struct drm_device *dev) iounmap(map->handle); } EXPORT_SYMBOL(drm_legacy_ioremapfree); - -bool drm_need_swiotlb(int dma_bits) -{ - struct resource *tmp; - resource_size_t max_iomem = 0; - - /* - * Xen paravirtual hosts require swiotlb regardless of requested dma - * transfer size. - * - * NOTE: Really, what it requires is use of the dma_alloc_coherent - * allocator used in ttm_dma_populate() instead of - * ttm_populate_and_map_pages(), which bounce buffers so much in - * Xen it leads to swiotlb buffer exhaustion. - */ - if (xen_pv_domain()) - return true; - - /* - * Enforce dma_alloc_coherent when memory encryption is active as well - * for the same reasons as for Xen paravirtual hosts. - */ - if (mem_encrypt_active()) - return true; - - for (tmp = iomem_resource.child; tmp; tmp = tmp->sibling) { - max_iomem = max(max_iomem, tmp->end); - } - - return max_iomem > ((u64)1 << dma_bits); -} -EXPORT_SYMBOL(drm_need_swiotlb); diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c index f1affc1bb679..37b4b9f0e468 100644 --- a/drivers/gpu/drm/drm_mode_config.c +++ b/drivers/gpu/drm/drm_mode_config.c @@ -195,7 +195,7 @@ void drm_mode_config_reset(struct drm_device *dev) crtc->funcs->reset(crtc); drm_for_each_encoder(encoder, dev) - if (encoder->funcs->reset) + if (encoder->funcs && encoder->funcs->reset) encoder->funcs->reset(encoder); drm_connector_list_iter_begin(dev, &conn_iter); @@ -625,6 +625,10 @@ static void validate_encoder_possible_crtcs(struct drm_encoder *encoder) void drm_mode_config_validate(struct drm_device *dev) { struct drm_encoder *encoder; + struct drm_crtc *crtc; + struct drm_plane *plane; + u32 primary_with_crtc = 0, cursor_with_crtc = 0; + unsigned int num_primary = 0; if (!drm_core_check_feature(dev, DRIVER_MODESET)) return; @@ -636,4 +640,49 @@ void drm_mode_config_validate(struct drm_device *dev) validate_encoder_possible_clones(encoder); validate_encoder_possible_crtcs(encoder); } + + drm_for_each_crtc(crtc, dev) { + WARN(!crtc->primary, "Missing primary plane on [CRTC:%d:%s]\n", + crtc->base.id, crtc->name); + + WARN(crtc->cursor && crtc->funcs->cursor_set, + "[CRTC:%d:%s] must not have both a cursor plane and a cursor_set func", + crtc->base.id, crtc->name); + WARN(crtc->cursor && crtc->funcs->cursor_set2, + "[CRTC:%d:%s] must not have both a cursor plane and a cursor_set2 func", + crtc->base.id, crtc->name); + WARN(crtc->cursor && crtc->funcs->cursor_move, + "[CRTC:%d:%s] must not have both a cursor plane and a cursor_move func", + crtc->base.id, crtc->name); + + if (crtc->primary) { + WARN(!(crtc->primary->possible_crtcs & drm_crtc_mask(crtc)), + "Bogus primary plane possible_crtcs: [PLANE:%d:%s] must be compatible with [CRTC:%d:%s]\n", + crtc->primary->base.id, crtc->primary->name, + crtc->base.id, crtc->name); + WARN(primary_with_crtc & drm_plane_mask(crtc->primary), + "Primary plane [PLANE:%d:%s] used for multiple CRTCs", + crtc->primary->base.id, crtc->primary->name); + primary_with_crtc |= drm_plane_mask(crtc->primary); + } + if (crtc->cursor) { + WARN(!(crtc->cursor->possible_crtcs & drm_crtc_mask(crtc)), + "Bogus cursor plane possible_crtcs: [PLANE:%d:%s] must be compatible with [CRTC:%d:%s]\n", + crtc->cursor->base.id, crtc->cursor->name, + crtc->base.id, crtc->name); + WARN(cursor_with_crtc & drm_plane_mask(crtc->cursor), + "Cursor plane [PLANE:%d:%s] used for multiple CRTCs", + crtc->cursor->base.id, crtc->cursor->name); + cursor_with_crtc |= drm_plane_mask(crtc->cursor); + } + } + + drm_for_each_plane(plane, dev) { + if (plane->type == DRM_PLANE_TYPE_PRIMARY) + num_primary++; + } + + WARN(num_primary != dev->mode_config.num_crtc, + "Must have as many primary planes as there are CRTCs, but have %u primary planes and %u CRTCs", + num_primary, dev->mode_config.num_crtc); } diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c index 33fb2f05ce66..1ac67d4505e0 100644 --- a/drivers/gpu/drm/drm_modes.c +++ b/drivers/gpu/drm/drm_modes.c @@ -762,7 +762,7 @@ int drm_mode_vrefresh(const struct drm_display_mode *mode) if (mode->htotal == 0 || mode->vtotal == 0) return 0; - num = mode->clock * 1000; + num = mode->clock; den = mode->htotal * mode->vtotal; if (mode->flags & DRM_MODE_FLAG_INTERLACE) @@ -772,7 +772,7 @@ int drm_mode_vrefresh(const struct drm_display_mode *mode) if (mode->vscan > 1) den *= mode->vscan; - return DIV_ROUND_CLOSEST(num, den); + return DIV_ROUND_CLOSEST_ULL(mul_u32_u32(num, 1000), den); } EXPORT_SYMBOL(drm_mode_vrefresh); diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c index 6dba4b8ce4fe..2294a1580d35 100644 --- a/drivers/gpu/drm/drm_pci.c +++ b/drivers/gpu/drm/drm_pci.c @@ -24,6 +24,8 @@ #include #include +#include +#include #include #include @@ -36,6 +38,9 @@ #include "drm_legacy.h" #ifdef CONFIG_DRM_LEGACY +/* List of devices hanging off drivers with stealth attach. */ +static LIST_HEAD(legacy_dev_list); +static DEFINE_MUTEX(legacy_dev_list_lock); /** * drm_pci_alloc - Allocate a PCI consistent memory block, for DMA. @@ -65,7 +70,7 @@ drm_dma_handle_t *drm_pci_alloc(struct drm_device * dev, size_t size, size_t ali return NULL; dmah->size = size; - dmah->vaddr = dma_alloc_coherent(&dev->pdev->dev, size, + dmah->vaddr = dma_alloc_coherent(dev->dev, size, &dmah->busaddr, GFP_KERNEL); @@ -88,7 +93,7 @@ EXPORT_SYMBOL(drm_pci_alloc); */ void drm_pci_free(struct drm_device * dev, drm_dma_handle_t * dmah) { - dma_free_coherent(&dev->pdev->dev, dmah->size, dmah->vaddr, + dma_free_coherent(dev->dev, dmah->size, dmah->vaddr, dmah->busaddr); kfree(dmah); } @@ -107,16 +112,18 @@ static int drm_get_pci_domain(struct drm_device *dev) return 0; #endif /* __alpha__ */ - return pci_domain_nr(dev->pdev->bus); + return pci_domain_nr(to_pci_dev(dev->dev)->bus); } int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master) { + struct pci_dev *pdev = to_pci_dev(dev->dev); + master->unique = kasprintf(GFP_KERNEL, "pci:%04x:%02x:%02x.%d", drm_get_pci_domain(dev), - dev->pdev->bus->number, - PCI_SLOT(dev->pdev->devfn), - PCI_FUNC(dev->pdev->devfn)); + pdev->bus->number, + PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); if (!master->unique) return -ENOMEM; @@ -126,12 +133,14 @@ int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master) static int drm_pci_irq_by_busid(struct drm_device *dev, struct drm_irq_busid *p) { + struct pci_dev *pdev = to_pci_dev(dev->dev); + if ((p->busnum >> 8) != drm_get_pci_domain(dev) || - (p->busnum & 0xff) != dev->pdev->bus->number || - p->devnum != PCI_SLOT(dev->pdev->devfn) || p->funcnum != PCI_FUNC(dev->pdev->devfn)) + (p->busnum & 0xff) != pdev->bus->number || + p->devnum != PCI_SLOT(pdev->devfn) || p->funcnum != PCI_FUNC(pdev->devfn)) return -EINVAL; - p->irq = dev->pdev->irq; + p->irq = pdev->irq; DRM_DEBUG("%d:%d:%d => IRQ %d\n", p->busnum, p->devnum, p->funcnum, p->irq); @@ -159,7 +168,7 @@ int drm_legacy_irq_by_busid(struct drm_device *dev, void *data, return -EOPNOTSUPP; /* UMS was only ever support on PCI devices. */ - if (WARN_ON(!dev->pdev)) + if (WARN_ON(!dev_is_pci(dev->dev))) return -EINVAL; if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ)) @@ -183,7 +192,7 @@ void drm_pci_agp_destroy(struct drm_device *dev) static void drm_pci_agp_init(struct drm_device *dev) { if (drm_core_check_feature(dev, DRIVER_USE_AGP)) { - if (pci_find_capability(dev->pdev, PCI_CAP_ID_AGP)) + if (pci_find_capability(to_pci_dev(dev->dev), PCI_CAP_ID_AGP)) dev->agp = drm_agp_init(dev); if (dev->agp) { dev->agp->agp_mtrr = arch_phys_wc_add( @@ -196,7 +205,7 @@ static void drm_pci_agp_init(struct drm_device *dev) static int drm_get_pci_dev(struct pci_dev *pdev, const struct pci_device_id *ent, - struct drm_driver *driver) + const struct drm_driver *driver) { struct drm_device *dev; int ret; @@ -225,10 +234,11 @@ static int drm_get_pci_dev(struct pci_dev *pdev, if (ret) goto err_agp; - /* No locking needed since shadow-attach is single-threaded since it may - * only be called from the per-driver module init hook. */ - if (drm_core_check_feature(dev, DRIVER_LEGACY)) - list_add_tail(&dev->legacy_dev_list, &driver->legacy_dev_list); + if (drm_core_check_feature(dev, DRIVER_LEGACY)) { + mutex_lock(&legacy_dev_list_lock); + list_add_tail(&dev->legacy_dev_list, &legacy_dev_list); + mutex_unlock(&legacy_dev_list_lock); + } return 0; @@ -249,7 +259,8 @@ err_free: * * Return: 0 on success or a negative error code on failure. */ -int drm_legacy_pci_init(struct drm_driver *driver, struct pci_driver *pdriver) +int drm_legacy_pci_init(const struct drm_driver *driver, + struct pci_driver *pdriver) { struct pci_dev *pdev = NULL; const struct pci_device_id *pid; @@ -261,7 +272,6 @@ int drm_legacy_pci_init(struct drm_driver *driver, struct pci_driver *pdriver) return -EINVAL; /* If not using KMS, fall back to stealth mode manual scanning. */ - INIT_LIST_HEAD(&driver->legacy_dev_list); for (i = 0; pdriver->id_table[i].vendor != 0; i++) { pid = &pdriver->id_table[i]; @@ -295,7 +305,8 @@ EXPORT_SYMBOL(drm_legacy_pci_init); * Unregister a DRM driver shadow-attached through drm_legacy_pci_init(). This * is deprecated and only used by dri1 drivers. */ -void drm_legacy_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver) +void drm_legacy_pci_exit(const struct drm_driver *driver, + struct pci_driver *pdriver) { struct drm_device *dev, *tmp; @@ -304,11 +315,15 @@ void drm_legacy_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver) if (!(driver->driver_features & DRIVER_LEGACY)) { WARN_ON(1); } else { - list_for_each_entry_safe(dev, tmp, &driver->legacy_dev_list, + mutex_lock(&legacy_dev_list_lock); + list_for_each_entry_safe(dev, tmp, &legacy_dev_list, legacy_dev_list) { - list_del(&dev->legacy_dev_list); - drm_put_dev(dev); + if (dev->driver == driver) { + list_del(&dev->legacy_dev_list); + drm_put_dev(dev); + } } + mutex_unlock(&legacy_dev_list_lock); } DRM_INFO("Module unloaded\n"); } diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c index a0cb746bcb0a..338650abd267 100644 --- a/drivers/gpu/drm/drm_plane.c +++ b/drivers/gpu/drm/drm_plane.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "drm_crtc_internal.h" @@ -40,7 +41,7 @@ * A plane represents an image source that can be blended with or overlayed on * top of a CRTC during the scanout process. Planes take their input data from a * &drm_framebuffer object. The plane itself specifies the cropping and scaling - * of that image, and where it is placed on the visible are of a display + * of that image, and where it is placed on the visible area of a display * pipeline, represented by &drm_crtc. A plane can also have additional * properties that specify how the pixels are positioned and blended, like * rotation or Z-position. All these properties are stored in &drm_plane_state. @@ -49,14 +50,34 @@ * &struct drm_plane (possibly as part of a larger structure) and registers it * with a call to drm_universal_plane_init(). * - * Cursor and overlay planes are optional. All drivers should provide one - * primary plane per CRTC to avoid surprising userspace too much. See enum - * drm_plane_type for a more in-depth discussion of these special uapi-relevant - * plane types. Special planes are associated with their CRTC by calling - * drm_crtc_init_with_planes(). - * * The type of a plane is exposed in the immutable "type" enumeration property, - * which has one of the following values: "Overlay", "Primary", "Cursor". + * which has one of the following values: "Overlay", "Primary", "Cursor" (see + * enum drm_plane_type). A plane can be compatible with multiple CRTCs, see + * &drm_plane.possible_crtcs. + * + * Each CRTC must have a unique primary plane userspace can attach to enable + * the CRTC. In other words, userspace must be able to attach a different + * primary plane to each CRTC at the same time. Primary planes can still be + * compatible with multiple CRTCs. There must be exactly as many primary planes + * as there are CRTCs. + * + * Legacy uAPI doesn't expose the primary and cursor planes directly. DRM core + * relies on the driver to set the primary and optionally the cursor plane used + * for legacy IOCTLs. This is done by calling drm_crtc_init_with_planes(). All + * drivers must provide one primary plane per CRTC to avoid surprising legacy + * userspace too much. + */ + +/** + * DOC: standard plane properties + * + * DRM planes have a few standardized properties: + * + * IN_FORMATS: + * Blob property which contains the set of buffer format and modifier + * pairs supported by this plane. The blob is a struct + * drm_format_modifier_blob. Without this property the plane doesn't + * support buffers with modifiers. Userspace cannot change this property. */ static unsigned int drm_num_planes(struct drm_device *dev) @@ -152,31 +173,16 @@ done: return 0; } -/** - * drm_universal_plane_init - Initialize a new universal plane object - * @dev: DRM device - * @plane: plane object to init - * @possible_crtcs: bitmask of possible CRTCs - * @funcs: callbacks for the new plane - * @formats: array of supported formats (DRM_FORMAT\_\*) - * @format_count: number of elements in @formats - * @format_modifiers: array of struct drm_format modifiers terminated by - * DRM_FORMAT_MOD_INVALID - * @type: type of plane (overlay, primary, cursor) - * @name: printf style format string for the plane name, or NULL for default name - * - * Initializes a plane object of type @type. - * - * Returns: - * Zero on success, error code on failure. - */ -int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, - uint32_t possible_crtcs, - const struct drm_plane_funcs *funcs, - const uint32_t *formats, unsigned int format_count, - const uint64_t *format_modifiers, - enum drm_plane_type type, - const char *name, ...) +__printf(9, 0) +static int __drm_universal_plane_init(struct drm_device *dev, + struct drm_plane *plane, + uint32_t possible_crtcs, + const struct drm_plane_funcs *funcs, + const uint32_t *formats, + unsigned int format_count, + const uint64_t *format_modifiers, + enum drm_plane_type type, + const char *name, va_list ap) { struct drm_mode_config *config = &dev->mode_config; unsigned int format_modifier_count = 0; @@ -237,11 +243,7 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, } if (name) { - va_list ap; - - va_start(ap, name); plane->name = kvasprintf(GFP_KERNEL, name, ap); - va_end(ap); } else { plane->name = kasprintf(GFP_KERNEL, "plane-%d", drm_num_planes(dev)); @@ -286,8 +288,102 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, return 0; } + +/** + * drm_universal_plane_init - Initialize a new universal plane object + * @dev: DRM device + * @plane: plane object to init + * @possible_crtcs: bitmask of possible CRTCs + * @funcs: callbacks for the new plane + * @formats: array of supported formats (DRM_FORMAT\_\*) + * @format_count: number of elements in @formats + * @format_modifiers: array of struct drm_format modifiers terminated by + * DRM_FORMAT_MOD_INVALID + * @type: type of plane (overlay, primary, cursor) + * @name: printf style format string for the plane name, or NULL for default name + * + * Initializes a plane object of type @type. The &drm_plane_funcs.destroy hook + * should call drm_plane_cleanup() and kfree() the plane structure. The plane + * structure should not be allocated with devm_kzalloc(). + * + * Note: consider using drmm_universal_plane_alloc() instead of + * drm_universal_plane_init() to let the DRM managed resource infrastructure + * take care of cleanup and deallocation. + * + * Returns: + * Zero on success, error code on failure. + */ +int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, + uint32_t possible_crtcs, + const struct drm_plane_funcs *funcs, + const uint32_t *formats, unsigned int format_count, + const uint64_t *format_modifiers, + enum drm_plane_type type, + const char *name, ...) +{ + va_list ap; + int ret; + + WARN_ON(!funcs->destroy); + + va_start(ap, name); + ret = __drm_universal_plane_init(dev, plane, possible_crtcs, funcs, + formats, format_count, format_modifiers, + type, name, ap); + va_end(ap); + return ret; +} EXPORT_SYMBOL(drm_universal_plane_init); +static void drmm_universal_plane_alloc_release(struct drm_device *dev, void *ptr) +{ + struct drm_plane *plane = ptr; + + if (WARN_ON(!plane->dev)) + return; + + drm_plane_cleanup(plane); +} + +void *__drmm_universal_plane_alloc(struct drm_device *dev, size_t size, + size_t offset, uint32_t possible_crtcs, + const struct drm_plane_funcs *funcs, + const uint32_t *formats, unsigned int format_count, + const uint64_t *format_modifiers, + enum drm_plane_type type, + const char *name, ...) +{ + void *container; + struct drm_plane *plane; + va_list ap; + int ret; + + if (WARN_ON(!funcs || funcs->destroy)) + return ERR_PTR(-EINVAL); + + container = drmm_kzalloc(dev, size, GFP_KERNEL); + if (!container) + return ERR_PTR(-ENOMEM); + + plane = container + offset; + + va_start(ap, name); + ret = __drm_universal_plane_init(dev, plane, possible_crtcs, funcs, + formats, format_count, format_modifiers, + type, name, ap); + va_end(ap); + if (ret) + return ERR_PTR(ret); + + ret = drmm_add_action_or_reset(dev, drmm_universal_plane_alloc_release, + plane); + if (ret) + return ERR_PTR(ret); + + return container; +} +EXPORT_SYMBOL(__drmm_universal_plane_alloc); + int drm_plane_register_all(struct drm_device *dev) { unsigned int num_planes = 0; diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c index 7db55fce35d8..2a54f86856af 100644 --- a/drivers/gpu/drm/drm_prime.c +++ b/drivers/gpu/drm/drm_prime.c @@ -717,6 +717,8 @@ int drm_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) vma->vm_pgoff += drm_vma_node_start(&obj->vma_node); if (obj->funcs && obj->funcs->mmap) { + vma->vm_ops = obj->funcs->vm_ops; + ret = obj->funcs->mmap(obj, vma); if (ret) return ret; @@ -978,44 +980,58 @@ struct drm_gem_object *drm_gem_prime_import(struct drm_device *dev, EXPORT_SYMBOL(drm_gem_prime_import); /** - * drm_prime_sg_to_page_addr_arrays - convert an sg table into a page array + * drm_prime_sg_to_page_array - convert an sg table into a page array * @sgt: scatter-gather table to convert - * @pages: optional array of page pointers to store the page array in - * @addrs: optional array to store the dma bus address of each page - * @max_entries: size of both the passed-in arrays + * @pages: array of page pointers to store the pages in + * @max_entries: size of the passed-in array * - * Exports an sg table into an array of pages and addresses. This is currently - * required by the TTM driver in order to do correct fault handling. + * Exports an sg table into an array of pages. * - * Drivers can use this in their &drm_driver.gem_prime_import_sg_table - * implementation. + * This function is deprecated and strongly discouraged to be used. + * The page array is only useful for page faults and those can corrupt fields + * in the struct page if they are not handled by the exporting driver. */ -int drm_prime_sg_to_page_addr_arrays(struct sg_table *sgt, struct page **pages, - dma_addr_t *addrs, int max_entries) +int __deprecated drm_prime_sg_to_page_array(struct sg_table *sgt, + struct page **pages, + int max_entries) { - struct sg_dma_page_iter dma_iter; struct sg_page_iter page_iter; struct page **p = pages; - dma_addr_t *a = addrs; - if (pages) { - for_each_sgtable_page(sgt, &page_iter, 0) { - if (WARN_ON(p - pages >= max_entries)) - return -1; - *p++ = sg_page_iter_page(&page_iter); - } + for_each_sgtable_page(sgt, &page_iter, 0) { + if (WARN_ON(p - pages >= max_entries)) + return -1; + *p++ = sg_page_iter_page(&page_iter); } - if (addrs) { - for_each_sgtable_dma_page(sgt, &dma_iter, 0) { - if (WARN_ON(a - addrs >= max_entries)) - return -1; - *a++ = sg_page_iter_dma_address(&dma_iter); - } - } - return 0; } -EXPORT_SYMBOL(drm_prime_sg_to_page_addr_arrays); +EXPORT_SYMBOL(drm_prime_sg_to_page_array); + +/** + * drm_prime_sg_to_dma_addr_array - convert an sg table into a dma addr array + * @sgt: scatter-gather table to convert + * @addrs: array to store the dma bus address of each page + * @max_entries: size of both the passed-in arrays + * + * Exports an sg table into an array of addresses. + * + * Drivers should use this in their &drm_driver.gem_prime_import_sg_table + * implementation. + */ +int drm_prime_sg_to_dma_addr_array(struct sg_table *sgt, dma_addr_t *addrs, + int max_entries) +{ + struct sg_dma_page_iter dma_iter; + dma_addr_t *a = addrs; + + for_each_sgtable_dma_page(sgt, &dma_iter, 0) { + if (WARN_ON(a - addrs >= max_entries)) + return -1; + *a++ = sg_page_iter_dma_address(&dma_iter); + } + return 0; +} +EXPORT_SYMBOL(drm_prime_sg_to_dma_addr_array); /** * drm_prime_gem_destroy - helper to clean up a PRIME-imported GEM object diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c index d6017726cc2a..ad59a51eab6d 100644 --- a/drivers/gpu/drm/drm_probe_helper.c +++ b/drivers/gpu/drm/drm_probe_helper.c @@ -515,7 +515,8 @@ retry: if (count == 0 && connector->status == connector_status_connected) count = drm_add_override_edid_modes(connector); - if (count == 0 && connector->status == connector_status_connected) + if (count == 0 && (connector->status == connector_status_connected || + connector->status == connector_status_unknown)) count = drm_add_modes_noedid(connector, 1024, 768); count += drm_helper_probe_add_cmdline_mode(connector); if (count == 0) diff --git a/drivers/gpu/drm/drm_simple_kms_helper.c b/drivers/gpu/drm/drm_simple_kms_helper.c index 743e57c1b44f..6ce8f5cd1eb5 100644 --- a/drivers/gpu/drm/drm_simple_kms_helper.c +++ b/drivers/gpu/drm/drm_simple_kms_helper.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -55,8 +56,9 @@ static const struct drm_encoder_funcs drm_simple_encoder_funcs_cleanup = { * stored in the device structure. Free the encoder's memory as part of * the device release function. * - * FIXME: Later improvements to DRM's resource management may allow for - * an automated kfree() of the encoder's memory. + * Note: consider using drmm_simple_encoder_alloc() instead of + * drm_simple_encoder_init() to let the DRM managed resource infrastructure + * take care of cleanup and deallocation. * * Returns: * Zero on success, error code on failure. @@ -71,6 +73,14 @@ int drm_simple_encoder_init(struct drm_device *dev, } EXPORT_SYMBOL(drm_simple_encoder_init); +void *__drmm_simple_encoder_alloc(struct drm_device *dev, size_t size, + size_t offset, int encoder_type) +{ + return __drmm_encoder_alloc(dev, size, offset, NULL, encoder_type, + NULL); +} +EXPORT_SYMBOL(__drmm_simple_encoder_alloc); + static enum drm_mode_status drm_simple_kms_crtc_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode *mode) diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c index d30e2f2b8f3c..893165eeddf3 100644 --- a/drivers/gpu/drm/drm_vblank.c +++ b/drivers/gpu/drm/drm_vblank.c @@ -74,7 +74,7 @@ * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the * | | frame as it * | | travels down - * | | ("sacn out") + * | | ("scan out") * | Old frame | * | | * | | @@ -1006,7 +1006,14 @@ static void send_vblank_event(struct drm_device *dev, break; } trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq); - drm_send_event_locked(dev, &e->base); + /* + * Use the same timestamp for any associated fence signal to avoid + * mismatch in timestamps for vsync & fence events triggered by the + * same HW event. Frameworks like SurfaceFlinger in Android expects the + * retire-fence timestamp to match exactly with HW vsync as it uses it + * for its software vsync modeling. + */ + drm_send_event_timestamp_locked(dev, &e->base, now); } /** diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c index 6d5a03b32238..9b3b989d7cad 100644 --- a/drivers/gpu/drm/drm_vm.c +++ b/drivers/gpu/drm/drm_vm.c @@ -278,7 +278,7 @@ static void drm_vm_shm_close(struct vm_area_struct *vma) case _DRM_SCATTER_GATHER: break; case _DRM_CONSISTENT: - dma_free_coherent(&dev->pdev->dev, + dma_free_coherent(dev->dev, map->size, map->handle, map->offset); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c index d9bd83203a15..b390dd4d60b7 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c @@ -135,8 +135,7 @@ struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev, goto fail; } - ret = drm_prime_sg_to_page_addr_arrays(sgt, etnaviv_obj->pages, - NULL, npages); + ret = drm_prime_sg_to_page_array(sgt, etnaviv_obj->pages, npages); if (ret) goto fail; diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig index 951d5f708e92..6a251e3aa779 100644 --- a/drivers/gpu/drm/exynos/Kconfig +++ b/drivers/gpu/drm/exynos/Kconfig @@ -89,7 +89,6 @@ comment "Sub-drivers" config DRM_EXYNOS_G2D bool "G2D" depends on VIDEO_SAMSUNG_S5P_G2D=n || COMPILE_TEST - select FRAME_VECTOR help Choose this option if you want to use Exynos G2D for DRM. diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c index 967a5cdc120e..1e0c5a7f206e 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c +++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c @@ -205,7 +205,8 @@ struct g2d_cmdlist_userptr { dma_addr_t dma_addr; unsigned long userptr; unsigned long size; - struct frame_vector *vec; + struct page **pages; + unsigned int npages; struct sg_table *sgt; atomic_t refcount; bool in_pool; @@ -378,7 +379,6 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, bool force) { struct g2d_cmdlist_userptr *g2d_userptr = obj; - struct page **pages; if (!obj) return; @@ -398,15 +398,9 @@ out: dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt, DMA_BIDIRECTIONAL, 0); - pages = frame_vector_pages(g2d_userptr->vec); - if (!IS_ERR(pages)) { - int i; - - for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) - set_page_dirty_lock(pages[i]); - } - put_vaddr_frames(g2d_userptr->vec); - frame_vector_destroy(g2d_userptr->vec); + unpin_user_pages_dirty_lock(g2d_userptr->pages, g2d_userptr->npages, + true); + kvfree(g2d_userptr->pages); if (!g2d_userptr->out_of_list) list_del_init(&g2d_userptr->list); @@ -474,35 +468,35 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, offset = userptr & ~PAGE_MASK; end = PAGE_ALIGN(userptr + size); npages = (end - start) >> PAGE_SHIFT; - g2d_userptr->vec = frame_vector_create(npages); - if (!g2d_userptr->vec) { + g2d_userptr->pages = kvmalloc_array(npages, sizeof(*g2d_userptr->pages), + GFP_KERNEL); + if (!g2d_userptr->pages) { ret = -ENOMEM; goto err_free; } - ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE, - g2d_userptr->vec); + ret = pin_user_pages_fast(start, npages, + FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM, + g2d_userptr->pages); if (ret != npages) { DRM_DEV_ERROR(g2d->dev, "failed to get user pages from userptr.\n"); if (ret < 0) - goto err_destroy_framevec; + goto err_destroy_pages; + npages = ret; ret = -EFAULT; - goto err_put_framevec; - } - if (frame_vector_to_pages(g2d_userptr->vec) < 0) { - ret = -EFAULT; - goto err_put_framevec; + goto err_unpin_pages; } + g2d_userptr->npages = npages; sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); if (!sgt) { ret = -ENOMEM; - goto err_put_framevec; + goto err_unpin_pages; } ret = sg_alloc_table_from_pages(sgt, - frame_vector_pages(g2d_userptr->vec), + g2d_userptr->pages, npages, offset, size, GFP_KERNEL); if (ret < 0) { DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n"); @@ -538,11 +532,11 @@ err_sg_free_table: err_free_sgt: kfree(sgt); -err_put_framevec: - put_vaddr_frames(g2d_userptr->vec); +err_unpin_pages: + unpin_user_pages(g2d_userptr->pages, npages); -err_destroy_framevec: - frame_vector_destroy(g2d_userptr->vec); +err_destroy_pages: + kvfree(g2d_userptr->pages); err_free: kfree(g2d_userptr); diff --git a/drivers/gpu/drm/gma500/Kconfig b/drivers/gpu/drm/gma500/Kconfig index 0e23c93a1094..ec395658a43f 100644 --- a/drivers/gpu/drm/gma500/Kconfig +++ b/drivers/gpu/drm/gma500/Kconfig @@ -1,9 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only config DRM_GMA500 - tristate "Intel GMA5/600 KMS Framebuffer" + tristate "Intel GMA500/600/3600/3650 KMS Framebuffer" depends on DRM && PCI && X86 && MMU select DRM_KMS_HELPER - select DRM_TTM # GMA500 depends on ACPI_VIDEO when ACPI is enabled, just like i915 select ACPI_VIDEO if ACPI select BACKLIGHT_CLASS_DEVICE if ACPI @@ -19,17 +18,3 @@ config DRM_GMA600 help Say yes to include support for GMA600 (Intel Moorestown/Oaktrail) platforms with LVDS ports. MIPI is not currently supported. - -config DRM_GMA3600 - bool "Intel GMA3600/3650 support (Experimental)" - depends on DRM_GMA500 - help - Say yes to include basic support for Intel GMA3600/3650 (Intel - Cedar Trail) platforms. - -config DRM_MEDFIELD - bool "Intel Medfield support (Experimental)" - depends on DRM_GMA500 && X86_INTEL_MID - help - Say yes to include support for the Intel Medfield platform. - diff --git a/drivers/gpu/drm/gma500/Makefile b/drivers/gpu/drm/gma500/Makefile index c8f2c89be99d..884ab1f9063e 100644 --- a/drivers/gpu/drm/gma500/Makefile +++ b/drivers/gpu/drm/gma500/Makefile @@ -6,36 +6,35 @@ gma500_gfx-y += \ accel_2d.o \ backlight.o \ + blitter.o \ + cdv_device.o \ + cdv_intel_crt.o \ + cdv_intel_display.o \ + cdv_intel_dp.o \ + cdv_intel_hdmi.o \ + cdv_intel_lvds.o \ framebuffer.o \ gem.o \ + gma_device.o \ + gma_display.o \ gtt.o \ intel_bios.o \ - intel_i2c.o \ intel_gmbus.o \ + intel_i2c.o \ + mid_bios.o \ mmu.o \ - blitter.o \ power.o \ + psb_device.o \ psb_drv.o \ - gma_display.o \ - gma_device.o \ psb_intel_display.o \ psb_intel_lvds.o \ psb_intel_modes.o \ psb_intel_sdvo.o \ psb_lid.o \ - psb_irq.o \ - psb_device.o \ - mid_bios.o + psb_irq.o gma500_gfx-$(CONFIG_ACPI) += opregion.o \ -gma500_gfx-$(CONFIG_DRM_GMA3600) += cdv_device.o \ - cdv_intel_crt.o \ - cdv_intel_display.o \ - cdv_intel_hdmi.o \ - cdv_intel_lvds.o \ - cdv_intel_dp.o - gma500_gfx-$(CONFIG_DRM_GMA600) += oaktrail_device.o \ oaktrail_crtc.o \ oaktrail_lvds.o \ @@ -43,14 +42,4 @@ gma500_gfx-$(CONFIG_DRM_GMA600) += oaktrail_device.o \ oaktrail_hdmi.o \ oaktrail_hdmi_i2c.o -gma500_gfx-$(CONFIG_DRM_MEDFIELD) += mdfld_device.o \ - mdfld_output.o \ - mdfld_intel_display.o \ - mdfld_dsi_output.o \ - mdfld_dsi_dpi.o \ - mdfld_dsi_pkg_sender.o \ - mdfld_tpo_vid.o \ - mdfld_tmd_vid.o \ - tc35876x-dsi-lvds.o - obj-$(CONFIG_DRM_GMA500) += gma500_gfx.o diff --git a/drivers/gpu/drm/gma500/cdv_device.c b/drivers/gpu/drm/gma500/cdv_device.c index e75293e4a52f..19e055dbd4c2 100644 --- a/drivers/gpu/drm/gma500/cdv_device.c +++ b/drivers/gpu/drm/gma500/cdv_device.c @@ -95,13 +95,14 @@ static u32 cdv_get_max_backlight(struct drm_device *dev) static int cdv_get_brightness(struct backlight_device *bd) { struct drm_device *dev = bl_get_data(bd); + struct pci_dev *pdev = to_pci_dev(dev->dev); u32 val = REG_READ(BLC_PWM_CTL) & BACKLIGHT_DUTY_CYCLE_MASK; if (cdv_backlight_combination_mode(dev)) { u8 lbpc; val &= ~1; - pci_read_config_byte(dev->pdev, 0xF4, &lbpc); + pci_read_config_byte(pdev, 0xF4, &lbpc); val *= lbpc; } return (val * 100)/cdv_get_max_backlight(dev); @@ -111,6 +112,7 @@ static int cdv_get_brightness(struct backlight_device *bd) static int cdv_set_brightness(struct backlight_device *bd) { struct drm_device *dev = bl_get_data(bd); + struct pci_dev *pdev = to_pci_dev(dev->dev); int level = bd->props.brightness; u32 blc_pwm_ctl; @@ -128,7 +130,7 @@ static int cdv_set_brightness(struct backlight_device *bd) lbpc = level * 0xfe / max + 1; level /= lbpc; - pci_write_config_byte(dev->pdev, 0xF4, lbpc); + pci_write_config_byte(pdev, 0xF4, lbpc); } blc_pwm_ctl = REG_READ(BLC_PWM_CTL) & ~BACKLIGHT_DUTY_CYCLE_MASK; @@ -205,8 +207,9 @@ static inline void CDV_MSG_WRITE32(int domain, uint port, uint offset, static void cdv_init_pm(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); u32 pwr_cnt; - int domain = pci_domain_nr(dev->pdev->bus); + int domain = pci_domain_nr(pdev->bus); int i; dev_priv->apm_base = CDV_MSG_READ32(domain, PSB_PUNIT_PORT, @@ -234,6 +237,8 @@ static void cdv_init_pm(struct drm_device *dev) static void cdv_errata(struct drm_device *dev) { + struct pci_dev *pdev = to_pci_dev(dev->dev); + /* Disable bonus launch. * CPU and GPU competes for memory and display misses updates and * flickers. Worst with dual core, dual displays. @@ -242,7 +247,7 @@ static void cdv_errata(struct drm_device *dev) * Bonus Launch to work around the issue, by degrading * performance. */ - CDV_MSG_WRITE32(pci_domain_nr(dev->pdev->bus), 3, 0x30, 0x08027108); + CDV_MSG_WRITE32(pci_domain_nr(pdev->bus), 3, 0x30, 0x08027108); } /** @@ -255,12 +260,13 @@ static void cdv_errata(struct drm_device *dev) static int cdv_save_display_registers(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct psb_save_area *regs = &dev_priv->regs; struct drm_connector *connector; dev_dbg(dev->dev, "Saving GPU registers.\n"); - pci_read_config_byte(dev->pdev, 0xF4, ®s->cdv.saveLBB); + pci_read_config_byte(pdev, 0xF4, ®s->cdv.saveLBB); regs->cdv.saveDSPCLK_GATE_D = REG_READ(DSPCLK_GATE_D); regs->cdv.saveRAMCLK_GATE_D = REG_READ(RAMCLK_GATE_D); @@ -309,11 +315,12 @@ static int cdv_save_display_registers(struct drm_device *dev) static int cdv_restore_display_registers(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct psb_save_area *regs = &dev_priv->regs; struct drm_connector *connector; u32 temp; - pci_write_config_byte(dev->pdev, 0xF4, regs->cdv.saveLBB); + pci_write_config_byte(pdev, 0xF4, regs->cdv.saveLBB); REG_WRITE(DSPCLK_GATE_D, regs->cdv.saveDSPCLK_GATE_D); REG_WRITE(RAMCLK_GATE_D, regs->cdv.saveRAMCLK_GATE_D); @@ -421,16 +428,16 @@ static int cdv_power_up(struct drm_device *dev) static void cdv_hotplug_work_func(struct work_struct *work) { struct drm_psb_private *dev_priv = container_of(work, struct drm_psb_private, - hotplug_work); + hotplug_work); struct drm_device *dev = dev_priv->dev; /* Just fire off a uevent and let userspace tell us what to do */ drm_helper_hpd_irq_event(dev); -} +} /* The core driver has received a hotplug IRQ. We are in IRQ context so extract the needed information and kick off queued processing */ - + static int cdv_hotplug_event(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; @@ -449,7 +456,7 @@ static void cdv_hotplug_enable(struct drm_device *dev, bool on) } else { REG_WRITE(PORT_HOTPLUG_EN, 0); REG_WRITE(PORT_HOTPLUG_STAT, REG_READ(PORT_HOTPLUG_STAT)); - } + } } static const char *force_audio_names[] = { @@ -568,9 +575,10 @@ static const struct psb_offset cdv_regmap[2] = { static int cdv_chip_setup(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); INIT_WORK(&dev_priv->hotplug_work, cdv_hotplug_work_func); - if (pci_enable_msi(dev->pdev)) + if (pci_enable_msi(pdev)) dev_warn(dev->dev, "Enabling MSI failed!\n"); dev_priv->regmap = cdv_regmap; gma_get_core_freq(dev); diff --git a/drivers/gpu/drm/gma500/cdv_intel_crt.c b/drivers/gpu/drm/gma500/cdv_intel_crt.c index 88535f5aacc5..c48c9d322dfb 100644 --- a/drivers/gpu/drm/gma500/cdv_intel_crt.c +++ b/drivers/gpu/drm/gma500/cdv_intel_crt.c @@ -127,7 +127,7 @@ static void cdv_intel_crt_mode_set(struct drm_encoder *encoder, } -/** +/* * Uses CRT_HOTPLUG_EN and CRT_HOTPLUG_STAT to detect CRT presence. * * \return true if CRT is connected. @@ -278,8 +278,7 @@ void cdv_intel_crt_init(struct drm_device *dev, gma_encoder->ddc_bus = psb_intel_i2c_create(dev, i2c_reg, "CRTDDC_A"); if (!gma_encoder->ddc_bus) { - dev_printk(KERN_ERR, &dev->pdev->dev, "DDC bus registration " - "failed.\n"); + dev_printk(KERN_ERR, dev->dev, "DDC bus registration failed.\n"); goto failed_ddc; } diff --git a/drivers/gpu/drm/gma500/cdv_intel_display.c b/drivers/gpu/drm/gma500/cdv_intel_display.c index 686385a66167..5d3302249779 100644 --- a/drivers/gpu/drm/gma500/cdv_intel_display.c +++ b/drivers/gpu/drm/gma500/cdv_intel_display.c @@ -551,7 +551,7 @@ void cdv_update_wm(struct drm_device *dev, struct drm_crtc *crtc) } } -/** +/* * Return the pipe currently connected to the panel fitter, * or -1 if the panel fitter is not present or not in use */ diff --git a/drivers/gpu/drm/gma500/cdv_intel_dp.c b/drivers/gpu/drm/gma500/cdv_intel_dp.c index bfd9a15d63b1..6d3ada39ff86 100644 --- a/drivers/gpu/drm/gma500/cdv_intel_dp.c +++ b/drivers/gpu/drm/gma500/cdv_intel_dp.c @@ -306,7 +306,7 @@ static uint32_t dp_vswing_premph_table[] = { }; /** * is_edp - is the given port attached to an eDP panel (either CPU or PCH) - * @intel_dp: DP struct + * @encoder: GMA encoder struct * * If a CPU or PCH DP output is attached to an eDP panel, this function * will return true, and false otherwise. @@ -1687,7 +1687,7 @@ static enum drm_connector_status cdv_dp_detect(struct gma_encoder *encoder) return status; } -/** +/* * Uses CRT_HOTPLUG_EN and CRT_HOTPLUG_STAT to detect DP connection. * * \return true if DP port is connected. diff --git a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c index 0d12c6ffbc40..e525689f84f0 100644 --- a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c +++ b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c @@ -22,9 +22,6 @@ * * Authors: * jim liu - * - * FIXME: - * We should probably make this generic and share it with Medfield */ #include @@ -56,7 +53,6 @@ struct mid_intel_hdmi_priv { bool has_hdmi_audio; /* Should set this when detect hotplug */ bool hdmi_device_connected; - struct mdfld_hdmi_i2c *i2c_bus; struct i2c_adapter *hdmi_i2c_adapter; /* for control functions */ struct drm_device *dev; }; diff --git a/drivers/gpu/drm/gma500/cdv_intel_lvds.c b/drivers/gpu/drm/gma500/cdv_intel_lvds.c index eaaf4efec217..5bff7d9e3aa6 100644 --- a/drivers/gpu/drm/gma500/cdv_intel_lvds.c +++ b/drivers/gpu/drm/gma500/cdv_intel_lvds.c @@ -74,7 +74,7 @@ static u32 cdv_intel_lvds_get_max_backlight(struct drm_device *dev) return retval; } -/** +/* * Sets the backlight level. * * level backlight level, from 0 to cdv_intel_lvds_get_max_backlight(). @@ -99,7 +99,7 @@ static void cdv_intel_lvds_set_backlight(struct drm_device *dev, int level) } } -/** +/* * Sets the power state for the panel. */ static void cdv_intel_lvds_set_power(struct drm_device *dev, @@ -291,7 +291,7 @@ static void cdv_intel_lvds_mode_set(struct drm_encoder *encoder, REG_WRITE(PFIT_CONTROL, pfit_control); } -/** +/* * Return the list of DDC modes if available, or the BIOS fixed mode otherwise. */ static int cdv_intel_lvds_get_modes(struct drm_connector *connector) @@ -471,6 +471,7 @@ static bool lvds_is_present_in_vbt(struct drm_device *dev, /** * cdv_intel_lvds_init - setup LVDS connectors on this device * @dev: drm device + * @mode_dev: PSB mode device * * Create the connector, register the LVDS DDC bus, and try to figure out what * modes we can display on the LVDS panel (if present). @@ -554,7 +555,7 @@ void cdv_intel_lvds_init(struct drm_device *dev, "LVDSBLC_B"); if (!gma_encoder->i2c_bus) { dev_printk(KERN_ERR, - &dev->pdev->dev, "I2C bus registration failed.\n"); + dev->dev, "I2C bus registration failed.\n"); goto failed_blc_i2c; } gma_encoder->i2c_bus->slave_addr = 0x2C; @@ -575,7 +576,7 @@ void cdv_intel_lvds_init(struct drm_device *dev, GPIOC, "LVDSDDC_C"); if (!gma_encoder->ddc_bus) { - dev_printk(KERN_ERR, &dev->pdev->dev, + dev_printk(KERN_ERR, dev->dev, "DDC bus registration " "failed.\n"); goto failed_ddc; } diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c index fc4fda1d258b..ebe9dccf2d83 100644 --- a/drivers/gpu/drm/gma500/framebuffer.c +++ b/drivers/gpu/drm/gma500/framebuffer.c @@ -159,7 +159,7 @@ static const struct fb_ops psbfb_unaccel_ops = { * @dev: our DRM device * @fb: framebuffer to set up * @mode_cmd: mode description - * @gt: backing object + * @obj: backing object * * Configure and fill in the boilerplate for our frame buffer. Return * 0 on success or an error code if we fail. @@ -197,7 +197,7 @@ static int psb_framebuffer_init(struct drm_device *dev, * psb_framebuffer_create - create a framebuffer backed by gt * @dev: our DRM device * @mode_cmd: the description of the requested mode - * @gt: the backing object + * @obj: the backing object * * Create a framebuffer object backed by the gt, and fill in the * boilerplate required @@ -252,7 +252,7 @@ static struct gtt_range *psbfb_alloc(struct drm_device *dev, int aligned_size) /** * psbfb_create - create a framebuffer - * @fbdev: the framebuffer device + * @fb_helper: the framebuffer helper * @sizes: specification of the layout * * Create a framebuffer to the specifications provided @@ -262,6 +262,7 @@ static int psbfb_create(struct drm_fb_helper *fb_helper, { struct drm_device *dev = fb_helper->dev; struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct fb_info *info; struct drm_framebuffer *fb; struct drm_mode_fb_cmd2 mode_cmd; @@ -325,8 +326,8 @@ static int psbfb_create(struct drm_fb_helper *fb_helper, drm_fb_helper_fill_info(info, fb_helper, sizes); - info->fix.mmio_start = pci_resource_start(dev->pdev, 0); - info->fix.mmio_len = pci_resource_len(dev->pdev, 0); + info->fix.mmio_start = pci_resource_start(pdev, 0); + info->fix.mmio_len = pci_resource_len(pdev, 0); /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ @@ -529,6 +530,7 @@ void psb_modeset_init(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; struct psb_intel_mode_device *mode_dev = &dev_priv->mode_dev; + struct pci_dev *pdev = to_pci_dev(dev->dev); int i; drm_mode_config_init(dev); @@ -540,8 +542,7 @@ void psb_modeset_init(struct drm_device *dev) /* set memory base */ /* Oaktrail and Poulsbo should use BAR 2*/ - pci_read_config_dword(dev->pdev, PSB_BSM, (u32 *) - &(dev->mode_config.fb_base)); + pci_read_config_dword(pdev, PSB_BSM, (u32 *)&(dev->mode_config.fb_base)); /* num pipes is 2 for PSB but 1 for Mrst */ for (i = 0; i < dev_priv->num_pipe; i++) diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c index db827e591403..fbf420051ef5 100644 --- a/drivers/gpu/drm/gma500/gem.c +++ b/drivers/gpu/drm/gma500/gem.c @@ -16,6 +16,7 @@ #include #include +#include "gem.h" #include "psb_drv.h" static vm_fault_t psb_gem_fault(struct vm_fault *vmf); @@ -49,6 +50,8 @@ const struct drm_gem_object_funcs psb_gem_object_funcs = { * @dev: our device * @size: the size requested * @handlep: returned handle (opaque number) + * @stolen: unused + * @align: unused * * Create a GEM object, fill in the boilerplate and attach a handle to * it so that userspace can speak about it. This does the core work @@ -97,7 +100,7 @@ int psb_gem_create(struct drm_file *file, struct drm_device *dev, u64 size, /** * psb_gem_dumb_create - create a dumb buffer - * @drm_file: our client file + * @file: our client file * @dev: our device * @args: the requested arguments copied from userspace * @@ -116,7 +119,6 @@ int psb_gem_dumb_create(struct drm_file *file, struct drm_device *dev, /** * psb_gem_fault - pagefault handler for GEM objects - * @vma: the VMA of the GEM object * @vmf: fault detail * * Invoked when a fault occurs on an mmap of a GEM managed area. GEM diff --git a/drivers/gpu/drm/gma500/gem.h b/drivers/gpu/drm/gma500/gem.h index 3741a711b9fd..bae6454ead29 100644 --- a/drivers/gpu/drm/gma500/gem.h +++ b/drivers/gpu/drm/gma500/gem.h @@ -8,6 +8,8 @@ #ifndef _GEM_H #define _GEM_H +struct drm_device; + extern const struct drm_gem_object_funcs psb_gem_object_funcs; extern int psb_gem_create(struct drm_file *file, struct drm_device *dev, diff --git a/drivers/gpu/drm/gma500/gma_device.c b/drivers/gpu/drm/gma500/gma_device.c index 869f30392566..4c91e86f4b14 100644 --- a/drivers/gpu/drm/gma500/gma_device.c +++ b/drivers/gpu/drm/gma500/gma_device.c @@ -6,12 +6,14 @@ **************************************************************************/ #include "psb_drv.h" +#include "gma_device.h" void gma_get_core_freq(struct drm_device *dev) { uint32_t clock; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct pci_dev *pci_root = - pci_get_domain_bus_and_slot(pci_domain_nr(dev->pdev->bus), + pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus), 0, 0); struct drm_psb_private *dev_priv = dev->dev_private; diff --git a/drivers/gpu/drm/gma500/gma_display.c b/drivers/gpu/drm/gma500/gma_display.c index 3df6d6e850f5..b03f7b8241f2 100644 --- a/drivers/gpu/drm/gma500/gma_display.c +++ b/drivers/gpu/drm/gma500/gma_display.c @@ -20,7 +20,7 @@ #include "psb_intel_drv.h" #include "psb_intel_reg.h" -/** +/* * Returns whether any output on the specified pipe is of the specified type */ bool gma_pipe_has_type(struct drm_crtc *crtc, int type) @@ -180,7 +180,7 @@ int gma_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, u16 *blue, return 0; } -/** +/* * Sets the power management mode of the pipe and plane. * * This code should probably grow support for turning the cursor off and back @@ -559,14 +559,14 @@ int gma_crtc_set_config(struct drm_mode_set *set, if (!dev_priv->rpm_enabled) return drm_crtc_helper_set_config(set, ctx); - pm_runtime_forbid(&dev->pdev->dev); + pm_runtime_forbid(dev->dev); ret = drm_crtc_helper_set_config(set, ctx); - pm_runtime_allow(&dev->pdev->dev); + pm_runtime_allow(dev->dev); return ret; } -/** +/* * Save HW states of given crtc */ void gma_crtc_save(struct drm_crtc *crtc) @@ -609,7 +609,7 @@ void gma_crtc_save(struct drm_crtc *crtc) crtc_state->savePalette[i] = REG_READ(palette_reg + (i << 2)); } -/** +/* * Restore HW states of given crtc */ void gma_crtc_restore(struct drm_crtc *crtc) diff --git a/drivers/gpu/drm/gma500/gtt.c b/drivers/gpu/drm/gma500/gtt.c index d246b1f70366..e884750bc123 100644 --- a/drivers/gpu/drm/gma500/gtt.c +++ b/drivers/gpu/drm/gma500/gtt.c @@ -340,13 +340,14 @@ static void psb_gtt_alloc(struct drm_device *dev) void psb_gtt_takedown(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); if (dev_priv->gtt_map) { iounmap(dev_priv->gtt_map); dev_priv->gtt_map = NULL; } if (dev_priv->gtt_initialized) { - pci_write_config_word(dev->pdev, PSB_GMCH_CTRL, + pci_write_config_word(pdev, PSB_GMCH_CTRL, dev_priv->gmch_ctrl); PSB_WVDC32(dev_priv->pge_ctl, PSB_PGETBL_CTL); (void) PSB_RVDC32(PSB_PGETBL_CTL); @@ -358,6 +359,7 @@ void psb_gtt_takedown(struct drm_device *dev) int psb_gtt_init(struct drm_device *dev, int resume) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); unsigned gtt_pages; unsigned long stolen_size, vram_stolen_size; unsigned i, num_pages; @@ -376,8 +378,8 @@ int psb_gtt_init(struct drm_device *dev, int resume) pg = &dev_priv->gtt; /* Enable the GTT */ - pci_read_config_word(dev->pdev, PSB_GMCH_CTRL, &dev_priv->gmch_ctrl); - pci_write_config_word(dev->pdev, PSB_GMCH_CTRL, + pci_read_config_word(pdev, PSB_GMCH_CTRL, &dev_priv->gmch_ctrl); + pci_write_config_word(pdev, PSB_GMCH_CTRL, dev_priv->gmch_ctrl | _PSB_GMCH_ENABLED); dev_priv->pge_ctl = PSB_RVDC32(PSB_PGETBL_CTL); @@ -397,8 +399,8 @@ int psb_gtt_init(struct drm_device *dev, int resume) */ pg->mmu_gatt_start = 0xE0000000; - pg->gtt_start = pci_resource_start(dev->pdev, PSB_GTT_RESOURCE); - gtt_pages = pci_resource_len(dev->pdev, PSB_GTT_RESOURCE) + pg->gtt_start = pci_resource_start(pdev, PSB_GTT_RESOURCE); + gtt_pages = pci_resource_len(pdev, PSB_GTT_RESOURCE) >> PAGE_SHIFT; /* CDV doesn't report this. In which case the system has 64 gtt pages */ if (pg->gtt_start == 0 || gtt_pages == 0) { @@ -407,10 +409,10 @@ int psb_gtt_init(struct drm_device *dev, int resume) pg->gtt_start = dev_priv->pge_ctl; } - pg->gatt_start = pci_resource_start(dev->pdev, PSB_GATT_RESOURCE); - pg->gatt_pages = pci_resource_len(dev->pdev, PSB_GATT_RESOURCE) + pg->gatt_start = pci_resource_start(pdev, PSB_GATT_RESOURCE); + pg->gatt_pages = pci_resource_len(pdev, PSB_GATT_RESOURCE) >> PAGE_SHIFT; - dev_priv->gtt_mem = &dev->pdev->resource[PSB_GATT_RESOURCE]; + dev_priv->gtt_mem = &pdev->resource[PSB_GATT_RESOURCE]; if (pg->gatt_pages == 0 || pg->gatt_start == 0) { static struct resource fudge; /* Preferably peppermint */ @@ -431,7 +433,7 @@ int psb_gtt_init(struct drm_device *dev, int resume) dev_priv->gtt_mem = &fudge; } - pci_read_config_dword(dev->pdev, PSB_BSM, &dev_priv->stolen_base); + pci_read_config_dword(pdev, PSB_BSM, &dev_priv->stolen_base); vram_stolen_size = pg->gtt_phys_start - dev_priv->stolen_base - PAGE_SIZE; diff --git a/drivers/gpu/drm/gma500/intel_bios.c b/drivers/gpu/drm/gma500/intel_bios.c index 8ad6337eeba3..d838369f0119 100644 --- a/drivers/gpu/drm/gma500/intel_bios.c +++ b/drivers/gpu/drm/gma500/intel_bios.c @@ -50,7 +50,7 @@ parse_edp(struct drm_psb_private *dev_priv, struct bdb_header *bdb) uint8_t panel_type; edp = find_section(bdb, BDB_EDP); - + dev_priv->edp.bpp = 18; if (!edp) { if (dev_priv->edp.support) { @@ -80,7 +80,7 @@ parse_edp(struct drm_psb_private *dev_priv, struct bdb_header *bdb) dev_priv->edp.pps = *edp_pps; DRM_DEBUG_KMS("EDP timing in vbt t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", - dev_priv->edp.pps.t1_t3, dev_priv->edp.pps.t8, + dev_priv->edp.pps.t1_t3, dev_priv->edp.pps.t8, dev_priv->edp.pps.t9, dev_priv->edp.pps.t10, dev_priv->edp.pps.t11_t12); @@ -516,7 +516,7 @@ parse_device_mapping(struct drm_psb_private *dev_priv, int psb_intel_init_bios(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct vbt_header *vbt = NULL; struct bdb_header *bdb = NULL; u8 __iomem *bios = NULL; @@ -574,7 +574,7 @@ int psb_intel_init_bios(struct drm_device *dev) return 0; } -/** +/* * Destroy and free VBT data */ void psb_intel_destroy_bios(struct drm_device *dev) diff --git a/drivers/gpu/drm/gma500/intel_gmbus.c b/drivers/gpu/drm/gma500/intel_gmbus.c index a083fbfe35b8..370bd6451bd9 100644 --- a/drivers/gpu/drm/gma500/intel_gmbus.c +++ b/drivers/gpu/drm/gma500/intel_gmbus.c @@ -196,7 +196,7 @@ intel_gpio_create(struct drm_psb_private *dev_priv, u32 pin) "gma500 GPIO%c", "?BACDE?F"[pin]); gpio->adapter.owner = THIS_MODULE; gpio->adapter.algo_data = &gpio->algo; - gpio->adapter.dev.parent = &dev_priv->dev->pdev->dev; + gpio->adapter.dev.parent = dev_priv->dev->dev; gpio->algo.setsda = set_data; gpio->algo.setscl = set_clock; gpio->algo.getsda = get_data; @@ -417,7 +417,7 @@ int gma_intel_setup_gmbus(struct drm_device *dev) "gma500 gmbus %s", names[i]); - bus->adapter.dev.parent = &dev->pdev->dev; + bus->adapter.dev.parent = dev->dev; bus->adapter.algo_data = dev_priv; bus->adapter.algo = &gmbus_algorithm; diff --git a/drivers/gpu/drm/gma500/intel_i2c.c b/drivers/gpu/drm/gma500/intel_i2c.c index de8810188190..5e1b4d70c317 100644 --- a/drivers/gpu/drm/gma500/intel_i2c.c +++ b/drivers/gpu/drm/gma500/intel_i2c.c @@ -85,7 +85,6 @@ static void set_data(void *data, int state_high) /** * psb_intel_i2c_create - instantiate an Intel i2c bus using the specified GPIO reg * @dev: DRM device - * @output: driver specific output device * @reg: GPIO reg to use * @name: name for this bus * @@ -117,7 +116,7 @@ struct psb_intel_i2c_chan *psb_intel_i2c_create(struct drm_device *dev, snprintf(chan->adapter.name, I2C_NAME_SIZE, "intel drm %s", name); chan->adapter.owner = THIS_MODULE; chan->adapter.algo_data = &chan->algo; - chan->adapter.dev.parent = &dev->pdev->dev; + chan->adapter.dev.parent = dev->dev; chan->algo.setsda = set_data; chan->algo.setscl = set_clock; chan->algo.getsda = get_data; @@ -145,7 +144,7 @@ out_free: /** * psb_intel_i2c_destroy - unregister and free i2c bus resources - * @output: channel to free + * @chan: channel to free * * Unregister the adapter from the i2c layer, then free the structure. */ diff --git a/drivers/gpu/drm/gma500/mdfld_device.c b/drivers/gpu/drm/gma500/mdfld_device.c deleted file mode 100644 index b83d59b21de5..000000000000 --- a/drivers/gpu/drm/gma500/mdfld_device.c +++ /dev/null @@ -1,562 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/************************************************************************** - * Copyright (c) 2011, Intel Corporation. - * All Rights Reserved. - * - **************************************************************************/ - -#include -#include - -#include - -#include "mdfld_dsi_output.h" -#include "mdfld_output.h" -#include "mid_bios.h" -#include "psb_drv.h" -#include "tc35876x-dsi-lvds.h" - -#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE - -#define MRST_BLC_MAX_PWM_REG_FREQ 0xFFFF -#define BLC_PWM_PRECISION_FACTOR 100 /* 10000000 */ -#define BLC_PWM_FREQ_CALC_CONSTANT 32 -#define MHz 1000000 -#define BRIGHTNESS_MIN_LEVEL 1 -#define BRIGHTNESS_MAX_LEVEL 100 -#define BRIGHTNESS_MASK 0xFF -#define BLC_POLARITY_NORMAL 0 -#define BLC_POLARITY_INVERSE 1 -#define BLC_ADJUSTMENT_MAX 100 - -#define MDFLD_BLC_PWM_PRECISION_FACTOR 10 -#define MDFLD_BLC_MAX_PWM_REG_FREQ 0xFFFE -#define MDFLD_BLC_MIN_PWM_REG_FREQ 0x2 - -#define MDFLD_BACKLIGHT_PWM_POLARITY_BIT_CLEAR (0xFFFE) -#define MDFLD_BACKLIGHT_PWM_CTL_SHIFT (16) - -static struct backlight_device *mdfld_backlight_device; - -int mdfld_set_brightness(struct backlight_device *bd) -{ - struct drm_device *dev = - (struct drm_device *)bl_get_data(mdfld_backlight_device); - struct drm_psb_private *dev_priv = dev->dev_private; - int level = bd->props.brightness; - - DRM_DEBUG_DRIVER("backlight level set to %d\n", level); - - /* Perform value bounds checking */ - if (level < BRIGHTNESS_MIN_LEVEL) - level = BRIGHTNESS_MIN_LEVEL; - - if (gma_power_begin(dev, false)) { - u32 adjusted_level = 0; - - /* - * Adjust the backlight level with the percent in - * dev_priv->blc_adj2 - */ - adjusted_level = level * dev_priv->blc_adj2; - adjusted_level = adjusted_level / BLC_ADJUSTMENT_MAX; - dev_priv->brightness_adjusted = adjusted_level; - - if (mdfld_get_panel_type(dev, 0) == TC35876X) { - if (dev_priv->dpi_panel_on[0] || - dev_priv->dpi_panel_on[2]) - tc35876x_brightness_control(dev, - dev_priv->brightness_adjusted); - } else { - if (dev_priv->dpi_panel_on[0]) - mdfld_dsi_brightness_control(dev, 0, - dev_priv->brightness_adjusted); - } - - if (dev_priv->dpi_panel_on[2]) - mdfld_dsi_brightness_control(dev, 2, - dev_priv->brightness_adjusted); - gma_power_end(dev); - } - - /* cache the brightness for later use */ - dev_priv->brightness = level; - return 0; -} - -static int mdfld_get_brightness(struct backlight_device *bd) -{ - struct drm_device *dev = - (struct drm_device *)bl_get_data(mdfld_backlight_device); - struct drm_psb_private *dev_priv = dev->dev_private; - - DRM_DEBUG_DRIVER("brightness = 0x%x \n", dev_priv->brightness); - - /* return locally cached var instead of HW read (due to DPST etc.) */ - return dev_priv->brightness; -} - -static const struct backlight_ops mdfld_ops = { - .get_brightness = mdfld_get_brightness, - .update_status = mdfld_set_brightness, -}; - -static int device_backlight_init(struct drm_device *dev) -{ - struct drm_psb_private *dev_priv = (struct drm_psb_private *) - dev->dev_private; - - dev_priv->blc_adj1 = BLC_ADJUSTMENT_MAX; - dev_priv->blc_adj2 = BLC_ADJUSTMENT_MAX; - - return 0; -} - -static int mdfld_backlight_init(struct drm_device *dev) -{ - struct backlight_properties props; - int ret = 0; - - memset(&props, 0, sizeof(struct backlight_properties)); - props.max_brightness = BRIGHTNESS_MAX_LEVEL; - props.type = BACKLIGHT_PLATFORM; - mdfld_backlight_device = backlight_device_register("mdfld-bl", - NULL, (void *)dev, &mdfld_ops, &props); - - if (IS_ERR(mdfld_backlight_device)) - return PTR_ERR(mdfld_backlight_device); - - ret = device_backlight_init(dev); - if (ret) - return ret; - - mdfld_backlight_device->props.brightness = BRIGHTNESS_MAX_LEVEL; - mdfld_backlight_device->props.max_brightness = BRIGHTNESS_MAX_LEVEL; - backlight_update_status(mdfld_backlight_device); - return 0; -} -#endif - -struct backlight_device *mdfld_get_backlight_device(void) -{ -#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE - return mdfld_backlight_device; -#else - return NULL; -#endif -} - -/* - * mdfld_save_display_registers - * - * Description: We are going to suspend so save current display - * register state. - * - * Notes: FIXME_JLIU7 need to add the support for DPI MIPI & HDMI audio - */ -static int mdfld_save_display_registers(struct drm_device *dev, int pipenum) -{ - struct drm_psb_private *dev_priv = dev->dev_private; - struct medfield_state *regs = &dev_priv->regs.mdfld; - struct psb_pipe *pipe = &dev_priv->regs.pipe[pipenum]; - const struct psb_offset *map = &dev_priv->regmap[pipenum]; - int i; - u32 *mipi_val; - - /* register */ - u32 mipi_reg = MIPI; - - switch (pipenum) { - case 0: - mipi_val = ®s->saveMIPI; - break; - case 1: - mipi_val = ®s->saveMIPI; - break; - case 2: - /* register */ - mipi_reg = MIPI_C; - /* pointer to values */ - mipi_val = ®s->saveMIPI_C; - break; - default: - DRM_ERROR("%s, invalid pipe number.\n", __func__); - return -EINVAL; - } - - /* Pipe & plane A info */ - pipe->dpll = PSB_RVDC32(map->dpll); - pipe->fp0 = PSB_RVDC32(map->fp0); - pipe->conf = PSB_RVDC32(map->conf); - pipe->htotal = PSB_RVDC32(map->htotal); - pipe->hblank = PSB_RVDC32(map->hblank); - pipe->hsync = PSB_RVDC32(map->hsync); - pipe->vtotal = PSB_RVDC32(map->vtotal); - pipe->vblank = PSB_RVDC32(map->vblank); - pipe->vsync = PSB_RVDC32(map->vsync); - pipe->src = PSB_RVDC32(map->src); - pipe->stride = PSB_RVDC32(map->stride); - pipe->linoff = PSB_RVDC32(map->linoff); - pipe->tileoff = PSB_RVDC32(map->tileoff); - pipe->size = PSB_RVDC32(map->size); - pipe->pos = PSB_RVDC32(map->pos); - pipe->surf = PSB_RVDC32(map->surf); - pipe->cntr = PSB_RVDC32(map->cntr); - pipe->status = PSB_RVDC32(map->status); - - /*save palette (gamma) */ - for (i = 0; i < 256; i++) - pipe->palette[i] = PSB_RVDC32(map->palette + (i << 2)); - - if (pipenum == 1) { - regs->savePFIT_CONTROL = PSB_RVDC32(PFIT_CONTROL); - regs->savePFIT_PGM_RATIOS = PSB_RVDC32(PFIT_PGM_RATIOS); - - regs->saveHDMIPHYMISCCTL = PSB_RVDC32(HDMIPHYMISCCTL); - regs->saveHDMIB_CONTROL = PSB_RVDC32(HDMIB_CONTROL); - return 0; - } - - *mipi_val = PSB_RVDC32(mipi_reg); - return 0; -} - -/* - * mdfld_restore_display_registers - * - * Description: We are going to resume so restore display register state. - * - * Notes: FIXME_JLIU7 need to add the support for DPI MIPI & HDMI audio - */ -static int mdfld_restore_display_registers(struct drm_device *dev, int pipenum) -{ - /* To get panel out of ULPS mode. */ - u32 temp = 0; - u32 device_ready_reg = DEVICE_READY_REG; - struct drm_psb_private *dev_priv = dev->dev_private; - struct mdfld_dsi_config *dsi_config = NULL; - struct medfield_state *regs = &dev_priv->regs.mdfld; - struct psb_pipe *pipe = &dev_priv->regs.pipe[pipenum]; - const struct psb_offset *map = &dev_priv->regmap[pipenum]; - u32 i; - u32 dpll; - u32 timeout = 0; - - /* register */ - u32 mipi_reg = MIPI; - - /* values */ - u32 dpll_val = pipe->dpll; - u32 mipi_val = regs->saveMIPI; - - switch (pipenum) { - case 0: - dpll_val &= ~DPLL_VCO_ENABLE; - dsi_config = dev_priv->dsi_configs[0]; - break; - case 1: - dpll_val &= ~DPLL_VCO_ENABLE; - break; - case 2: - mipi_reg = MIPI_C; - mipi_val = regs->saveMIPI_C; - dsi_config = dev_priv->dsi_configs[1]; - break; - default: - DRM_ERROR("%s, invalid pipe number.\n", __func__); - return -EINVAL; - } - - /*make sure VGA plane is off. it initializes to on after reset!*/ - PSB_WVDC32(0x80000000, VGACNTRL); - - if (pipenum == 1) { - PSB_WVDC32(dpll_val & ~DPLL_VCO_ENABLE, map->dpll); - PSB_RVDC32(map->dpll); - - PSB_WVDC32(pipe->fp0, map->fp0); - } else { - - dpll = PSB_RVDC32(map->dpll); - - if (!(dpll & DPLL_VCO_ENABLE)) { - - /* When ungating power of DPLL, needs to wait 0.5us - before enable the VCO */ - if (dpll & MDFLD_PWR_GATE_EN) { - dpll &= ~MDFLD_PWR_GATE_EN; - PSB_WVDC32(dpll, map->dpll); - /* FIXME_MDFLD PO - change 500 to 1 after PO */ - udelay(500); - } - - PSB_WVDC32(pipe->fp0, map->fp0); - PSB_WVDC32(dpll_val, map->dpll); - /* FIXME_MDFLD PO - change 500 to 1 after PO */ - udelay(500); - - dpll_val |= DPLL_VCO_ENABLE; - PSB_WVDC32(dpll_val, map->dpll); - PSB_RVDC32(map->dpll); - - /* wait for DSI PLL to lock */ - while (timeout < 20000 && - !(PSB_RVDC32(map->conf) & PIPECONF_DSIPLL_LOCK)) { - udelay(150); - timeout++; - } - - if (timeout == 20000) { - DRM_ERROR("%s, can't lock DSIPLL.\n", - __func__); - return -EINVAL; - } - } - } - /* Restore mode */ - PSB_WVDC32(pipe->htotal, map->htotal); - PSB_WVDC32(pipe->hblank, map->hblank); - PSB_WVDC32(pipe->hsync, map->hsync); - PSB_WVDC32(pipe->vtotal, map->vtotal); - PSB_WVDC32(pipe->vblank, map->vblank); - PSB_WVDC32(pipe->vsync, map->vsync); - PSB_WVDC32(pipe->src, map->src); - PSB_WVDC32(pipe->status, map->status); - - /*set up the plane*/ - PSB_WVDC32(pipe->stride, map->stride); - PSB_WVDC32(pipe->linoff, map->linoff); - PSB_WVDC32(pipe->tileoff, map->tileoff); - PSB_WVDC32(pipe->size, map->size); - PSB_WVDC32(pipe->pos, map->pos); - PSB_WVDC32(pipe->surf, map->surf); - - if (pipenum == 1) { - /* restore palette (gamma) */ - /* udelay(50000); */ - for (i = 0; i < 256; i++) - PSB_WVDC32(pipe->palette[i], map->palette + (i << 2)); - - PSB_WVDC32(regs->savePFIT_CONTROL, PFIT_CONTROL); - PSB_WVDC32(regs->savePFIT_PGM_RATIOS, PFIT_PGM_RATIOS); - - /*TODO: resume HDMI port */ - - /*TODO: resume pipe*/ - - /*enable the plane*/ - PSB_WVDC32(pipe->cntr & ~DISPLAY_PLANE_ENABLE, map->cntr); - - return 0; - } - - /*set up pipe related registers*/ - PSB_WVDC32(mipi_val, mipi_reg); - - /*setup MIPI adapter + MIPI IP registers*/ - if (dsi_config) - mdfld_dsi_controller_init(dsi_config, pipenum); - - if (in_atomic() || in_interrupt()) - mdelay(20); - else - msleep(20); - - /*enable the plane*/ - PSB_WVDC32(pipe->cntr, map->cntr); - - if (in_atomic() || in_interrupt()) - mdelay(20); - else - msleep(20); - - /* LP Hold Release */ - temp = REG_READ(mipi_reg); - temp |= LP_OUTPUT_HOLD_RELEASE; - REG_WRITE(mipi_reg, temp); - mdelay(1); - - - /* Set DSI host to exit from Utra Low Power State */ - temp = REG_READ(device_ready_reg); - temp &= ~ULPS_MASK; - temp |= 0x3; - temp |= EXIT_ULPS_DEV_READY; - REG_WRITE(device_ready_reg, temp); - mdelay(1); - - temp = REG_READ(device_ready_reg); - temp &= ~ULPS_MASK; - temp |= EXITING_ULPS; - REG_WRITE(device_ready_reg, temp); - mdelay(1); - - /*enable the pipe*/ - PSB_WVDC32(pipe->conf, map->conf); - - /* restore palette (gamma) */ - /* udelay(50000); */ - for (i = 0; i < 256; i++) - PSB_WVDC32(pipe->palette[i], map->palette + (i << 2)); - - return 0; -} - -static int mdfld_save_registers(struct drm_device *dev) -{ - /* mdfld_save_cursor_overlay_registers(dev); */ - mdfld_save_display_registers(dev, 0); - mdfld_save_display_registers(dev, 2); - mdfld_disable_crtc(dev, 0); - mdfld_disable_crtc(dev, 2); - - return 0; -} - -static int mdfld_restore_registers(struct drm_device *dev) -{ - mdfld_restore_display_registers(dev, 2); - mdfld_restore_display_registers(dev, 0); - /* mdfld_restore_cursor_overlay_registers(dev); */ - - return 0; -} - -static int mdfld_power_down(struct drm_device *dev) -{ - /* FIXME */ - return 0; -} - -static int mdfld_power_up(struct drm_device *dev) -{ - /* FIXME */ - return 0; -} - -/* Medfield */ -static const struct psb_offset mdfld_regmap[3] = { - { - .fp0 = MRST_FPA0, - .fp1 = MRST_FPA1, - .cntr = DSPACNTR, - .conf = PIPEACONF, - .src = PIPEASRC, - .dpll = MRST_DPLL_A, - .htotal = HTOTAL_A, - .hblank = HBLANK_A, - .hsync = HSYNC_A, - .vtotal = VTOTAL_A, - .vblank = VBLANK_A, - .vsync = VSYNC_A, - .stride = DSPASTRIDE, - .size = DSPASIZE, - .pos = DSPAPOS, - .surf = DSPASURF, - .addr = MRST_DSPABASE, - .status = PIPEASTAT, - .linoff = DSPALINOFF, - .tileoff = DSPATILEOFF, - .palette = PALETTE_A, - }, - { - .fp0 = MDFLD_DPLL_DIV0, - .cntr = DSPBCNTR, - .conf = PIPEBCONF, - .src = PIPEBSRC, - .dpll = MDFLD_DPLL_B, - .htotal = HTOTAL_B, - .hblank = HBLANK_B, - .hsync = HSYNC_B, - .vtotal = VTOTAL_B, - .vblank = VBLANK_B, - .vsync = VSYNC_B, - .stride = DSPBSTRIDE, - .size = DSPBSIZE, - .pos = DSPBPOS, - .surf = DSPBSURF, - .addr = MRST_DSPBBASE, - .status = PIPEBSTAT, - .linoff = DSPBLINOFF, - .tileoff = DSPBTILEOFF, - .palette = PALETTE_B, - }, - { - .fp0 = MRST_FPA0, /* This is what the old code did ?? */ - .cntr = DSPCCNTR, - .conf = PIPECCONF, - .src = PIPECSRC, - /* No DPLL_C */ - .dpll = MRST_DPLL_A, - .htotal = HTOTAL_C, - .hblank = HBLANK_C, - .hsync = HSYNC_C, - .vtotal = VTOTAL_C, - .vblank = VBLANK_C, - .vsync = VSYNC_C, - .stride = DSPCSTRIDE, - .size = DSPBSIZE, - .pos = DSPCPOS, - .surf = DSPCSURF, - .addr = MDFLD_DSPCBASE, - .status = PIPECSTAT, - .linoff = DSPCLINOFF, - .tileoff = DSPCTILEOFF, - .palette = PALETTE_C, - }, -}; - -/* - * The GPIO lines for resetting DSI pipe 0 and 2 are available in the - * PCI device 0000:00:0c.0 on the Medfield. - */ -static struct gpiod_lookup_table mdfld_dsi_pipe_gpio_table = { - .table = { - GPIO_LOOKUP("0000:00:0c.0", 128, "dsi-pipe0-reset", - GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", 34, "dsi-pipe2-reset", - GPIO_ACTIVE_HIGH), - { }, - }, -}; - -static int mdfld_chip_setup(struct drm_device *dev) -{ - struct drm_psb_private *dev_priv = dev->dev_private; - if (pci_enable_msi(dev->pdev)) - dev_warn(dev->dev, "Enabling MSI failed!\n"); - dev_priv->regmap = mdfld_regmap; - - /* Associate the GPIO lines with the DRM device */ - mdfld_dsi_pipe_gpio_table.dev_id = dev_name(dev->dev); - gpiod_add_lookup_table(&mdfld_dsi_pipe_gpio_table); - - return mid_chip_setup(dev); -} - -const struct psb_ops mdfld_chip_ops = { - .name = "mdfld", - .pipes = 3, - .crtcs = 3, - .lvds_mask = (1 << 1), - .hdmi_mask = (1 << 1), - .cursor_needs_phys = 0, - .sgx_offset = MRST_SGX_OFFSET, - - .chip_setup = mdfld_chip_setup, - .crtc_helper = &mdfld_helper_funcs, - .crtc_funcs = &psb_intel_crtc_funcs, - - .output_init = mdfld_output_init, - -#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE - .backlight_init = mdfld_backlight_init, -#endif - - .save_regs = mdfld_save_registers, - .restore_regs = mdfld_restore_registers, - .save_crtc = gma_crtc_save, - .restore_crtc = gma_crtc_restore, - .power_down = mdfld_power_down, - .power_up = mdfld_power_up, -}; diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_dpi.c b/drivers/gpu/drm/gma500/mdfld_dsi_dpi.c deleted file mode 100644 index ae1223f631a7..000000000000 --- a/drivers/gpu/drm/gma500/mdfld_dsi_dpi.c +++ /dev/null @@ -1,1017 +0,0 @@ -/* - * Copyright © 2010 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * jim liu - * Jackie Li - */ - -#include - -#include - -#include "mdfld_dsi_dpi.h" -#include "mdfld_dsi_pkg_sender.h" -#include "mdfld_output.h" -#include "psb_drv.h" -#include "tc35876x-dsi-lvds.h" - -static void mdfld_dsi_dpi_shut_down(struct mdfld_dsi_dpi_output *output, - int pipe); - -static void mdfld_wait_for_HS_DATA_FIFO(struct drm_device *dev, u32 pipe) -{ - u32 gen_fifo_stat_reg = MIPI_GEN_FIFO_STAT_REG(pipe); - int timeout = 0; - - udelay(500); - - /* This will time out after approximately 2+ seconds */ - while ((timeout < 20000) && - (REG_READ(gen_fifo_stat_reg) & DSI_FIFO_GEN_HS_DATA_FULL)) { - udelay(100); - timeout++; - } - - if (timeout == 20000) - DRM_INFO("MIPI: HS Data FIFO was never cleared!\n"); -} - -static void mdfld_wait_for_HS_CTRL_FIFO(struct drm_device *dev, u32 pipe) -{ - u32 gen_fifo_stat_reg = MIPI_GEN_FIFO_STAT_REG(pipe); - int timeout = 0; - - udelay(500); - - /* This will time out after approximately 2+ seconds */ - while ((timeout < 20000) && (REG_READ(gen_fifo_stat_reg) - & DSI_FIFO_GEN_HS_CTRL_FULL)) { - udelay(100); - timeout++; - } - if (timeout == 20000) - DRM_INFO("MIPI: HS CMD FIFO was never cleared!\n"); -} - -static void mdfld_wait_for_DPI_CTRL_FIFO(struct drm_device *dev, u32 pipe) -{ - u32 gen_fifo_stat_reg = MIPI_GEN_FIFO_STAT_REG(pipe); - int timeout = 0; - - udelay(500); - - /* This will time out after approximately 2+ seconds */ - while ((timeout < 20000) && ((REG_READ(gen_fifo_stat_reg) & - DPI_FIFO_EMPTY) != DPI_FIFO_EMPTY)) { - udelay(100); - timeout++; - } - - if (timeout == 20000) - DRM_ERROR("MIPI: DPI FIFO was never cleared\n"); -} - -static void mdfld_wait_for_SPL_PKG_SENT(struct drm_device *dev, u32 pipe) -{ - u32 intr_stat_reg = MIPI_INTR_STAT_REG(pipe); - int timeout = 0; - - udelay(500); - - /* This will time out after approximately 2+ seconds */ - while ((timeout < 20000) && (!(REG_READ(intr_stat_reg) - & DSI_INTR_STATE_SPL_PKG_SENT))) { - udelay(100); - timeout++; - } - - if (timeout == 20000) - DRM_ERROR("MIPI: SPL_PKT_SENT_INTERRUPT was not sent successfully!\n"); -} - -/* For TC35876X */ - -static void dsi_set_device_ready_state(struct drm_device *dev, int state, - int pipe) -{ - REG_FLD_MOD(MIPI_DEVICE_READY_REG(pipe), !!state, 0, 0); -} - -static void dsi_set_pipe_plane_enable_state(struct drm_device *dev, - int state, int pipe) -{ - struct drm_psb_private *dev_priv = dev->dev_private; - u32 pipeconf_reg = PIPEACONF; - u32 dspcntr_reg = DSPACNTR; - - u32 dspcntr = dev_priv->dspcntr[pipe]; - u32 mipi = MIPI_PORT_EN | PASS_FROM_SPHY_TO_AFE | SEL_FLOPPED_HSTX; - - if (pipe) { - pipeconf_reg = PIPECCONF; - dspcntr_reg = DSPCCNTR; - } else - mipi &= (~0x03); - - if (state) { - /*Set up pipe */ - REG_WRITE(pipeconf_reg, BIT(31)); - - if (REG_BIT_WAIT(pipeconf_reg, 1, 30)) - dev_err(&dev->pdev->dev, "%s: Pipe enable timeout\n", - __func__); - - /*Set up display plane */ - REG_WRITE(dspcntr_reg, dspcntr); - } else { - u32 dspbase_reg = pipe ? MDFLD_DSPCBASE : MRST_DSPABASE; - - /* Put DSI lanes to ULPS to disable pipe */ - REG_FLD_MOD(MIPI_DEVICE_READY_REG(pipe), 2, 2, 1); - REG_READ(MIPI_DEVICE_READY_REG(pipe)); /* posted write? */ - - /* LP Hold */ - REG_FLD_MOD(MIPI_PORT_CONTROL(pipe), 0, 16, 16); - REG_READ(MIPI_PORT_CONTROL(pipe)); /* posted write? */ - - /* Disable display plane */ - REG_FLD_MOD(dspcntr_reg, 0, 31, 31); - - /* Flush the plane changes ??? posted write? */ - REG_WRITE(dspbase_reg, REG_READ(dspbase_reg)); - REG_READ(dspbase_reg); - - /* Disable PIPE */ - REG_FLD_MOD(pipeconf_reg, 0, 31, 31); - - if (REG_BIT_WAIT(pipeconf_reg, 0, 30)) - dev_err(&dev->pdev->dev, "%s: Pipe disable timeout\n", - __func__); - - if (REG_BIT_WAIT(MIPI_GEN_FIFO_STAT_REG(pipe), 1, 28)) - dev_err(&dev->pdev->dev, "%s: FIFO not empty\n", - __func__); - } -} - -static void mdfld_dsi_configure_down(struct mdfld_dsi_encoder *dsi_encoder, - int pipe) -{ - struct mdfld_dsi_dpi_output *dpi_output = - MDFLD_DSI_DPI_OUTPUT(dsi_encoder); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_encoder_get_config(dsi_encoder); - struct drm_device *dev = dsi_config->dev; - struct drm_psb_private *dev_priv = dev->dev_private; - - if (!dev_priv->dpi_panel_on[pipe]) { - dev_err(dev->dev, "DPI panel is already off\n"); - return; - } - tc35876x_toshiba_bridge_panel_off(dev); - tc35876x_set_bridge_reset_state(dev, 1); - dsi_set_pipe_plane_enable_state(dev, 0, pipe); - mdfld_dsi_dpi_shut_down(dpi_output, pipe); - dsi_set_device_ready_state(dev, 0, pipe); -} - -static void mdfld_dsi_configure_up(struct mdfld_dsi_encoder *dsi_encoder, - int pipe) -{ - struct mdfld_dsi_dpi_output *dpi_output = - MDFLD_DSI_DPI_OUTPUT(dsi_encoder); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_encoder_get_config(dsi_encoder); - struct drm_device *dev = dsi_config->dev; - struct drm_psb_private *dev_priv = dev->dev_private; - - if (dev_priv->dpi_panel_on[pipe]) { - dev_err(dev->dev, "DPI panel is already on\n"); - return; - } - - /* For resume path sequence */ - mdfld_dsi_dpi_shut_down(dpi_output, pipe); - dsi_set_device_ready_state(dev, 0, pipe); - - dsi_set_device_ready_state(dev, 1, pipe); - tc35876x_set_bridge_reset_state(dev, 0); - tc35876x_configure_lvds_bridge(dev); - mdfld_dsi_dpi_turn_on(dpi_output, pipe); /* Send turn on command */ - dsi_set_pipe_plane_enable_state(dev, 1, pipe); -} -/* End for TC35876X */ - -/* ************************************************************************* *\ - * FUNCTION: mdfld_dsi_tpo_ic_init - * - * DESCRIPTION: This function is called only by mrst_dsi_mode_set and - * restore_display_registers. since this function does not - * acquire the mutex, it is important that the calling function - * does! -\* ************************************************************************* */ -static void mdfld_dsi_tpo_ic_init(struct mdfld_dsi_config *dsi_config, u32 pipe) -{ - struct drm_device *dev = dsi_config->dev; - u32 dcsChannelNumber = dsi_config->channel_num; - u32 gen_data_reg = MIPI_HS_GEN_DATA_REG(pipe); - u32 gen_ctrl_reg = MIPI_HS_GEN_CTRL_REG(pipe); - u32 gen_ctrl_val = GEN_LONG_WRITE; - - DRM_INFO("Enter mrst init TPO MIPI display.\n"); - - gen_ctrl_val |= dcsChannelNumber << DCS_CHANNEL_NUMBER_POS; - - /* Flip page order */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00008036); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x02 << WORD_COUNTS_POS)); - - /* 0xF0 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x005a5af0); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x03 << WORD_COUNTS_POS)); - - /* Write protection key */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x005a5af1); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x03 << WORD_COUNTS_POS)); - - /* 0xFC */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x005a5afc); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x03 << WORD_COUNTS_POS)); - - /* 0xB7 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x770000b7); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000044); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x05 << WORD_COUNTS_POS)); - - /* 0xB6 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x000a0ab6); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x03 << WORD_COUNTS_POS)); - - /* 0xF2 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x081010f2); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x4a070708); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x000000c5); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x09 << WORD_COUNTS_POS)); - - /* 0xF8 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x024003f8); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x01030a04); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x0e020220); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000004); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x0d << WORD_COUNTS_POS)); - - /* 0xE2 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x398fc3e2); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x0000916f); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x06 << WORD_COUNTS_POS)); - - /* 0xB0 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x000000b0); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x02 << WORD_COUNTS_POS)); - - /* 0xF4 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x240242f4); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x78ee2002); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x2a071050); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x507fee10); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x10300710); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x14 << WORD_COUNTS_POS)); - - /* 0xBA */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x19fe07ba); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x101c0a31); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000010); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x09 << WORD_COUNTS_POS)); - - /* 0xBB */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x28ff07bb); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x24280a31); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000034); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x09 << WORD_COUNTS_POS)); - - /* 0xFB */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x535d05fb); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1b1a2130); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x221e180e); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x131d2120); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x535d0508); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1c1a2131); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x231f160d); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x111b2220); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x535c2008); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1f1d2433); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x2c251a10); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x2c34372d); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000023); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x31 << WORD_COUNTS_POS)); - - /* 0xFA */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x525c0bfa); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1c1c232f); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x2623190e); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x18212625); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x545d0d0e); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1e1d2333); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x26231a10); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1a222725); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x545d280f); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x21202635); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x31292013); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x31393d33); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000029); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x31 << WORD_COUNTS_POS)); - - /* Set DM */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x000100f7); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x03 << WORD_COUNTS_POS)); -} - -static u16 mdfld_dsi_dpi_to_byte_clock_count(int pixel_clock_count, - int num_lane, int bpp) -{ - return (u16)((pixel_clock_count * bpp) / (num_lane * 8)); -} - -/* - * Calculate the dpi time basing on a given drm mode @mode - * return 0 on success. - * FIXME: I was using proposed mode value for calculation, may need to - * use crtc mode values later - */ -int mdfld_dsi_dpi_timing_calculation(struct drm_display_mode *mode, - struct mdfld_dsi_dpi_timing *dpi_timing, - int num_lane, int bpp) -{ - int pclk_hsync, pclk_hfp, pclk_hbp, pclk_hactive; - int pclk_vsync, pclk_vfp, pclk_vbp; - - pclk_hactive = mode->hdisplay; - pclk_hfp = mode->hsync_start - mode->hdisplay; - pclk_hsync = mode->hsync_end - mode->hsync_start; - pclk_hbp = mode->htotal - mode->hsync_end; - - pclk_vfp = mode->vsync_start - mode->vdisplay; - pclk_vsync = mode->vsync_end - mode->vsync_start; - pclk_vbp = mode->vtotal - mode->vsync_end; - - /* - * byte clock counts were calculated by following formula - * bclock_count = pclk_count * bpp / num_lane / 8 - */ - dpi_timing->hsync_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_hsync, num_lane, bpp); - dpi_timing->hbp_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_hbp, num_lane, bpp); - dpi_timing->hfp_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_hfp, num_lane, bpp); - dpi_timing->hactive_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_hactive, num_lane, bpp); - dpi_timing->vsync_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_vsync, num_lane, bpp); - dpi_timing->vbp_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_vbp, num_lane, bpp); - dpi_timing->vfp_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_vfp, num_lane, bpp); - - return 0; -} - -void mdfld_dsi_dpi_controller_init(struct mdfld_dsi_config *dsi_config, - int pipe) -{ - struct drm_device *dev = dsi_config->dev; - int lane_count = dsi_config->lane_count; - struct mdfld_dsi_dpi_timing dpi_timing; - struct drm_display_mode *mode = dsi_config->mode; - u32 val; - - /*un-ready device*/ - REG_FLD_MOD(MIPI_DEVICE_READY_REG(pipe), 0, 0, 0); - - /*init dsi adapter before kicking off*/ - REG_WRITE(MIPI_CTRL_REG(pipe), 0x00000018); - - /*enable all interrupts*/ - REG_WRITE(MIPI_INTR_EN_REG(pipe), 0xffffffff); - - /*set up func_prg*/ - val = lane_count; - val |= dsi_config->channel_num << DSI_DPI_VIRT_CHANNEL_OFFSET; - - switch (dsi_config->bpp) { - case 16: - val |= DSI_DPI_COLOR_FORMAT_RGB565; - break; - case 18: - val |= DSI_DPI_COLOR_FORMAT_RGB666; - break; - case 24: - val |= DSI_DPI_COLOR_FORMAT_RGB888; - break; - default: - DRM_ERROR("unsupported color format, bpp = %d\n", - dsi_config->bpp); - } - REG_WRITE(MIPI_DSI_FUNC_PRG_REG(pipe), val); - - REG_WRITE(MIPI_HS_TX_TIMEOUT_REG(pipe), - (mode->vtotal * mode->htotal * dsi_config->bpp / - (8 * lane_count)) & DSI_HS_TX_TIMEOUT_MASK); - REG_WRITE(MIPI_LP_RX_TIMEOUT_REG(pipe), - 0xffff & DSI_LP_RX_TIMEOUT_MASK); - - /*max value: 20 clock cycles of txclkesc*/ - REG_WRITE(MIPI_TURN_AROUND_TIMEOUT_REG(pipe), - 0x14 & DSI_TURN_AROUND_TIMEOUT_MASK); - - /*min 21 txclkesc, max: ffffh*/ - REG_WRITE(MIPI_DEVICE_RESET_TIMER_REG(pipe), - 0xffff & DSI_RESET_TIMER_MASK); - - REG_WRITE(MIPI_DPI_RESOLUTION_REG(pipe), - mode->vdisplay << 16 | mode->hdisplay); - - /*set DPI timing registers*/ - mdfld_dsi_dpi_timing_calculation(mode, &dpi_timing, - dsi_config->lane_count, dsi_config->bpp); - - REG_WRITE(MIPI_HSYNC_COUNT_REG(pipe), - dpi_timing.hsync_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HBP_COUNT_REG(pipe), - dpi_timing.hbp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HFP_COUNT_REG(pipe), - dpi_timing.hfp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HACTIVE_COUNT_REG(pipe), - dpi_timing.hactive_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VSYNC_COUNT_REG(pipe), - dpi_timing.vsync_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VBP_COUNT_REG(pipe), - dpi_timing.vbp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VFP_COUNT_REG(pipe), - dpi_timing.vfp_count & DSI_DPI_TIMING_MASK); - - REG_WRITE(MIPI_HIGH_LOW_SWITCH_COUNT_REG(pipe), 0x46); - - /*min: 7d0 max: 4e20*/ - REG_WRITE(MIPI_INIT_COUNT_REG(pipe), 0x000007d0); - - /*set up video mode*/ - val = dsi_config->video_mode | DSI_DPI_COMPLETE_LAST_LINE; - REG_WRITE(MIPI_VIDEO_MODE_FORMAT_REG(pipe), val); - - REG_WRITE(MIPI_EOT_DISABLE_REG(pipe), 0x00000000); - - REG_WRITE(MIPI_LP_BYTECLK_REG(pipe), 0x00000004); - - /*TODO: figure out how to setup these registers*/ - if (mdfld_get_panel_type(dev, pipe) == TC35876X) - REG_WRITE(MIPI_DPHY_PARAM_REG(pipe), 0x2A0c6008); - else - REG_WRITE(MIPI_DPHY_PARAM_REG(pipe), 0x150c3408); - - REG_WRITE(MIPI_CLK_LANE_SWITCH_TIME_CNT_REG(pipe), (0xa << 16) | 0x14); - - if (mdfld_get_panel_type(dev, pipe) == TC35876X) - tc35876x_set_bridge_reset_state(dev, 0); /*Pull High Reset */ - - /*set device ready*/ - REG_FLD_MOD(MIPI_DEVICE_READY_REG(pipe), 1, 0, 0); -} - -void mdfld_dsi_dpi_turn_on(struct mdfld_dsi_dpi_output *output, int pipe) -{ - struct drm_device *dev = output->dev; - - /* clear special packet sent bit */ - if (REG_READ(MIPI_INTR_STAT_REG(pipe)) & DSI_INTR_STATE_SPL_PKG_SENT) - REG_WRITE(MIPI_INTR_STAT_REG(pipe), - DSI_INTR_STATE_SPL_PKG_SENT); - - /*send turn on package*/ - REG_WRITE(MIPI_DPI_CONTROL_REG(pipe), DSI_DPI_CTRL_HS_TURN_ON); - - /*wait for SPL_PKG_SENT interrupt*/ - mdfld_wait_for_SPL_PKG_SENT(dev, pipe); - - if (REG_READ(MIPI_INTR_STAT_REG(pipe)) & DSI_INTR_STATE_SPL_PKG_SENT) - REG_WRITE(MIPI_INTR_STAT_REG(pipe), - DSI_INTR_STATE_SPL_PKG_SENT); - - output->panel_on = 1; - - /* FIXME the following is disabled to WA the X slow start issue - for TMD panel - if (pipe == 2) - dev_priv->dpi_panel_on2 = true; - else if (pipe == 0) - dev_priv->dpi_panel_on = true; */ -} - -static void mdfld_dsi_dpi_shut_down(struct mdfld_dsi_dpi_output *output, - int pipe) -{ - struct drm_device *dev = output->dev; - - /*if output is on, or mode setting didn't happen, ignore this*/ - if ((!output->panel_on) || output->first_boot) { - output->first_boot = 0; - return; - } - - /* Wait for dpi fifo to empty */ - mdfld_wait_for_DPI_CTRL_FIFO(dev, pipe); - - /* Clear the special packet interrupt bit if set */ - if (REG_READ(MIPI_INTR_STAT_REG(pipe)) & DSI_INTR_STATE_SPL_PKG_SENT) - REG_WRITE(MIPI_INTR_STAT_REG(pipe), - DSI_INTR_STATE_SPL_PKG_SENT); - - if (REG_READ(MIPI_DPI_CONTROL_REG(pipe)) == DSI_DPI_CTRL_HS_SHUTDOWN) - goto shutdown_out; - - REG_WRITE(MIPI_DPI_CONTROL_REG(pipe), DSI_DPI_CTRL_HS_SHUTDOWN); - -shutdown_out: - output->panel_on = 0; - output->first_boot = 0; - - /* FIXME the following is disabled to WA the X slow start issue - for TMD panel - if (pipe == 2) - dev_priv->dpi_panel_on2 = false; - else if (pipe == 0) - dev_priv->dpi_panel_on = false; */ -} - -static void mdfld_dsi_dpi_set_power(struct drm_encoder *encoder, bool on) -{ - struct mdfld_dsi_encoder *dsi_encoder = mdfld_dsi_encoder(encoder); - struct mdfld_dsi_dpi_output *dpi_output = - MDFLD_DSI_DPI_OUTPUT(dsi_encoder); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_encoder_get_config(dsi_encoder); - int pipe = mdfld_dsi_encoder_get_pipe(dsi_encoder); - struct drm_device *dev = dsi_config->dev; - struct drm_psb_private *dev_priv = dev->dev_private; - - /*start up display island if it was shutdown*/ - if (!gma_power_begin(dev, true)) - return; - - if (on) { - if (mdfld_get_panel_type(dev, pipe) == TMD_VID) - mdfld_dsi_dpi_turn_on(dpi_output, pipe); - else if (mdfld_get_panel_type(dev, pipe) == TC35876X) - mdfld_dsi_configure_up(dsi_encoder, pipe); - else { - /*enable mipi port*/ - REG_WRITE(MIPI_PORT_CONTROL(pipe), - REG_READ(MIPI_PORT_CONTROL(pipe)) | BIT(31)); - REG_READ(MIPI_PORT_CONTROL(pipe)); - - mdfld_dsi_dpi_turn_on(dpi_output, pipe); - mdfld_dsi_tpo_ic_init(dsi_config, pipe); - } - dev_priv->dpi_panel_on[pipe] = true; - } else { - if (mdfld_get_panel_type(dev, pipe) == TMD_VID) - mdfld_dsi_dpi_shut_down(dpi_output, pipe); - else if (mdfld_get_panel_type(dev, pipe) == TC35876X) - mdfld_dsi_configure_down(dsi_encoder, pipe); - else { - mdfld_dsi_dpi_shut_down(dpi_output, pipe); - - /*disable mipi port*/ - REG_WRITE(MIPI_PORT_CONTROL(pipe), - REG_READ(MIPI_PORT_CONTROL(pipe)) & ~BIT(31)); - REG_READ(MIPI_PORT_CONTROL(pipe)); - } - dev_priv->dpi_panel_on[pipe] = false; - } - gma_power_end(dev); -} - -void mdfld_dsi_dpi_dpms(struct drm_encoder *encoder, int mode) -{ - mdfld_dsi_dpi_set_power(encoder, mode == DRM_MODE_DPMS_ON); -} - -bool mdfld_dsi_dpi_mode_fixup(struct drm_encoder *encoder, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) -{ - struct mdfld_dsi_encoder *dsi_encoder = mdfld_dsi_encoder(encoder); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_encoder_get_config(dsi_encoder); - struct drm_display_mode *fixed_mode = dsi_config->fixed_mode; - - if (fixed_mode) { - adjusted_mode->hdisplay = fixed_mode->hdisplay; - adjusted_mode->hsync_start = fixed_mode->hsync_start; - adjusted_mode->hsync_end = fixed_mode->hsync_end; - adjusted_mode->htotal = fixed_mode->htotal; - adjusted_mode->vdisplay = fixed_mode->vdisplay; - adjusted_mode->vsync_start = fixed_mode->vsync_start; - adjusted_mode->vsync_end = fixed_mode->vsync_end; - adjusted_mode->vtotal = fixed_mode->vtotal; - adjusted_mode->clock = fixed_mode->clock; - drm_mode_set_crtcinfo(adjusted_mode, CRTC_INTERLACE_HALVE_V); - } - return true; -} - -void mdfld_dsi_dpi_prepare(struct drm_encoder *encoder) -{ - mdfld_dsi_dpi_set_power(encoder, false); -} - -void mdfld_dsi_dpi_commit(struct drm_encoder *encoder) -{ - mdfld_dsi_dpi_set_power(encoder, true); -} - -/* For TC35876X */ -/* This functionality was implemented in FW in iCDK */ -/* But removed in DV0 and later. So need to add here. */ -static void mipi_set_properties(struct mdfld_dsi_config *dsi_config, int pipe) -{ - struct drm_device *dev = dsi_config->dev; - - REG_WRITE(MIPI_CTRL_REG(pipe), 0x00000018); - REG_WRITE(MIPI_INTR_EN_REG(pipe), 0xffffffff); - REG_WRITE(MIPI_HS_TX_TIMEOUT_REG(pipe), 0xffffff); - REG_WRITE(MIPI_LP_RX_TIMEOUT_REG(pipe), 0xffffff); - REG_WRITE(MIPI_TURN_AROUND_TIMEOUT_REG(pipe), 0x14); - REG_WRITE(MIPI_DEVICE_RESET_TIMER_REG(pipe), 0xff); - REG_WRITE(MIPI_HIGH_LOW_SWITCH_COUNT_REG(pipe), 0x25); - REG_WRITE(MIPI_INIT_COUNT_REG(pipe), 0xf0); - REG_WRITE(MIPI_EOT_DISABLE_REG(pipe), 0x00000000); - REG_WRITE(MIPI_LP_BYTECLK_REG(pipe), 0x00000004); - REG_WRITE(MIPI_DBI_BW_CTRL_REG(pipe), 0x00000820); - REG_WRITE(MIPI_CLK_LANE_SWITCH_TIME_CNT_REG(pipe), (0xa << 16) | 0x14); -} - -static void mdfld_mipi_set_video_timing(struct mdfld_dsi_config *dsi_config, - int pipe) -{ - struct drm_device *dev = dsi_config->dev; - struct mdfld_dsi_dpi_timing dpi_timing; - struct drm_display_mode *mode = dsi_config->mode; - - mdfld_dsi_dpi_timing_calculation(mode, &dpi_timing, - dsi_config->lane_count, - dsi_config->bpp); - - REG_WRITE(MIPI_DPI_RESOLUTION_REG(pipe), - mode->vdisplay << 16 | mode->hdisplay); - REG_WRITE(MIPI_HSYNC_COUNT_REG(pipe), - dpi_timing.hsync_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HBP_COUNT_REG(pipe), - dpi_timing.hbp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HFP_COUNT_REG(pipe), - dpi_timing.hfp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HACTIVE_COUNT_REG(pipe), - dpi_timing.hactive_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VSYNC_COUNT_REG(pipe), - dpi_timing.vsync_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VBP_COUNT_REG(pipe), - dpi_timing.vbp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VFP_COUNT_REG(pipe), - dpi_timing.vfp_count & DSI_DPI_TIMING_MASK); -} - -static void mdfld_mipi_config(struct mdfld_dsi_config *dsi_config, int pipe) -{ - struct drm_device *dev = dsi_config->dev; - int lane_count = dsi_config->lane_count; - - if (pipe) { - REG_WRITE(MIPI_PORT_CONTROL(0), 0x00000002); - REG_WRITE(MIPI_PORT_CONTROL(2), 0x80000000); - } else { - REG_WRITE(MIPI_PORT_CONTROL(0), 0x80010000); - REG_WRITE(MIPI_PORT_CONTROL(2), 0x00); - } - - REG_WRITE(MIPI_DPHY_PARAM_REG(pipe), 0x150A600F); - REG_WRITE(MIPI_VIDEO_MODE_FORMAT_REG(pipe), 0x0000000F); - - /* lane_count = 3 */ - REG_WRITE(MIPI_DSI_FUNC_PRG_REG(pipe), 0x00000200 | lane_count); - - mdfld_mipi_set_video_timing(dsi_config, pipe); -} - -static void mdfld_set_pipe_timing(struct mdfld_dsi_config *dsi_config, int pipe) -{ - struct drm_device *dev = dsi_config->dev; - struct drm_display_mode *mode = dsi_config->mode; - - REG_WRITE(HTOTAL_A, ((mode->htotal - 1) << 16) | (mode->hdisplay - 1)); - REG_WRITE(HBLANK_A, ((mode->htotal - 1) << 16) | (mode->hdisplay - 1)); - REG_WRITE(HSYNC_A, - ((mode->hsync_end - 1) << 16) | (mode->hsync_start - 1)); - - REG_WRITE(VTOTAL_A, ((mode->vtotal - 1) << 16) | (mode->vdisplay - 1)); - REG_WRITE(VBLANK_A, ((mode->vtotal - 1) << 16) | (mode->vdisplay - 1)); - REG_WRITE(VSYNC_A, - ((mode->vsync_end - 1) << 16) | (mode->vsync_start - 1)); - - REG_WRITE(PIPEASRC, - ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1)); -} -/* End for TC35876X */ - -void mdfld_dsi_dpi_mode_set(struct drm_encoder *encoder, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) -{ - struct mdfld_dsi_encoder *dsi_encoder = mdfld_dsi_encoder(encoder); - struct mdfld_dsi_dpi_output *dpi_output = - MDFLD_DSI_DPI_OUTPUT(dsi_encoder); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_encoder_get_config(dsi_encoder); - struct drm_device *dev = dsi_config->dev; - struct drm_psb_private *dev_priv = dev->dev_private; - int pipe = mdfld_dsi_encoder_get_pipe(dsi_encoder); - u32 pipeconf_reg = PIPEACONF; - u32 dspcntr_reg = DSPACNTR; - u32 pipeconf, dspcntr; - - u32 mipi = MIPI_PORT_EN | PASS_FROM_SPHY_TO_AFE | SEL_FLOPPED_HSTX; - - if (WARN_ON(pipe < 0)) - return; - - pipeconf = dev_priv->pipeconf[pipe]; - dspcntr = dev_priv->dspcntr[pipe]; - - if (pipe) { - pipeconf_reg = PIPECCONF; - dspcntr_reg = DSPCCNTR; - } else { - if (mdfld_get_panel_type(dev, pipe) == TC35876X) - mipi &= (~0x03); /* Use all four lanes */ - else - mipi |= 2; - } - - /*start up display island if it was shutdown*/ - if (!gma_power_begin(dev, true)) - return; - - if (mdfld_get_panel_type(dev, pipe) == TC35876X) { - /* - * The following logic is required to reset the bridge and - * configure. This also starts the DSI clock at 200MHz. - */ - tc35876x_set_bridge_reset_state(dev, 0); /*Pull High Reset */ - tc35876x_toshiba_bridge_panel_on(dev); - udelay(100); - /* Now start the DSI clock */ - REG_WRITE(MRST_DPLL_A, 0x00); - REG_WRITE(MRST_FPA0, 0xC1); - REG_WRITE(MRST_DPLL_A, 0x00800000); - udelay(500); - REG_WRITE(MRST_DPLL_A, 0x80800000); - - if (REG_BIT_WAIT(pipeconf_reg, 1, 29)) - dev_err(&dev->pdev->dev, "%s: DSI PLL lock timeout\n", - __func__); - - REG_WRITE(MIPI_DPHY_PARAM_REG(pipe), 0x2A0c6008); - - mipi_set_properties(dsi_config, pipe); - mdfld_mipi_config(dsi_config, pipe); - mdfld_set_pipe_timing(dsi_config, pipe); - - REG_WRITE(DSPABASE, 0x00); - REG_WRITE(DSPASIZE, - ((mode->vdisplay - 1) << 16) | (mode->hdisplay - 1)); - - REG_WRITE(DSPACNTR, 0x98000000); - REG_WRITE(DSPASURF, 0x00); - - REG_WRITE(VGACNTRL, 0x80000000); - REG_WRITE(DEVICE_READY_REG, 0x00000001); - - REG_WRITE(MIPI_PORT_CONTROL(pipe), 0x80810000); - } else { - /*set up mipi port FIXME: do at init time */ - REG_WRITE(MIPI_PORT_CONTROL(pipe), mipi); - } - REG_READ(MIPI_PORT_CONTROL(pipe)); - - if (mdfld_get_panel_type(dev, pipe) == TMD_VID) { - /* NOP */ - } else if (mdfld_get_panel_type(dev, pipe) == TC35876X) { - /* set up DSI controller DPI interface */ - mdfld_dsi_dpi_controller_init(dsi_config, pipe); - - /* Configure MIPI Bridge and Panel */ - tc35876x_configure_lvds_bridge(dev); - dev_priv->dpi_panel_on[pipe] = true; - } else { - /*turn on DPI interface*/ - mdfld_dsi_dpi_turn_on(dpi_output, pipe); - } - - /*set up pipe*/ - REG_WRITE(pipeconf_reg, pipeconf); - REG_READ(pipeconf_reg); - - /*set up display plane*/ - REG_WRITE(dspcntr_reg, dspcntr); - REG_READ(dspcntr_reg); - - msleep(20); /* FIXME: this should wait for vblank */ - - if (mdfld_get_panel_type(dev, pipe) == TMD_VID) { - /* NOP */ - } else if (mdfld_get_panel_type(dev, pipe) == TC35876X) { - mdfld_dsi_dpi_turn_on(dpi_output, pipe); - } else { - /* init driver ic */ - mdfld_dsi_tpo_ic_init(dsi_config, pipe); - /*init backlight*/ - mdfld_dsi_brightness_init(dsi_config, pipe); - } - - gma_power_end(dev); -} - -/* - * Init DSI DPI encoder. - * Allocate an mdfld_dsi_encoder and attach it to given @dsi_connector - * return pointer of newly allocated DPI encoder, NULL on error - */ -struct mdfld_dsi_encoder *mdfld_dsi_dpi_init(struct drm_device *dev, - struct mdfld_dsi_connector *dsi_connector, - const struct panel_funcs *p_funcs) -{ - struct mdfld_dsi_dpi_output *dpi_output = NULL; - struct mdfld_dsi_config *dsi_config; - struct drm_connector *connector = NULL; - struct drm_encoder *encoder = NULL; - int pipe; - u32 data; - int ret; - - pipe = dsi_connector->pipe; - - if (mdfld_get_panel_type(dev, pipe) != TC35876X) { - dsi_config = mdfld_dsi_get_config(dsi_connector); - - /* panel hard-reset */ - if (p_funcs->reset) { - ret = p_funcs->reset(dev, pipe); - if (ret) { - DRM_ERROR("Panel %d hard-reset failed\n", pipe); - return NULL; - } - } - - /* panel drvIC init */ - if (p_funcs->drv_ic_init) - p_funcs->drv_ic_init(dsi_config, pipe); - - /* panel power mode detect */ - ret = mdfld_dsi_get_power_mode(dsi_config, &data, false); - if (ret) { - DRM_ERROR("Panel %d get power mode failed\n", pipe); - dsi_connector->status = connector_status_disconnected; - } else { - DRM_INFO("pipe %d power mode 0x%x\n", pipe, data); - dsi_connector->status = connector_status_connected; - } - } - - dpi_output = kzalloc(sizeof(struct mdfld_dsi_dpi_output), GFP_KERNEL); - if (!dpi_output) { - DRM_ERROR("No memory\n"); - return NULL; - } - - dpi_output->panel_on = 0; - dpi_output->dev = dev; - if (mdfld_get_panel_type(dev, pipe) != TC35876X) - dpi_output->p_funcs = p_funcs; - dpi_output->first_boot = 1; - - /*get fixed mode*/ - dsi_config = mdfld_dsi_get_config(dsi_connector); - - /*create drm encoder object*/ - connector = &dsi_connector->base.base; - encoder = &dpi_output->base.base.base; - drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_LVDS); - drm_encoder_helper_add(encoder, - p_funcs->encoder_helper_funcs); - - /*attach to given connector*/ - drm_connector_attach_encoder(connector, encoder); - - /*set possible crtcs and clones*/ - if (dsi_connector->pipe) { - encoder->possible_crtcs = (1 << 2); - encoder->possible_clones = 0; - } else { - encoder->possible_crtcs = (1 << 0); - encoder->possible_clones = 0; - } - - dsi_connector->base.encoder = &dpi_output->base.base; - - return &dpi_output->base; -} diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_dpi.h b/drivers/gpu/drm/gma500/mdfld_dsi_dpi.h deleted file mode 100644 index 2b40663e1696..000000000000 --- a/drivers/gpu/drm/gma500/mdfld_dsi_dpi.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright © 2010 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * jim liu - * Jackie Li - */ - -#ifndef __MDFLD_DSI_DPI_H__ -#define __MDFLD_DSI_DPI_H__ - -#include "mdfld_dsi_output.h" -#include "mdfld_output.h" - -struct mdfld_dsi_dpi_timing { - u16 hsync_count; - u16 hbp_count; - u16 hfp_count; - u16 hactive_count; - u16 vsync_count; - u16 vbp_count; - u16 vfp_count; -}; - -struct mdfld_dsi_dpi_output { - struct mdfld_dsi_encoder base; - struct drm_device *dev; - - int panel_on; - int first_boot; - - const struct panel_funcs *p_funcs; -}; - -#define MDFLD_DSI_DPI_OUTPUT(dsi_encoder)\ - container_of(dsi_encoder, struct mdfld_dsi_dpi_output, base) - -/* Export functions */ -extern int mdfld_dsi_dpi_timing_calculation(struct drm_display_mode *mode, - struct mdfld_dsi_dpi_timing *dpi_timing, - int num_lane, int bpp); -extern struct mdfld_dsi_encoder *mdfld_dsi_dpi_init(struct drm_device *dev, - struct mdfld_dsi_connector *dsi_connector, - const struct panel_funcs *p_funcs); - -/* MDFLD DPI helper functions */ -extern void mdfld_dsi_dpi_dpms(struct drm_encoder *encoder, int mode); -extern bool mdfld_dsi_dpi_mode_fixup(struct drm_encoder *encoder, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode); -extern void mdfld_dsi_dpi_prepare(struct drm_encoder *encoder); -extern void mdfld_dsi_dpi_commit(struct drm_encoder *encoder); -extern void mdfld_dsi_dpi_mode_set(struct drm_encoder *encoder, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode); -extern void mdfld_dsi_dpi_turn_on(struct mdfld_dsi_dpi_output *output, - int pipe); -extern void mdfld_dsi_dpi_controller_init(struct mdfld_dsi_config *dsi_config, - int pipe); -#endif /*__MDFLD_DSI_DPI_H__*/ diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_output.c b/drivers/gpu/drm/gma500/mdfld_dsi_output.c deleted file mode 100644 index 4aab76613bd9..000000000000 --- a/drivers/gpu/drm/gma500/mdfld_dsi_output.c +++ /dev/null @@ -1,603 +0,0 @@ -/* - * Copyright © 2010 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * jim liu - * Jackie Li - */ - -#include -#include -#include -#include - -#include - -#include "mdfld_dsi_dpi.h" -#include "mdfld_dsi_output.h" -#include "mdfld_dsi_pkg_sender.h" -#include "mdfld_output.h" -#include "tc35876x-dsi-lvds.h" - -/* get the LABC from command line. */ -static int LABC_control = 1; - -#ifdef MODULE -module_param(LABC_control, int, 0644); -#else - -static int __init parse_LABC_control(char *arg) -{ - /* LABC control can be passed in as a cmdline parameter */ - /* to enable this feature add LABC=1 to cmdline */ - /* to disable this feature add LABC=0 to cmdline */ - if (!arg) - return -EINVAL; - - if (!strcasecmp(arg, "0")) - LABC_control = 0; - else if (!strcasecmp(arg, "1")) - LABC_control = 1; - - return 0; -} -early_param("LABC", parse_LABC_control); -#endif - -/** - * Check and see if the generic control or data buffer is empty and ready. - */ -void mdfld_dsi_gen_fifo_ready(struct drm_device *dev, u32 gen_fifo_stat_reg, - u32 fifo_stat) -{ - u32 GEN_BF_time_out_count; - - /* Check MIPI Adatper command registers */ - for (GEN_BF_time_out_count = 0; - GEN_BF_time_out_count < GEN_FB_TIME_OUT; - GEN_BF_time_out_count++) { - if ((REG_READ(gen_fifo_stat_reg) & fifo_stat) == fifo_stat) - break; - udelay(100); - } - - if (GEN_BF_time_out_count == GEN_FB_TIME_OUT) - DRM_ERROR("mdfld_dsi_gen_fifo_ready, Timeout. gen_fifo_stat_reg = 0x%x.\n", - gen_fifo_stat_reg); -} - -/** - * Manage the DSI MIPI keyboard and display brightness. - * FIXME: this is exported to OSPM code. should work out an specific - * display interface to OSPM. - */ - -void mdfld_dsi_brightness_init(struct mdfld_dsi_config *dsi_config, int pipe) -{ - struct mdfld_dsi_pkg_sender *sender = - mdfld_dsi_get_pkg_sender(dsi_config); - struct drm_device *dev; - struct drm_psb_private *dev_priv; - u32 gen_ctrl_val; - - if (!sender) { - DRM_ERROR("No sender found\n"); - return; - } - - dev = sender->dev; - dev_priv = dev->dev_private; - - /* Set default display backlight value to 85% (0xd8)*/ - mdfld_dsi_send_mcs_short(sender, write_display_brightness, 0xd8, 1, - true); - - /* Set minimum brightness setting of CABC function to 20% (0x33)*/ - mdfld_dsi_send_mcs_short(sender, write_cabc_min_bright, 0x33, 1, true); - - /* Enable backlight or/and LABC */ - gen_ctrl_val = BRIGHT_CNTL_BLOCK_ON | DISPLAY_DIMMING_ON | - BACKLIGHT_ON; - if (LABC_control == 1) - gen_ctrl_val |= DISPLAY_DIMMING_ON | DISPLAY_BRIGHTNESS_AUTO - | GAMMA_AUTO; - - if (LABC_control == 1) - gen_ctrl_val |= AMBIENT_LIGHT_SENSE_ON; - - dev_priv->mipi_ctrl_display = gen_ctrl_val; - - mdfld_dsi_send_mcs_short(sender, write_ctrl_display, (u8)gen_ctrl_val, - 1, true); - - mdfld_dsi_send_mcs_short(sender, write_ctrl_cabc, UI_IMAGE, 1, true); -} - -void mdfld_dsi_brightness_control(struct drm_device *dev, int pipe, int level) -{ - struct mdfld_dsi_pkg_sender *sender; - struct drm_psb_private *dev_priv; - struct mdfld_dsi_config *dsi_config; - u32 gen_ctrl_val = 0; - int p_type = TMD_VID; - - if (!dev || (pipe != 0 && pipe != 2)) { - DRM_ERROR("Invalid parameter\n"); - return; - } - - p_type = mdfld_get_panel_type(dev, 0); - - dev_priv = dev->dev_private; - - if (pipe) - dsi_config = dev_priv->dsi_configs[1]; - else - dsi_config = dev_priv->dsi_configs[0]; - - sender = mdfld_dsi_get_pkg_sender(dsi_config); - - if (!sender) { - DRM_ERROR("No sender found\n"); - return; - } - - gen_ctrl_val = (level * 0xff / MDFLD_DSI_BRIGHTNESS_MAX_LEVEL) & 0xff; - - dev_dbg(sender->dev->dev, "pipe = %d, gen_ctrl_val = %d.\n", - pipe, gen_ctrl_val); - - if (p_type == TMD_VID) { - /* Set display backlight value */ - mdfld_dsi_send_mcs_short(sender, tmd_write_display_brightness, - (u8)gen_ctrl_val, 1, true); - } else { - /* Set display backlight value */ - mdfld_dsi_send_mcs_short(sender, write_display_brightness, - (u8)gen_ctrl_val, 1, true); - - /* Enable backlight control */ - if (level == 0) - gen_ctrl_val = 0; - else - gen_ctrl_val = dev_priv->mipi_ctrl_display; - - mdfld_dsi_send_mcs_short(sender, write_ctrl_display, - (u8)gen_ctrl_val, 1, true); - } -} - -static int mdfld_dsi_get_panel_status(struct mdfld_dsi_config *dsi_config, - u8 dcs, u32 *data, bool hs) -{ - struct mdfld_dsi_pkg_sender *sender - = mdfld_dsi_get_pkg_sender(dsi_config); - - if (!sender || !data) { - DRM_ERROR("Invalid parameter\n"); - return -EINVAL; - } - - return mdfld_dsi_read_mcs(sender, dcs, data, 1, hs); -} - -int mdfld_dsi_get_power_mode(struct mdfld_dsi_config *dsi_config, u32 *mode, - bool hs) -{ - if (!dsi_config || !mode) { - DRM_ERROR("Invalid parameter\n"); - return -EINVAL; - } - - return mdfld_dsi_get_panel_status(dsi_config, 0x0a, mode, hs); -} - -/* - * NOTE: this function was used by OSPM. - * TODO: will be removed later, should work out display interfaces for OSPM - */ -void mdfld_dsi_controller_init(struct mdfld_dsi_config *dsi_config, int pipe) -{ - if (!dsi_config || ((pipe != 0) && (pipe != 2))) { - DRM_ERROR("Invalid parameters\n"); - return; - } - - mdfld_dsi_dpi_controller_init(dsi_config, pipe); -} - -static void mdfld_dsi_connector_save(struct drm_connector *connector) -{ -} - -static void mdfld_dsi_connector_restore(struct drm_connector *connector) -{ -} - -/* FIXME: start using the force parameter */ -static enum drm_connector_status -mdfld_dsi_connector_detect(struct drm_connector *connector, bool force) -{ - struct mdfld_dsi_connector *dsi_connector - = mdfld_dsi_connector(connector); - - dsi_connector->status = connector_status_connected; - - return dsi_connector->status; -} - -static int mdfld_dsi_connector_set_property(struct drm_connector *connector, - struct drm_property *property, - uint64_t value) -{ - struct drm_encoder *encoder = connector->encoder; - - if (!strcmp(property->name, "scaling mode") && encoder) { - struct gma_crtc *gma_crtc = to_gma_crtc(encoder->crtc); - bool centerechange; - uint64_t val; - - if (!gma_crtc) - goto set_prop_error; - - switch (value) { - case DRM_MODE_SCALE_FULLSCREEN: - break; - case DRM_MODE_SCALE_NO_SCALE: - break; - case DRM_MODE_SCALE_ASPECT: - break; - default: - goto set_prop_error; - } - - if (drm_object_property_get_value(&connector->base, property, &val)) - goto set_prop_error; - - if (val == value) - goto set_prop_done; - - if (drm_object_property_set_value(&connector->base, - property, value)) - goto set_prop_error; - - centerechange = (val == DRM_MODE_SCALE_NO_SCALE) || - (value == DRM_MODE_SCALE_NO_SCALE); - - if (gma_crtc->saved_mode.hdisplay != 0 && - gma_crtc->saved_mode.vdisplay != 0) { - if (centerechange) { - if (!drm_crtc_helper_set_mode(encoder->crtc, - &gma_crtc->saved_mode, - encoder->crtc->x, - encoder->crtc->y, - encoder->crtc->primary->fb)) - goto set_prop_error; - } else { - const struct drm_encoder_helper_funcs *funcs = - encoder->helper_private; - funcs->mode_set(encoder, - &gma_crtc->saved_mode, - &gma_crtc->saved_adjusted_mode); - } - } - } else if (!strcmp(property->name, "backlight") && encoder) { - if (drm_object_property_set_value(&connector->base, property, - value)) - goto set_prop_error; - else - gma_backlight_set(encoder->dev, value); - } -set_prop_done: - return 0; -set_prop_error: - return -1; -} - -static void mdfld_dsi_connector_destroy(struct drm_connector *connector) -{ - struct mdfld_dsi_connector *dsi_connector = - mdfld_dsi_connector(connector); - struct mdfld_dsi_pkg_sender *sender; - - if (!dsi_connector) - return; - drm_connector_unregister(connector); - drm_connector_cleanup(connector); - sender = dsi_connector->pkg_sender; - mdfld_dsi_pkg_sender_destroy(sender); - kfree(dsi_connector); -} - -static int mdfld_dsi_connector_get_modes(struct drm_connector *connector) -{ - struct mdfld_dsi_connector *dsi_connector = - mdfld_dsi_connector(connector); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_get_config(dsi_connector); - struct drm_display_mode *fixed_mode = dsi_config->fixed_mode; - struct drm_display_mode *dup_mode = NULL; - struct drm_device *dev = connector->dev; - - if (fixed_mode) { - dev_dbg(dev->dev, "fixed_mode %dx%d\n", - fixed_mode->hdisplay, fixed_mode->vdisplay); - dup_mode = drm_mode_duplicate(dev, fixed_mode); - drm_mode_probed_add(connector, dup_mode); - return 1; - } - DRM_ERROR("Didn't get any modes!\n"); - return 0; -} - -static enum drm_mode_status mdfld_dsi_connector_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) -{ - struct mdfld_dsi_connector *dsi_connector = - mdfld_dsi_connector(connector); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_get_config(dsi_connector); - struct drm_display_mode *fixed_mode = dsi_config->fixed_mode; - - if (mode->flags & DRM_MODE_FLAG_DBLSCAN) - return MODE_NO_DBLESCAN; - - if (mode->flags & DRM_MODE_FLAG_INTERLACE) - return MODE_NO_INTERLACE; - - /** - * FIXME: current DC has no fitting unit, reject any mode setting - * request - * Will figure out a way to do up-scaling(panel fitting) later. - **/ - if (fixed_mode) { - if (mode->hdisplay != fixed_mode->hdisplay) - return MODE_PANEL; - - if (mode->vdisplay != fixed_mode->vdisplay) - return MODE_PANEL; - } - - return MODE_OK; -} - -static struct drm_encoder *mdfld_dsi_connector_best_encoder( - struct drm_connector *connector) -{ - struct mdfld_dsi_connector *dsi_connector = - mdfld_dsi_connector(connector); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_get_config(dsi_connector); - return &dsi_config->encoder->base.base; -} - -/*DSI connector funcs*/ -static const struct drm_connector_funcs mdfld_dsi_connector_funcs = { - .dpms = drm_helper_connector_dpms, - .detect = mdfld_dsi_connector_detect, - .fill_modes = drm_helper_probe_single_connector_modes, - .set_property = mdfld_dsi_connector_set_property, - .destroy = mdfld_dsi_connector_destroy, -}; - -/*DSI connector helper funcs*/ -static const struct drm_connector_helper_funcs - mdfld_dsi_connector_helper_funcs = { - .get_modes = mdfld_dsi_connector_get_modes, - .mode_valid = mdfld_dsi_connector_mode_valid, - .best_encoder = mdfld_dsi_connector_best_encoder, -}; - -static int mdfld_dsi_get_default_config(struct drm_device *dev, - struct mdfld_dsi_config *config, int pipe) -{ - if (!dev || !config) { - DRM_ERROR("Invalid parameters"); - return -EINVAL; - } - - config->bpp = 24; - if (mdfld_get_panel_type(dev, pipe) == TC35876X) - config->lane_count = 4; - else - config->lane_count = 2; - config->channel_num = 0; - - if (mdfld_get_panel_type(dev, pipe) == TMD_VID) - config->video_mode = MDFLD_DSI_VIDEO_NON_BURST_MODE_SYNC_PULSE; - else if (mdfld_get_panel_type(dev, pipe) == TC35876X) - config->video_mode = - MDFLD_DSI_VIDEO_NON_BURST_MODE_SYNC_EVENTS; - else - config->video_mode = MDFLD_DSI_VIDEO_BURST_MODE; - - return 0; -} - -int mdfld_dsi_panel_reset(struct drm_device *ddev, int pipe) -{ - struct device *dev = ddev->dev; - struct gpio_desc *gpiod; - - /* - * Raise the GPIO reset line for the corresponding pipe to HIGH, - * this is probably because it is active low so this takes the - * respective pipe out of reset. (We have no code to put it back - * into reset in this driver.) - */ - switch (pipe) { - case 0: - gpiod = gpiod_get(dev, "dsi-pipe0-reset", GPIOD_OUT_HIGH); - if (IS_ERR(gpiod)) - return PTR_ERR(gpiod); - break; - case 2: - gpiod = gpiod_get(dev, "dsi-pipe2-reset", GPIOD_OUT_HIGH); - if (IS_ERR(gpiod)) - return PTR_ERR(gpiod); - break; - default: - DRM_DEV_ERROR(dev, "Invalid output pipe\n"); - return -EINVAL; - } - gpiod_put(gpiod); - - /* Flush posted writes on the device */ - gpiod = gpiod_get(dev, "dsi-pipe0-reset", GPIOD_ASIS); - if (IS_ERR(gpiod)) - return PTR_ERR(gpiod); - gpiod_get_value(gpiod); - gpiod_put(gpiod); - - return 0; -} - -/* - * MIPI output init - * @dev drm device - * @pipe pipe number. 0 or 2 - * @config - * - * Do the initialization of a MIPI output, including create DRM mode objects - * initialization of DSI output on @pipe - */ -void mdfld_dsi_output_init(struct drm_device *dev, - int pipe, - const struct panel_funcs *p_vid_funcs) -{ - struct mdfld_dsi_config *dsi_config; - struct mdfld_dsi_connector *dsi_connector; - struct drm_connector *connector; - struct mdfld_dsi_encoder *encoder; - struct drm_psb_private *dev_priv = dev->dev_private; - struct panel_info dsi_panel_info; - u32 width_mm, height_mm; - - dev_dbg(dev->dev, "init DSI output on pipe %d\n", pipe); - - if (pipe != 0 && pipe != 2) { - DRM_ERROR("Invalid parameter\n"); - return; - } - - /*create a new connector*/ - dsi_connector = kzalloc(sizeof(struct mdfld_dsi_connector), GFP_KERNEL); - if (!dsi_connector) { - DRM_ERROR("No memory"); - return; - } - - dsi_connector->pipe = pipe; - - dsi_config = kzalloc(sizeof(struct mdfld_dsi_config), - GFP_KERNEL); - if (!dsi_config) { - DRM_ERROR("cannot allocate memory for DSI config\n"); - goto dsi_init_err0; - } - mdfld_dsi_get_default_config(dev, dsi_config, pipe); - - dsi_connector->private = dsi_config; - - dsi_config->changed = 1; - dsi_config->dev = dev; - - dsi_config->fixed_mode = p_vid_funcs->get_config_mode(dev); - if (p_vid_funcs->get_panel_info(dev, pipe, &dsi_panel_info)) - goto dsi_init_err0; - - width_mm = dsi_panel_info.width_mm; - height_mm = dsi_panel_info.height_mm; - - dsi_config->mode = dsi_config->fixed_mode; - dsi_config->connector = dsi_connector; - - if (!dsi_config->fixed_mode) { - DRM_ERROR("No panel fixed mode was found\n"); - goto dsi_init_err0; - } - - if (pipe && dev_priv->dsi_configs[0]) { - dsi_config->dvr_ic_inited = 0; - dev_priv->dsi_configs[1] = dsi_config; - } else if (pipe == 0) { - dsi_config->dvr_ic_inited = 1; - dev_priv->dsi_configs[0] = dsi_config; - } else { - DRM_ERROR("Trying to init MIPI1 before MIPI0\n"); - goto dsi_init_err0; - } - - - connector = &dsi_connector->base.base; - dsi_connector->base.save = mdfld_dsi_connector_save; - dsi_connector->base.restore = mdfld_dsi_connector_restore; - - drm_connector_init(dev, connector, &mdfld_dsi_connector_funcs, - DRM_MODE_CONNECTOR_LVDS); - drm_connector_helper_add(connector, &mdfld_dsi_connector_helper_funcs); - - connector->display_info.subpixel_order = SubPixelHorizontalRGB; - connector->display_info.width_mm = width_mm; - connector->display_info.height_mm = height_mm; - connector->interlace_allowed = false; - connector->doublescan_allowed = false; - - /*attach properties*/ - drm_object_attach_property(&connector->base, - dev->mode_config.scaling_mode_property, - DRM_MODE_SCALE_FULLSCREEN); - drm_object_attach_property(&connector->base, - dev_priv->backlight_property, - MDFLD_DSI_BRIGHTNESS_MAX_LEVEL); - - /*init DSI package sender on this output*/ - if (mdfld_dsi_pkg_sender_init(dsi_connector, pipe)) { - DRM_ERROR("Package Sender initialization failed on pipe %d\n", - pipe); - goto dsi_init_err0; - } - - encoder = mdfld_dsi_dpi_init(dev, dsi_connector, p_vid_funcs); - if (!encoder) { - DRM_ERROR("Create DPI encoder failed\n"); - goto dsi_init_err1; - } - encoder->private = dsi_config; - dsi_config->encoder = encoder; - encoder->base.type = (pipe == 0) ? INTEL_OUTPUT_MIPI : - INTEL_OUTPUT_MIPI2; - drm_connector_register(connector); - return; - - /*TODO: add code to destroy outputs on error*/ -dsi_init_err1: - /*destroy sender*/ - mdfld_dsi_pkg_sender_destroy(dsi_connector->pkg_sender); - - drm_connector_cleanup(connector); - - kfree(dsi_config->fixed_mode); - kfree(dsi_config); -dsi_init_err0: - kfree(dsi_connector); -} diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_output.h b/drivers/gpu/drm/gma500/mdfld_dsi_output.h deleted file mode 100644 index 5c0db3c2903f..000000000000 --- a/drivers/gpu/drm/gma500/mdfld_dsi_output.h +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Copyright © 2010 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * jim liu - * Jackie Li - */ - -#ifndef __MDFLD_DSI_OUTPUT_H__ -#define __MDFLD_DSI_OUTPUT_H__ - -#include - -#include - -#include -#include -#include - -#include "mdfld_output.h" -#include "psb_drv.h" -#include "psb_intel_drv.h" -#include "psb_intel_reg.h" - -#define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end)) -#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end)) -#define FLD_GET(val, start, end) (((val) & FLD_MASK(start, end)) >> (end)) -#define FLD_MOD(orig, val, start, end) \ - (((orig) & ~FLD_MASK(start, end)) | FLD_VAL(val, start, end)) - -#define REG_FLD_MOD(reg, val, start, end) \ - REG_WRITE(reg, FLD_MOD(REG_READ(reg), val, start, end)) - -static inline int REGISTER_FLD_WAIT(struct drm_device *dev, u32 reg, - u32 val, int start, int end) -{ - int t = 100000; - - while (FLD_GET(REG_READ(reg), start, end) != val) { - if (--t == 0) - return 1; - } - - return 0; -} - -#define REG_FLD_WAIT(reg, val, start, end) \ - REGISTER_FLD_WAIT(dev, reg, val, start, end) - -#define REG_BIT_WAIT(reg, val, bitnum) \ - REGISTER_FLD_WAIT(dev, reg, val, bitnum, bitnum) - -#define MDFLD_DSI_BRIGHTNESS_MAX_LEVEL 100 - -#ifdef DEBUG -#define CHECK_PIPE(pipe) ({ \ - const typeof(pipe) __pipe = (pipe); \ - BUG_ON(__pipe != 0 && __pipe != 2); \ - __pipe; }) -#else -#define CHECK_PIPE(pipe) (pipe) -#endif - -/* - * Actual MIPIA->MIPIC reg offset is 0x800, value 0x400 is valid for 0 and 2 - */ -#define REG_OFFSET(pipe) (CHECK_PIPE(pipe) * 0x400) - -/* mdfld DSI controller registers */ -#define MIPI_DEVICE_READY_REG(pipe) (0xb000 + REG_OFFSET(pipe)) -#define MIPI_INTR_STAT_REG(pipe) (0xb004 + REG_OFFSET(pipe)) -#define MIPI_INTR_EN_REG(pipe) (0xb008 + REG_OFFSET(pipe)) -#define MIPI_DSI_FUNC_PRG_REG(pipe) (0xb00c + REG_OFFSET(pipe)) -#define MIPI_HS_TX_TIMEOUT_REG(pipe) (0xb010 + REG_OFFSET(pipe)) -#define MIPI_LP_RX_TIMEOUT_REG(pipe) (0xb014 + REG_OFFSET(pipe)) -#define MIPI_TURN_AROUND_TIMEOUT_REG(pipe) (0xb018 + REG_OFFSET(pipe)) -#define MIPI_DEVICE_RESET_TIMER_REG(pipe) (0xb01c + REG_OFFSET(pipe)) -#define MIPI_DPI_RESOLUTION_REG(pipe) (0xb020 + REG_OFFSET(pipe)) -#define MIPI_DBI_FIFO_THROTTLE_REG(pipe) (0xb024 + REG_OFFSET(pipe)) -#define MIPI_HSYNC_COUNT_REG(pipe) (0xb028 + REG_OFFSET(pipe)) -#define MIPI_HBP_COUNT_REG(pipe) (0xb02c + REG_OFFSET(pipe)) -#define MIPI_HFP_COUNT_REG(pipe) (0xb030 + REG_OFFSET(pipe)) -#define MIPI_HACTIVE_COUNT_REG(pipe) (0xb034 + REG_OFFSET(pipe)) -#define MIPI_VSYNC_COUNT_REG(pipe) (0xb038 + REG_OFFSET(pipe)) -#define MIPI_VBP_COUNT_REG(pipe) (0xb03c + REG_OFFSET(pipe)) -#define MIPI_VFP_COUNT_REG(pipe) (0xb040 + REG_OFFSET(pipe)) -#define MIPI_HIGH_LOW_SWITCH_COUNT_REG(pipe) (0xb044 + REG_OFFSET(pipe)) -#define MIPI_DPI_CONTROL_REG(pipe) (0xb048 + REG_OFFSET(pipe)) -#define MIPI_DPI_DATA_REG(pipe) (0xb04c + REG_OFFSET(pipe)) -#define MIPI_INIT_COUNT_REG(pipe) (0xb050 + REG_OFFSET(pipe)) -#define MIPI_MAX_RETURN_PACK_SIZE_REG(pipe) (0xb054 + REG_OFFSET(pipe)) -#define MIPI_VIDEO_MODE_FORMAT_REG(pipe) (0xb058 + REG_OFFSET(pipe)) -#define MIPI_EOT_DISABLE_REG(pipe) (0xb05c + REG_OFFSET(pipe)) -#define MIPI_LP_BYTECLK_REG(pipe) (0xb060 + REG_OFFSET(pipe)) -#define MIPI_LP_GEN_DATA_REG(pipe) (0xb064 + REG_OFFSET(pipe)) -#define MIPI_HS_GEN_DATA_REG(pipe) (0xb068 + REG_OFFSET(pipe)) -#define MIPI_LP_GEN_CTRL_REG(pipe) (0xb06c + REG_OFFSET(pipe)) -#define MIPI_HS_GEN_CTRL_REG(pipe) (0xb070 + REG_OFFSET(pipe)) -#define MIPI_GEN_FIFO_STAT_REG(pipe) (0xb074 + REG_OFFSET(pipe)) -#define MIPI_HS_LS_DBI_ENABLE_REG(pipe) (0xb078 + REG_OFFSET(pipe)) -#define MIPI_DPHY_PARAM_REG(pipe) (0xb080 + REG_OFFSET(pipe)) -#define MIPI_DBI_BW_CTRL_REG(pipe) (0xb084 + REG_OFFSET(pipe)) -#define MIPI_CLK_LANE_SWITCH_TIME_CNT_REG(pipe) (0xb088 + REG_OFFSET(pipe)) - -#define MIPI_CTRL_REG(pipe) (0xb104 + REG_OFFSET(pipe)) -#define MIPI_DATA_ADD_REG(pipe) (0xb108 + REG_OFFSET(pipe)) -#define MIPI_DATA_LEN_REG(pipe) (0xb10c + REG_OFFSET(pipe)) -#define MIPI_CMD_ADD_REG(pipe) (0xb110 + REG_OFFSET(pipe)) -#define MIPI_CMD_LEN_REG(pipe) (0xb114 + REG_OFFSET(pipe)) - -/* non-uniform reg offset */ -#define MIPI_PORT_CONTROL(pipe) (CHECK_PIPE(pipe) ? MIPI_C : MIPI) - -#define DSI_DEVICE_READY (0x1) -#define DSI_POWER_STATE_ULPS_ENTER (0x2 << 1) -#define DSI_POWER_STATE_ULPS_EXIT (0x1 << 1) -#define DSI_POWER_STATE_ULPS_OFFSET (0x1) - - -#define DSI_ONE_DATA_LANE (0x1) -#define DSI_TWO_DATA_LANE (0x2) -#define DSI_THREE_DATA_LANE (0X3) -#define DSI_FOUR_DATA_LANE (0x4) -#define DSI_DPI_VIRT_CHANNEL_OFFSET (0x3) -#define DSI_DBI_VIRT_CHANNEL_OFFSET (0x5) -#define DSI_DPI_COLOR_FORMAT_RGB565 (0x01 << 7) -#define DSI_DPI_COLOR_FORMAT_RGB666 (0x02 << 7) -#define DSI_DPI_COLOR_FORMAT_RGB666_UNPACK (0x03 << 7) -#define DSI_DPI_COLOR_FORMAT_RGB888 (0x04 << 7) -#define DSI_DBI_COLOR_FORMAT_OPTION2 (0x05 << 13) - -#define DSI_INTR_STATE_RXSOTERROR BIT(0) - -#define DSI_INTR_STATE_SPL_PKG_SENT BIT(30) -#define DSI_INTR_STATE_TE BIT(31) - -#define DSI_HS_TX_TIMEOUT_MASK (0xffffff) - -#define DSI_LP_RX_TIMEOUT_MASK (0xffffff) - -#define DSI_TURN_AROUND_TIMEOUT_MASK (0x3f) - -#define DSI_RESET_TIMER_MASK (0xffff) - -#define DSI_DBI_FIFO_WM_HALF (0x0) -#define DSI_DBI_FIFO_WM_QUARTER (0x1) -#define DSI_DBI_FIFO_WM_LOW (0x2) - -#define DSI_DPI_TIMING_MASK (0xffff) - -#define DSI_INIT_TIMER_MASK (0xffff) - -#define DSI_DBI_RETURN_PACK_SIZE_MASK (0x3ff) - -#define DSI_LP_BYTECLK_MASK (0x0ffff) - -#define DSI_HS_CTRL_GEN_SHORT_W0 (0x03) -#define DSI_HS_CTRL_GEN_SHORT_W1 (0x13) -#define DSI_HS_CTRL_GEN_SHORT_W2 (0x23) -#define DSI_HS_CTRL_GEN_R0 (0x04) -#define DSI_HS_CTRL_GEN_R1 (0x14) -#define DSI_HS_CTRL_GEN_R2 (0x24) -#define DSI_HS_CTRL_GEN_LONG_W (0x29) -#define DSI_HS_CTRL_MCS_SHORT_W0 (0x05) -#define DSI_HS_CTRL_MCS_SHORT_W1 (0x15) -#define DSI_HS_CTRL_MCS_R0 (0x06) -#define DSI_HS_CTRL_MCS_LONG_W (0x39) -#define DSI_HS_CTRL_VC_OFFSET (0x06) -#define DSI_HS_CTRL_WC_OFFSET (0x08) - -#define DSI_FIFO_GEN_HS_DATA_FULL BIT(0) -#define DSI_FIFO_GEN_HS_DATA_HALF_EMPTY BIT(1) -#define DSI_FIFO_GEN_HS_DATA_EMPTY BIT(2) -#define DSI_FIFO_GEN_LP_DATA_FULL BIT(8) -#define DSI_FIFO_GEN_LP_DATA_HALF_EMPTY BIT(9) -#define DSI_FIFO_GEN_LP_DATA_EMPTY BIT(10) -#define DSI_FIFO_GEN_HS_CTRL_FULL BIT(16) -#define DSI_FIFO_GEN_HS_CTRL_HALF_EMPTY BIT(17) -#define DSI_FIFO_GEN_HS_CTRL_EMPTY BIT(18) -#define DSI_FIFO_GEN_LP_CTRL_FULL BIT(24) -#define DSI_FIFO_GEN_LP_CTRL_HALF_EMPTY BIT(25) -#define DSI_FIFO_GEN_LP_CTRL_EMPTY BIT(26) -#define DSI_FIFO_DBI_EMPTY BIT(27) -#define DSI_FIFO_DPI_EMPTY BIT(28) - -#define DSI_DBI_HS_LP_SWITCH_MASK (0x1) - -#define DSI_HS_LP_SWITCH_COUNTER_OFFSET (0x0) -#define DSI_LP_HS_SWITCH_COUNTER_OFFSET (0x16) - -#define DSI_DPI_CTRL_HS_SHUTDOWN (0x00000001) -#define DSI_DPI_CTRL_HS_TURN_ON (0x00000002) - -/*dsi power modes*/ -#define DSI_POWER_MODE_DISPLAY_ON BIT(2) -#define DSI_POWER_MODE_NORMAL_ON BIT(3) -#define DSI_POWER_MODE_SLEEP_OUT BIT(4) -#define DSI_POWER_MODE_PARTIAL_ON BIT(5) -#define DSI_POWER_MODE_IDLE_ON BIT(6) - -enum { - MDFLD_DSI_VIDEO_NON_BURST_MODE_SYNC_PULSE = 1, - MDFLD_DSI_VIDEO_NON_BURST_MODE_SYNC_EVENTS = 2, - MDFLD_DSI_VIDEO_BURST_MODE = 3, -}; - -#define DSI_DPI_COMPLETE_LAST_LINE BIT(2) -#define DSI_DPI_DISABLE_BTA BIT(3) - -struct mdfld_dsi_connector { - struct gma_connector base; - - int pipe; - void *private; - void *pkg_sender; - - /* Connection status */ - enum drm_connector_status status; -}; - -struct mdfld_dsi_encoder { - struct gma_encoder base; - void *private; -}; - -/* - * DSI config, consists of one DSI connector, two DSI encoders. - * DRM will pick up on DSI encoder basing on differents configs. - */ -struct mdfld_dsi_config { - struct drm_device *dev; - struct drm_display_mode *fixed_mode; - struct drm_display_mode *mode; - - struct mdfld_dsi_connector *connector; - struct mdfld_dsi_encoder *encoder; - - int changed; - - int bpp; - int lane_count; - /*Virtual channel number for this encoder*/ - int channel_num; - /*video mode configure*/ - int video_mode; - - int dvr_ic_inited; -}; - -static inline struct mdfld_dsi_connector *mdfld_dsi_connector( - struct drm_connector *connector) -{ - struct gma_connector *gma_connector; - - gma_connector = to_gma_connector(connector); - - return container_of(gma_connector, struct mdfld_dsi_connector, base); -} - -static inline struct mdfld_dsi_encoder *mdfld_dsi_encoder( - struct drm_encoder *encoder) -{ - struct gma_encoder *gma_encoder; - - gma_encoder = to_gma_encoder(encoder); - - return container_of(gma_encoder, struct mdfld_dsi_encoder, base); -} - -static inline struct mdfld_dsi_config * - mdfld_dsi_get_config(struct mdfld_dsi_connector *connector) -{ - if (!connector) - return NULL; - return (struct mdfld_dsi_config *)connector->private; -} - -static inline void *mdfld_dsi_get_pkg_sender(struct mdfld_dsi_config *config) -{ - struct mdfld_dsi_connector *dsi_connector; - - if (!config) - return NULL; - - dsi_connector = config->connector; - - if (!dsi_connector) - return NULL; - - return dsi_connector->pkg_sender; -} - -static inline struct mdfld_dsi_config * - mdfld_dsi_encoder_get_config(struct mdfld_dsi_encoder *encoder) -{ - if (!encoder) - return NULL; - return (struct mdfld_dsi_config *)encoder->private; -} - -static inline struct mdfld_dsi_connector * - mdfld_dsi_encoder_get_connector(struct mdfld_dsi_encoder *encoder) -{ - struct mdfld_dsi_config *config; - - if (!encoder) - return NULL; - - config = mdfld_dsi_encoder_get_config(encoder); - if (!config) - return NULL; - - return config->connector; -} - -static inline void *mdfld_dsi_encoder_get_pkg_sender( - struct mdfld_dsi_encoder *encoder) -{ - struct mdfld_dsi_config *dsi_config; - - dsi_config = mdfld_dsi_encoder_get_config(encoder); - if (!dsi_config) - return NULL; - - return mdfld_dsi_get_pkg_sender(dsi_config); -} - -static inline int mdfld_dsi_encoder_get_pipe(struct mdfld_dsi_encoder *encoder) -{ - struct mdfld_dsi_connector *connector; - - if (!encoder) - return -1; - - connector = mdfld_dsi_encoder_get_connector(encoder); - if (!connector) - return -1; - return connector->pipe; -} - -/* Export functions */ -extern void mdfld_dsi_gen_fifo_ready(struct drm_device *dev, - u32 gen_fifo_stat_reg, u32 fifo_stat); -extern void mdfld_dsi_brightness_init(struct mdfld_dsi_config *dsi_config, - int pipe); -extern void mdfld_dsi_brightness_control(struct drm_device *dev, int pipe, - int level); -extern void mdfld_dsi_output_init(struct drm_device *dev, - int pipe, - const struct panel_funcs *p_vid_funcs); -extern void mdfld_dsi_controller_init(struct mdfld_dsi_config *dsi_config, - int pipe); - -extern int mdfld_dsi_get_power_mode(struct mdfld_dsi_config *dsi_config, - u32 *mode, bool hs); -extern int mdfld_dsi_panel_reset(struct drm_device *dev, int pipe); - -#endif /*__MDFLD_DSI_OUTPUT_H__*/ diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c b/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c deleted file mode 100644 index 6e0de83e9f7d..000000000000 --- a/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c +++ /dev/null @@ -1,679 +0,0 @@ -/* - * Copyright © 2010 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * Jackie Li - */ - -#include -#include - -#include